Skip to content

Commit f1fab98

Browse files
jorgectfjorgectf
authored
Merge branch 'github:main' into python_jwt
2 parents 1f1b7a5 + 3206384 commit f1fab98

File tree

3,112 files changed

+405475
-108041
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,112 files changed

+405475
-108041
lines changed

‎.codeqlmanifest.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,19 @@
44
"*/ql/lib/qlpack.yml",
55
"*/ql/test/qlpack.yml",
66
"*/ql/examples/qlpack.yml",
7-
"*/upgrades/qlpack.yml",
87
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
98
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
9+
"javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
1010
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
11+
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
12+
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",
13+
"csharp/ql/campaigns/Solorigate/test/qlpack.yml",
1114
"misc/legacy-support/*/qlpack.yml",
1215
"misc/suite-helpers/qlpack.yml",
1316
"ruby/extractor-pack/codeql-extractor.yml",
14-
"ruby/ql/consistency-queries/qlpack.yml"
17+
"ruby/ql/consistency-queries/qlpack.yml",
18+
"ql/ql/consistency-queries/qlpack.yml",
19+
"ql/extractor-pack/codeql-extractor.yml"
1520
],
1621
"versionPolicies": {
1722
"default": {

‎.gitattributes‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,9 @@
5050
*.pdb -text
5151

5252
java/ql/test/stubs/**/*.java linguist-generated=true
53-
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
53+
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
54+
55+
# Generated test files - these are synced from the standard JavaScript libraries using
56+
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
57+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
58+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

‎.github/labeler.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,6 @@ documentation:
2626
- "**/*.qhelp"
2727
- "**/*.md"
2828
- docs/**/*
29+
30+
"QL-for-QL":
31+
- ql/**/*

‎.github/workflows/check-change-note.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
99
- "!**/experimental/**"
10+
- "!ql/**"
1011

1112
jobs:
1213
check-change-note:

‎.github/workflows/codeql-analysis.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
pull-requests: read
2828

2929
steps:
30+
- name: Setup dotnet
31+
uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: 6.0.101
34+
3035
- name: Checkout repository
3136
uses: actions/checkout@v2
3237

@@ -51,7 +56,7 @@ jobs:
5156
# uses a compiled language
5257

5358
- run: |
54-
dotnet build csharp
59+
dotnet build csharp /p:UseSharedCompilation=false
5560
5661
- name: Perform CodeQL Analysis
5762
uses: github/codeql-action/analyze@main

‎.github/workflows/csv-coverage-metrics.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Publish framework coverage as metrics"
2+
3+
on:
4+
schedule:
5+
- cron: '5 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- ".github/workflows/csv-coverage-metrics.yml"
15+
16+
jobs:
17+
publish:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v2
22+
- name: Setup CodeQL
23+
uses: ./.github/actions/fetch-codeql
24+
- name: Create empty database
25+
run: |
26+
DATABASE="${{ runner.temp }}/java-database"
27+
PROJECT="${{ runner.temp }}/java-project"
28+
mkdir -p "$PROJECT/src/tmp/empty"
29+
echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
30+
codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
31+
- name: Capture coverage information
32+
run: |
33+
DATABASE="${{ runner.temp }}/java-database"
34+
codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
35+
- uses: actions/upload-artifact@v2
36+
with:
37+
name: metrics.sarif
38+
path: metrics.sarif
39+
retention-days: 20
40+
- name: Upload SARIF file
41+
uses: github/codeql-action/upload-sarif@v1
42+
with:
43+
sarif_file: metrics.sarif

‎.github/workflows/js-ml-tests.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: JS ML-powered queries tests
2+
3+
on:
4+
push:
5+
paths:
6+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
7+
- .github/workflows/js-ml-tests.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
pull_request:
12+
paths:
13+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
14+
- .github/workflows/js-ml-tests.yml
15+
16+
defaults:
17+
run:
18+
working-directory: javascript/ql/experimental/adaptivethreatmodeling
19+
20+
jobs:
21+
qlformat:
22+
name: Check QL formatting
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- uses: ./.github/actions/fetch-codeql
28+
29+
- name: Check QL formatting
30+
run: |
31+
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
32+
xargs -0 codeql query format --check-only
33+
34+
qlcompile:
35+
name: Check QL compilation
36+
runs-on: ubuntu-latest
37+
steps:
38+
- uses: actions/checkout@v2
39+
40+
- uses: ./.github/actions/fetch-codeql
41+
42+
- name: Check QL compilation
43+
run: |
44+
codeql query compile \
45+
--check-only \
46+
--ram 5120 \
47+
--additional-packs "${{ github.workspace }}" \
48+
--threads=0 \
49+
-- \
50+
lib modelbuilding src
51+
52+
qltest:
53+
name: Run QL tests
54+
runs-on: ubuntu-latest
55+
steps:
56+
- uses: actions/checkout@v2
57+
58+
- uses: ./.github/actions/fetch-codeql
59+
60+
- name: Run QL tests
61+
run: |
62+
codeql test run \
63+
--threads=0 \
64+
--ram 5120 \
65+
--additional-packs "${{ github.workspace }}" \
66+
-- \
67+
test

‎.github/workflows/mad_modelDiff.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Models as Data - Diff
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
projects:
7+
description: "The projects to generate models for"
8+
required: true
9+
default: '["netty/netty"]'
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- "java/ql/src/utils/model-generator/**/*.*"
15+
- ".github/workflows/mad_modelDiff.yml"
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
model-diff:
22+
name: Model Difference
23+
runs-on: ubuntu-latest
24+
if: github.repository == 'github/codeql'
25+
strategy:
26+
matrix:
27+
slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
28+
steps:
29+
- name: Clone github/codeql from PR
30+
uses: actions/checkout@v2
31+
if: github.event.pull_request
32+
with:
33+
path: codeql-pr
34+
- name: Clone github/codeql from main
35+
uses: actions/checkout@v2
36+
with:
37+
path: codeql-main
38+
ref: main
39+
- uses: ./codeql-main/.github/actions/fetch-codeql
40+
- name: Download database
41+
env:
42+
SLUG: ${{ matrix.slug }}
43+
run: |
44+
set -x
45+
mkdir lib-dbs
46+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
47+
projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
48+
curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
49+
unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
50+
mkdir "lib-dbs/$SHORTNAME/"
51+
mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
52+
- name: Generate Models (PR and main)
53+
run: |
54+
set -x
55+
mkdir tmp-models
56+
MODELS=`pwd`/tmp-models
57+
DATABASES=`pwd`/lib-dbs
58+
59+
analyzeDatabaseWithCheckout() {
60+
QL_VARIANT=$1
61+
DATABASE=$2
62+
cd codeql-$QL_VARIANT
63+
SHORTNAME=`basename $DATABASE`
64+
python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
65+
mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
66+
cd ..
67+
}
68+
69+
for d in $DATABASES/*/ ; do
70+
ls -1 "$d"
71+
72+
analyzeDatabaseWithCheckout "main" $d
73+
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
74+
then
75+
analyzeDatabaseWithCheckout "pr" $d
76+
fi
77+
done
78+
- name: Install diff2html
79+
if: github.event.pull_request
80+
run: |
81+
npm install -g diff2html-cli
82+
- name: Generate Model Diff
83+
if: github.event.pull_request
84+
run: |
85+
set -x
86+
MODELS=`pwd`/tmp-models
87+
ls -1 tmp-models/
88+
for m in $MODELS/*_main.qll ; do
89+
t="${m/main/"pr"}"
90+
basename=`basename $m`
91+
name="diff_${basename/_main.qll/""}"
92+
(diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
93+
done
94+
- uses: actions/upload-artifact@v2
95+
with:
96+
name: models
97+
path: tmp-models/*.qll
98+
retention-days: 20
99+
- uses: actions/upload-artifact@v2
100+
with:
101+
name: diffs
102+
path: tmp-models/*.html
103+
retention-days: 20

‎.github/workflows/mad_regenerate-models.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Regenerate framework models
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "30 2 * * *"
7+
pull_request:
8+
branches:
9+
- main
10+
paths:
11+
- ".github/workflows/mad_regenerate-models.yml"
12+
13+
jobs:
14+
regenerate-models:
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
# placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
19+
slug: ["placeholder"]
20+
ref: ["placeholder"]
21+
include:
22+
- slug: "apache/commons-io"
23+
ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
24+
exclude:
25+
- slug: "placeholder"
26+
ref: "placeholder"
27+
steps:
28+
- name: Clone self (github/codeql)
29+
uses: actions/checkout@v2
30+
- name: Setup CodeQL binaries
31+
uses: ./.github/actions/fetch-codeql
32+
- name: Clone repositories
33+
uses: actions/checkout@v2
34+
with:
35+
path: repos/${{ matrix.ref }}
36+
ref: ${{ matrix.ref }}
37+
repository: ${{ matrix.slug }}
38+
- name: Build database
39+
env:
40+
SLUG: ${{ matrix.slug }}
41+
REF: ${{ matrix.ref }}
42+
run: |
43+
mkdir dbs
44+
cd repos/${REF}
45+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
46+
codeql database create --language=java ../../dbs/${SHORTNAME}
47+
- name: Regenerate models in-place
48+
env:
49+
SLUG: ${{ matrix.slug }}
50+
run: |
51+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
52+
java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
53+
- name: Stage changes
54+
run: |
55+
find java -name "*.qll" -print0 | xargs -0 git add
56+
git status
57+
git diff --cached > models.patch
58+
- uses: actions/upload-artifact@v2
59+
with:
60+
name: patch
61+
path: models.patch
62+
retention-days: 7

0 commit comments

Comments
 (0)