Skip to content

Commit 9e9febf

Browse files
MathiasVPIdrissRio
MathiasVP
authored and
IdrissRio
committed
Bulk generator: Specify 'with-summaries', 'with-sources', and 'with-sinks' in the config file.
1 parent fb6e5a6 commit 9e9febf

File tree

3 files changed

+75
-32
lines changed

3 files changed

+75
-32
lines changed

‎cpp/misc/bulk_generation_targets.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
22
"strategy": "dca",
33
"targets": [
4-
{ "name": "openssl" },
5-
{ "name": "sqlite" }
4+
{ "name": "openssl", "with_summaries": true },
5+
{ "name": "sqlite", "with_summaries": true }
66
],
77
"destination": "cpp/ql/lib/ext/generated"
88
}

‎misc/scripts/models-as-data/bulk_generate_mad.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,18 @@ class Project(TypedDict):
4141
name: str
4242
git_repo: NotRequired[str]
4343
git_tag: NotRequired[str]
44+
with_sinks: NotRequired[bool]
45+
with_sinks: NotRequired[bool]
46+
with_summaries: NotRequired[bool]
4447

48+
def shouldGenerateSinks(project: Project) -> bool:
49+
return project.get("with_sinks", False)
50+
51+
def shouldGenerateSources(project: Project) -> bool:
52+
return project.get("with_sources", False)
53+
54+
def shouldGenerateSummaries(project: Project) -> bool:
55+
return project.get("with_summaries", False)
4556

4657
def clone_project(project: Project) -> str:
4758
"""
@@ -185,7 +196,7 @@ def build_database(
185196
return database_dir
186197

187198

188-
def generate_models(args, project: Project, database_dir: str) -> None:
199+
def generate_models(language: str, config, project: Project, database_dir: str) -> None:
189200
"""
190201
Generate models for a project.
191202
@@ -196,10 +207,11 @@ def generate_models(args, project: Project, database_dir: str) -> None:
196207
"""
197208
name = project["name"]
198209

199-
generator = mad.Generator(args.lang)
200-
generator.generateSinks = args.with_sinks
201-
generator.generateSources = args.with_sources
202-
generator.generateSummaries = args.with_summaries
210+
generator = mad.Generator(language)
211+
# Note: The argument parser converts with-sinks to with_sinks, etc.
212+
generator.generateSinks = shouldGenerateSinks(project)
213+
generator.generateSources = shouldGenerateSources(project)
214+
generator.generateSummaries = shouldGenerateSummaries(project)
203215
generator.setenvironment(database=database_dir, folder=name)
204216
generator.run()
205217

@@ -309,13 +321,14 @@ def download_dca_databases(
309321
pat,
310322
)
311323
targets = response["targets"]
324+
project_map = {project["name"]: project for project in projects}
312325
for data in targets.values():
313326
downloads = data["downloads"]
314327
analyzed_database = downloads["analyzed_database"]
315328
artifact_name = analyzed_database["artifact_name"]
316329
pretty_name = pretty_name_from_artifact_name(artifact_name)
317330

318-
if not pretty_name in [project["name"] for project in projects]:
331+
if not pretty_name in project_map:
319332
print(f"Skipping {pretty_name} as it is not in the list of projects")
320333
continue
321334

@@ -350,7 +363,7 @@ def download_dca_databases(
350363
tar_ref.extractall(artifact_unzipped_location)
351364
database_results.append(
352365
(
353-
{"name": pretty_name},
366+
project_map[pretty_name],
354367
os.path.join(
355368
artifact_unzipped_location, remove_extension(entry)
356369
),
@@ -451,7 +464,7 @@ def main(config, args) -> None:
451464

452465
for project, database_dir in database_results:
453466
if database_dir is not None:
454-
generate_models(args, project, database_dir)
467+
generate_models(language, config, project, database_dir)
455468

456469

457470
if __name__ == "__main__":
@@ -474,15 +487,6 @@ def main(config, args) -> None:
474487
parser.add_argument(
475488
"--lang", type=str, help="The language to generate models for", required=True
476489
)
477-
parser.add_argument(
478-
"--with-sources", action="store_true", help="Generate sources", required=False
479-
)
480-
parser.add_argument(
481-
"--with-sinks", action="store_true", help="Generate sinks", required=False
482-
)
483-
parser.add_argument(
484-
"--with-summaries", action="store_true", help="Generate sinks", required=False
485-
)
486490
args = parser.parse_args()
487491

488492
# Load config file

‎rust/misc/bulk_generation_targets.json‎

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,67 +4,106 @@
44
{
55
"name": "libc",
66
"git_repo": "https://github.com/rust-lang/libc",
7-
"git_tag": "0.2.172"
7+
"git_tag": "0.2.172",
8+
"with-sources": true,
9+
"with-sinks": true,
10+
"with-summaries": true
811
},
912
{
1013
"name": "log",
1114
"git_repo": "https://github.com/rust-lang/log",
12-
"git_tag": "0.4.27"
15+
"git_tag": "0.4.27",
16+
"with-sources": true,
17+
"with-sinks": true,
18+
"with-summaries": true
1319
},
1420
{
1521
"name": "memchr",
1622
"git_repo": "https://github.com/BurntSushi/memchr",
17-
"git_tag": "2.7.4"
23+
"git_tag": "2.7.4",
24+
"with-sources": true,
25+
"with-sinks": true,
26+
"with-summaries": true
1827
},
1928
{
2029
"name": "once_cell",
2130
"git_repo": "https://github.com/matklad/once_cell",
22-
"git_tag": "v1.21.3"
31+
"git_tag": "v1.21.3",
32+
"with-sources": true,
33+
"with-sinks": true,
34+
"with-summaries": true
2335
},
2436
{
2537
"name": "rand",
2638
"git_repo": "https://github.com/rust-random/rand",
27-
"git_tag": "0.9.1"
39+
"git_tag": "0.9.1",
40+
"with-sources": true,
41+
"with-sinks": true,
42+
"with-summaries": true
2843
},
2944
{
3045
"name": "smallvec",
3146
"git_repo": "https://github.com/servo/rust-smallvec",
32-
"git_tag": "v1.15.0"
47+
"git_tag": "v1.15.0",
48+
"with-sources": true,
49+
"with-sinks": true,
50+
"with-summaries": true
3351
},
3452
{
3553
"name": "serde",
3654
"git_repo": "https://github.com/serde-rs/serde",
37-
"git_tag": "v1.0.219"
55+
"git_tag": "v1.0.219",
56+
"with-sources": true,
57+
"with-sinks": true,
58+
"with-summaries": true
3859
},
3960
{
4061
"name": "tokio",
4162
"git_repo": "https://github.com/tokio-rs/tokio",
42-
"git_tag": "tokio-1.45.0"
63+
"git_tag": "tokio-1.45.0",
64+
"with-sources": true,
65+
"with-sinks": true,
66+
"with-summaries": true
4367
},
4468
{
4569
"name": "reqwest",
4670
"git_repo": "https://github.com/seanmonstar/reqwest",
47-
"git_tag": "v0.12.15"
71+
"git_tag": "v0.12.15",
72+
"with-sources": true,
73+
"with-sinks": true,
74+
"with-summaries": true
4875
},
4976
{
5077
"name": "rocket",
5178
"git_repo": "https://github.com/SergioBenitez/Rocket",
52-
"git_tag": "v0.5.1"
79+
"git_tag": "v0.5.1",
80+
"with-sources": true,
81+
"with-sinks": true,
82+
"with-summaries": true
5383
},
5484
{
5585
"name": "actix-web",
5686
"git_repo": "https://github.com/actix/actix-web",
57-
"git_tag": "web-v4.11.0"
87+
"git_tag": "web-v4.11.0",
88+
"with-sources": true,
89+
"with-sinks": true,
90+
"with-summaries": true
5891
},
5992
{
6093
"name": "hyper",
6194
"git_repo": "https://github.com/hyperium/hyper",
62-
"git_tag": "v1.6.0"
95+
"git_tag": "v1.6.0",
96+
"with-sources": true,
97+
"with-sinks": true,
98+
"with-summaries": true
6399
},
64100
{
65101
"name": "clap",
66102
"git_repo": "https://github.com/clap-rs/clap",
67-
"git_tag": "v4.5.38"
103+
"git_tag": "v4.5.38",
104+
"with-sources": true,
105+
"with-sinks": true,
106+
"with-summaries": true
68107
}
69108
],
70109
"destination": "rust/ql/lib/ext/generated",

0 commit comments

Comments
 (0)