From 35fa7754bdd7dd63781fbc38510ba09395626607 Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:54:17 -0400 Subject: [PATCH 01/12] - fixed coordinates; --- .../python_features/assignments/starred/main_gt.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/assignments/starred/main_gt.json b/micro-benchmark-autogen-templates/python_features/assignments/starred/main_gt.json index 0aa56cdff..1a67dc737 100644 --- a/micro-benchmark-autogen-templates/python_features/assignments/starred/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/assignments/starred/main_gt.json @@ -65,8 +65,8 @@ }, { "file": "main.py", - "line_number": 18, - "col_offset": 8, + "line_number": 17, + "col_offset": 5, "variable": "b[0]", "type": [ "callable" @@ -74,8 +74,8 @@ }, { "file": "main.py", - "line_number": 18, - "col_offset": 8, + "line_number": 17, + "col_offset": 5, "variable": "b[1]", "type": [ "callable" @@ -127,4 +127,4 @@ ] } ] -} \ No newline at end of file +} From e6da0048110fedcc543ce10589879e84b4bce0cc Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:54:56 -0400 Subject: [PATCH 02/12] - fixed types not being parametrized; --- .../python_features/builtins/functions/main_gt.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/builtins/functions/main_gt.json b/micro-benchmark-autogen-templates/python_features/builtins/functions/main_gt.json index d6a0da903..66e2020f8 100644 --- a/micro-benchmark-autogen-templates/python_features/builtins/functions/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/builtins/functions/main_gt.json @@ -47,7 +47,7 @@ "col_offset": 1, "variable": "total", "type": [ - "int" + "" ] }, { @@ -56,7 +56,7 @@ "col_offset": 1, "variable": "largest", "type": [ - "int" + "" ] }, { @@ -65,7 +65,7 @@ "col_offset": 1, "variable": "smallest", "type": [ - "int" + "" ] }, { @@ -96,4 +96,4 @@ ] } ] -} \ No newline at end of file +} From 81f2ce886d83f4e84cbcd23c7db8075f473c9cce Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:56:17 -0400 Subject: [PATCH 03/12] - fixed missing "function" context; --- .../python_features/dicts/new_key_param/main_gt.json | 1 + 1 file changed, 1 insertion(+) diff --git a/micro-benchmark-autogen-templates/python_features/dicts/new_key_param/main_gt.json b/micro-benchmark-autogen-templates/python_features/dicts/new_key_param/main_gt.json index ea444ec33..a69560191 100644 --- a/micro-benchmark-autogen-templates/python_features/dicts/new_key_param/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/dicts/new_key_param/main_gt.json @@ -43,6 +43,7 @@ "line_number": 9, "col_offset": 5, "variable": "d['a']", + "function": "func", "type": [ "callable" ] From 09bdce58443073679b5234271f4f8266c1393dce Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:58:50 -0400 Subject: [PATCH 04/12] - fixed coordinate; --- .../python_features/dicts/type_coercion/main_gt.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micro-benchmark-autogen-templates/python_features/dicts/type_coercion/main_gt.json b/micro-benchmark-autogen-templates/python_features/dicts/type_coercion/main_gt.json index 61c6a42ea..feddc89f2 100644 --- a/micro-benchmark-autogen-templates/python_features/dicts/type_coercion/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/dicts/type_coercion/main_gt.json @@ -40,7 +40,7 @@ { "file": "main.py", "line_number": 12, - "col_offset": 5, + "col_offset": 1, "variable": "d[1]", "type": [ "callable" From 03c842f47c1bd0f4208bafe3efb6517cb30bb3a7 Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:59:30 -0400 Subject: [PATCH 05/12] - fixed incorrect function context; --- .../python_features/returns/return_lambda/main_gt.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/returns/return_lambda/main_gt.json b/micro-benchmark-autogen-templates/python_features/returns/return_lambda/main_gt.json index 788cb4935..1054004c7 100644 --- a/micro-benchmark-autogen-templates/python_features/returns/return_lambda/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/returns/return_lambda/main_gt.json @@ -18,7 +18,7 @@ "file": "main.py", "line_number": 5, "col_offset": 19, - "function": "func", + "function": "lambda", "variable": "x", "type": [ "", @@ -53,4 +53,4 @@ ] } ] -} \ No newline at end of file +} From 4959d1cc17fd01a02f6f4984ca77a01fd28e66b5 Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 11:59:49 -0400 Subject: [PATCH 06/12] - fixed type not being parametrized; --- .../python_features/imports/import_as/main_gt.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/imports/import_as/main_gt.json b/micro-benchmark-autogen-templates/python_features/imports/import_as/main_gt.json index 2d8d47e3a..46bf37f1d 100644 --- a/micro-benchmark-autogen-templates/python_features/imports/import_as/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/imports/import_as/main_gt.json @@ -28,7 +28,7 @@ "col_offset": 1, "variable": "a", "type": [ - "str" + "" ] }, { @@ -41,4 +41,4 @@ ] } ] -} \ No newline at end of file +} From a40d4dbcbda04b770b1fd0dff6ec44caba359788 Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 12:24:45 -0400 Subject: [PATCH 07/12] - deleted entry from ground truth, as it describes a call, not a variable definition; --- .../python_features/dicts/update/main_gt.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/dicts/update/main_gt.json b/micro-benchmark-autogen-templates/python_features/dicts/update/main_gt.json index 1e7cfa0fa..b3a804265 100644 --- a/micro-benchmark-autogen-templates/python_features/dicts/update/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/dicts/update/main_gt.json @@ -46,15 +46,6 @@ "callable" ] }, - { - "file": "main.py", - "line_number": 14, - "col_offset": 1, - "variable": "d['a']", - "type": [ - "callable" - ] - }, { "file": "main.py", "line_number": 15, @@ -65,4 +56,4 @@ ] } ] -} \ No newline at end of file +} From 378745ab56bdc2aa9db526445a50a0c777b29e11 Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 12:25:47 -0400 Subject: [PATCH 08/12] - fixed invalid dictionary keys in subscript expressions; --- .../python_features/dicts/zip/main.py | 4 ++-- .../python_features/dicts/zip/main_gt.json | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/dicts/zip/main.py b/micro-benchmark-autogen-templates/python_features/dicts/zip/main.py index c54c7fd85..94cd89512 100644 --- a/micro-benchmark-autogen-templates/python_features/dicts/zip/main.py +++ b/micro-benchmark-autogen-templates/python_features/dicts/zip/main.py @@ -1,6 +1,6 @@ # Creating dictionaries from lists using the zip() function -keys = [, , ] -values = [, , ] +keys = ['a', 0, True] +values = [, , ] my_dict = dict(zip(keys, values)) diff --git a/micro-benchmark-autogen-templates/python_features/dicts/zip/main_gt.json b/micro-benchmark-autogen-templates/python_features/dicts/zip/main_gt.json index 4dc024198..75b927fd3 100644 --- a/micro-benchmark-autogen-templates/python_features/dicts/zip/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/dicts/zip/main_gt.json @@ -22,7 +22,7 @@ "col_offset": 1, "variable": "keys[0]", "type": [ - "" + "str" ] }, { @@ -31,7 +31,7 @@ "col_offset": 1, "variable": "keys[1]", "type": [ - "" + "int" ] }, { @@ -40,7 +40,7 @@ "col_offset": 1, "variable": "keys[2]", "type": [ - "" + "bool" ] }, { @@ -58,7 +58,7 @@ "col_offset": 1, "variable": "values[0]", "type": [ - "" + "" ] }, { @@ -76,7 +76,7 @@ "col_offset": 1, "variable": "values[2]", "type": [ - "" + "" ] }, { @@ -94,14 +94,14 @@ "col_offset": 1, "variable": "my_dict['a']", "type": [ - "" + "" ] }, { "file": "main.py", "line_number": 6, "col_offset": 1, - "variable": "my_dict['b']", + "variable": "my_dict[0]", "type": [ "" ] @@ -110,10 +110,10 @@ "file": "main.py", "line_number": 6, "col_offset": 1, - "variable": "my_dict['c']", + "variable": "my_dict[True]", "type": [ - "" + "" ] } ] -} \ No newline at end of file +} From 399120820efde5b8ee314b477bd761670c8c00ba Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 13:32:09 -0400 Subject: [PATCH 09/12] - fixed type, which wasn't parametrized; --- .../python_features/generators/iterable_assigned/main_gt.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micro-benchmark-autogen-templates/python_features/generators/iterable_assigned/main_gt.json b/micro-benchmark-autogen-templates/python_features/generators/iterable_assigned/main_gt.json index 740452e7d..ebef48ed0 100644 --- a/micro-benchmark-autogen-templates/python_features/generators/iterable_assigned/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/generators/iterable_assigned/main_gt.json @@ -62,7 +62,7 @@ "variable": "result", "function": "Cls.__next__", "type": [ - "int" + "" ] }, { From e010ba90425ecc22b4733da79e40af552d85b52b Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 31 Oct 2025 16:35:55 -0400 Subject: [PATCH 10/12] - fixed index out of range error due to interfering with enumeration; - made test more interesting by substituting with more than just "int"; --- .../python_features/generators/iterable/main.py | 2 +- .../generators/iterable/main_gt.json | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/micro-benchmark-autogen-templates/python_features/generators/iterable/main.py b/micro-benchmark-autogen-templates/python_features/generators/iterable/main.py index 5bd3c54e1..b9e10bdab 100644 --- a/micro-benchmark-autogen-templates/python_features/generators/iterable/main.py +++ b/micro-benchmark-autogen-templates/python_features/generators/iterable/main.py @@ -11,7 +11,7 @@ def __iter__(self): def __next__(self): if self.num < self.n: - cur, self.num = self.num, self.num + + cur, self.num = * self.num, self.num + 1 return cur else: raise StopIteration() diff --git a/micro-benchmark-autogen-templates/python_features/generators/iterable/main_gt.json b/micro-benchmark-autogen-templates/python_features/generators/iterable/main_gt.json index 713816eec..4e3b1a1c5 100644 --- a/micro-benchmark-autogen-templates/python_features/generators/iterable/main_gt.json +++ b/micro-benchmark-autogen-templates/python_features/generators/iterable/main_gt.json @@ -1,7 +1,9 @@ { "replacement_mode": "Complex", "type_replacements": [ - "int" + "int", + "str", + "float" ], "ground_truth": [ { @@ -49,7 +51,7 @@ "col_offset": 9, "function": "func.__next__", "type": [ - "int" + "" ] }, { @@ -59,7 +61,7 @@ "variable": "cur", "function": "func.__next__", "type": [ - "int" + "" ] }, { @@ -87,7 +89,7 @@ "col_offset": 1, "variable": "output_list[0]", "type": [ - "int" + "" ] }, { @@ -96,7 +98,7 @@ "col_offset": 1, "variable": "output_list[1]", "type": [ - "int" + "" ] }, { @@ -105,7 +107,7 @@ "col_offset": 1, "variable": "output_list[2]", "type": [ - "int" + "" ] }, { From 00311c4804762286d191bb6d47f95e14acdd0a9c Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Tue, 4 Nov 2025 14:14:53 -0500 Subject: [PATCH 11/12] - deleted entry from ground truth, as it describes a call, not a variable definition. Corresponds change a40d4dbc in the templates; --- .../python_features/dicts/update/main_gt.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/micro-benchmark/python_features/dicts/update/main_gt.json b/micro-benchmark/python_features/dicts/update/main_gt.json index a28ae31a7..23c294773 100644 --- a/micro-benchmark/python_features/dicts/update/main_gt.json +++ b/micro-benchmark/python_features/dicts/update/main_gt.json @@ -35,15 +35,6 @@ "callable" ] }, - { - "file": "main.py", - "line_number": 14, - "col_offset": 1, - "variable": "d['a']", - "type": [ - "callable" - ] - }, { "file": "main.py", "line_number": 15, From da3f9eda4ec60589b5f9d085e8841c710a00ef2e Mon Sep 17 00:00:00 2001 From: Juan Altmayer Pizzorno Date: Fri, 13 Mar 2026 14:33:12 -0400 Subject: [PATCH 12/12] Fix ground truth for inheritance/MRO benchmarks (Liskov substitution) The previous ground truth annotated each method with only its body's return type, ignoring that subclass overrides must have compatible return types per the Liskov substitution principle. When annotated as given, mypy --strict reports override errors on every affected benchmark. The corrected annotations widen parent method return types to include subclass override types, making all benchmarks pass mypy. Affected benchmarks: - classes/inheritance_overriding: MyClass.func str -> int|str - mro/parents_same_superclass: A.func str -> int|str - mro/self_assignment: B.func int -> int|str - mro/two_parents: B.func str -> int|str - mro/two_parents_method_defined: A.func float -> float|str, B.func int -> float|int|str --- .../classes/inheritance_overriding/main_gt.json | 1 + .../mro/parents_same_superclass/main_gt.json | 1 + .../python_features/mro/self_assignment/main_gt.json | 3 ++- .../python_features/mro/two_parents/main_gt.json | 1 + .../mro/two_parents_method_defined/main_gt.json | 7 +++++-- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/micro-benchmark/python_features/classes/inheritance_overriding/main_gt.json b/micro-benchmark/python_features/classes/inheritance_overriding/main_gt.json index 5f0b8cbe8..e3670b465 100644 --- a/micro-benchmark/python_features/classes/inheritance_overriding/main_gt.json +++ b/micro-benchmark/python_features/classes/inheritance_overriding/main_gt.json @@ -5,6 +5,7 @@ "col_offset": 9, "function": "MyClass.func", "type": [ + "int", "str" ] }, diff --git a/micro-benchmark/python_features/mro/parents_same_superclass/main_gt.json b/micro-benchmark/python_features/mro/parents_same_superclass/main_gt.json index dd7f9e0f9..0372e4a37 100644 --- a/micro-benchmark/python_features/mro/parents_same_superclass/main_gt.json +++ b/micro-benchmark/python_features/mro/parents_same_superclass/main_gt.json @@ -5,6 +5,7 @@ "col_offset": 9, "function": "A.func", "type": [ + "int", "str" ] }, diff --git a/micro-benchmark/python_features/mro/self_assignment/main_gt.json b/micro-benchmark/python_features/mro/self_assignment/main_gt.json index 085a436e6..be7937a8b 100644 --- a/micro-benchmark/python_features/mro/self_assignment/main_gt.json +++ b/micro-benchmark/python_features/mro/self_assignment/main_gt.json @@ -24,7 +24,8 @@ "col_offset": 9, "function": "B.func", "type": [ - "int" + "int", + "str" ] }, { diff --git a/micro-benchmark/python_features/mro/two_parents/main_gt.json b/micro-benchmark/python_features/mro/two_parents/main_gt.json index ff9c930af..facc3d279 100644 --- a/micro-benchmark/python_features/mro/two_parents/main_gt.json +++ b/micro-benchmark/python_features/mro/two_parents/main_gt.json @@ -14,6 +14,7 @@ "col_offset": 9, "function": "B.func", "type": [ + "int", "str" ] }, diff --git a/micro-benchmark/python_features/mro/two_parents_method_defined/main_gt.json b/micro-benchmark/python_features/mro/two_parents_method_defined/main_gt.json index 40a4055a9..3c24e29ea 100644 --- a/micro-benchmark/python_features/mro/two_parents_method_defined/main_gt.json +++ b/micro-benchmark/python_features/mro/two_parents_method_defined/main_gt.json @@ -5,7 +5,8 @@ "col_offset": 9, "function": "A.func", "type": [ - "float" + "float", + "str" ] }, { @@ -14,7 +15,9 @@ "col_offset": 9, "function": "B.func", "type": [ - "int" + "float", + "int", + "str" ] }, {