Skip to content

Python: Allow comments in subscripts #18702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yoff merged 3 commits into from
Feb 6, 2025
Merged

Python: Allow comments in subscripts #18702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c5be2a3
Python: Allow comments in subscripts
tausbn Feb 6, 2025
7124e80
Python: Regenerate parser files
tausbn Feb 6, 2025
3d25cd3
Python: Add change note
tausbn Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions python/extractor/tests/parser/subscripts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
a[b]

c[d,e]

c1[d1,]

# And now with many comments

e[
# comment1
f
# comment2
]

g[
# comment3
h,
# comment4
i
# comment5
]
60 changes: 7 additions & 53 deletions python/extractor/tsg-python/python.tsg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
(assignment !type) @assign
{ let @assign.node = (ast-node @assign "Assign") }

[ (expression_list) (tuple) (tuple_pattern) (pattern_list) ] @tuple
[ (expression_list) (tuple) (tuple_pattern) (pattern_list) (index_expression_list) ] @tuple
{ let @tuple.node = (ast-node @tuple "Tuple") }

(list_pattern) @list
Expand Down Expand Up @@ -2543,66 +2543,16 @@

(subscript
value: (_) @value
subscript: (_) @index
) @subscript
{
attr (@subscript.node) value = @value.node
attr (@value.node) ctx = "load"
}

; Single subscript
(subscript
value: (_)
.
subscript: (_) @index
.
) @subscript
{
attr (@subscript.node) index = @index.node
attr (@index.node) ctx = "load"
}

; For expressions of the form `a[b, c]` we must explicitly synthesize an internal tuple node
; We do this and also hook it up:
(subscript
value: (_)
.
subscript: (_) @first
.
subscript: (_)
) @subscript
{
let @subscript.tuple = (ast-node @first "Tuple")
attr (@subscript.tuple) ctx = "load"
attr (@subscript.node) index = @subscript.tuple
edge @subscript.tuple -> @first.node
attr (@subscript.tuple -> @first.node) elts = (named-child-index @first)
attr (@first.node) ctx = "load"
}

(subscript
value: (_)
.
subscript: (_)
subscript: (_) @elt
) @subscript
{
edge @subscript.tuple -> @elt.node
attr (@subscript.tuple -> @elt.node) elts = (named-child-index @elt)
attr (@elt.node) ctx = "load"
}


; Set the end position correctly
(subscript
value: (_)
.
subscript: (_)
subscript: (_) @last
.
) @subscript
{
attr (@subscript.tuple) _location_end = (location-end @last)
}



Expand Down Expand Up @@ -3448,9 +3398,12 @@
; Left hand side of an assignment such as `[foo, bar] = ...`
(list_pattern element: (_) @elt) @parent

; An unadorned tuple (such as in `x = y, z`)
; An unadorned tuple such as in `x = y, z`
(expression_list element: (_) @elt) @parent

; An index containing multiple indices such as in `x[y, z]`
(index_expression_list element: (_) @elt) @parent

; A regular tuple such as `(x, y, z)`
(tuple element: (_) @elt) @parent

Expand Down Expand Up @@ -3486,6 +3439,7 @@
(pattern_list element: (_) @elt)
(list_pattern element: (_) @elt)
(expression_list element: (_) @elt)
(index_expression_list element: (_) @elt)
(parenthesized_expression inner: (_) @elt)
(set element: (_) @elt)
(match_sequence_pattern (_) @elt)
Expand Down
11 changes: 9 additions & 2 deletions python/extractor/tsg-python/tsp/grammar.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -929,11 +929,18 @@ module.exports = grammar({
field('attribute', $.identifier)
)),

_index_expression: $ => choice(
$.list_splat,
$.expression,
$.slice
),

index_expression_list: $ => open_sequence(field('element', $._index_expression)),

subscript: $ => prec(PREC.call, seq(
field('value', $.primary_expression),
'[',
commaSep1(field('subscript', choice($.list_splat, $.expression, $.slice))),
optional(','),
field('subscript', choice($._index_expression, $.index_expression_list)),
']'
)),

Expand Down
160 changes: 93 additions & 67 deletions python/extractor/tsg-python/tsp/src/grammar.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5045,94 +5045,120 @@
]
}
},
"subscript": {
"type": "PREC",
"value": 21,
"_index_expression": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
},
{
"type": "SYMBOL",
"name": "slice"
}
]
},
"index_expression_list": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "value",
"name": "element",
"content": {
"type": "SYMBOL",
"name": "primary_expression"
"name": "_index_expression"
}
},
{
"type": "STRING",
"value": "["
},
{
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "FIELD",
"name": "subscript",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
},
{
"type": "SYMBOL",
"name": "slice"
}
]
}
"type": "STRING",
"value": ","
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "FIELD",
"name": "subscript",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
},
{
"type": "SEQ",
"members": [
{
"type": "REPEAT1",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "FIELD",
"name": "element",
"content": {
"type": "SYMBOL",
"name": "slice"
"name": "_index_expression"
}
]
}
}
]
}
]
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "BLANK"
}
]
}
]
}
]
}
]
}
},
"subscript": {
"type": "PREC",
"value": 21,
"content": {
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "value",
"content": {
"type": "SYMBOL",
"name": "primary_expression"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "BLANK"
}
]
"type": "STRING",
"value": "["
},
{
"type": "FIELD",
"name": "subscript",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_index_expression"
},
{
"type": "SYMBOL",
"name": "index_expression_list"
}
]
}
},
{
"type": "STRING",
Expand Down
30 changes: 29 additions & 1 deletion python/extractor/tsg-python/tsp/src/node-types.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1829,6 +1829,30 @@
}
}
},
{
"type": "index_expression_list",
"named": true,
"fields": {
"element": {
"multiple": true,
"required": true,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
},
{
"type": "slice",
"named": true
}
]
}
}
},
{
"type": "interpolation",
"named": true,
Expand Down Expand Up @@ -3200,13 +3224,17 @@
"named": true,
"fields": {
"subscript": {
"multiple": true,
"multiple": false,
"required": true,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "index_expression_list",
"named": true
},
{
"type": "list_splat",
"named": true
Expand Down
Loading