Skip to content

Commit b7edcd6

Browse files
lysnikolaoulysnikolaou
committed
Do not use _PyAST_ExprAsUnicode (unparse) for interpolation expression
1 parent bf89325 commit b7edcd6

File tree

6 files changed

+30
-32
lines changed

6 files changed

+30
-32
lines changed

‎Lib/test/test_tstring.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ def test_debug_specifier(self):
207207
t = t"Value: {value=}"
208208
self.assertEqual(t.strings, ("Value: value=", ""))
209209
self.assertEqual(t.interpolations[0].value, value)
210-
self.assertEqual(t.interpolations[0].expression, "value")
210+
self.assertEqual(t.interpolations[0].expression, "value=")
211211
self.assertEqual(t.interpolations[0].conversion, "r")
212212
self.assertEqual(t.interpolations[0].format_spec, "")
213213
self.assertEqual(f(t), "Value: value=42")
@@ -216,7 +216,7 @@ def test_debug_specifier(self):
216216
t = t"Value: {value=:.2f}"
217217
self.assertEqual(t.strings, ("Value: value=", ""))
218218
self.assertEqual(t.interpolations[0].value, value)
219-
self.assertEqual(t.interpolations[0].expression, "value")
219+
self.assertEqual(t.interpolations[0].expression, "value=")
220220
self.assertEqual(t.interpolations[0].conversion, None)
221221
self.assertEqual(t.interpolations[0].format_spec, ".2f")
222222
self.assertEqual(f(t), "Value: value=42.00")
@@ -225,15 +225,15 @@ def test_debug_specifier(self):
225225
t = t"Value: {value=!s}"
226226
self.assertEqual(t.strings, ("Value: value=", ""))
227227
self.assertEqual(t.interpolations[0].value, value)
228-
self.assertEqual(t.interpolations[0].expression, "value")
228+
self.assertEqual(t.interpolations[0].expression, "value=")
229229
self.assertEqual(t.interpolations[0].conversion, "s")
230230
self.assertEqual(t.interpolations[0].format_spec, "")
231231

232232
# Test white space in debug specifier
233233
t = t"Value: {value = }"
234234
self.assertEqual(t.strings, ("Value: value = ", ""))
235235
self.assertEqual(t.interpolations[0].value, value)
236-
self.assertEqual(t.interpolations[0].expression, "value")
236+
self.assertEqual(t.interpolations[0].expression, "value = ")
237237
self.assertEqual(t.interpolations[0].conversion, "r")
238238
self.assertEqual(t.interpolations[0].format_spec, "")
239239
self.assertEqual(f(t), "Value: value = 42")

‎Parser/action_helpers.c‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,11 +1479,6 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu
14791479
ResultTokenWithMetadata *format, Token *closing_brace, int lineno, int col_offset,
14801480
int end_lineno, int end_col_offset, PyArena *arena) {
14811481

1482-
constant exprstr = _PyAST_ExprAsUnicode(expression);
1483-
if (exprstr == NULL) {
1484-
return NULL;
1485-
}
1486-
14871482
constant convstr = NULL;
14881483
int conversion_val = _get_interpolation_conversion(p, debug, conversion, format);
14891484
if (conversion_val >= 0) {
@@ -1495,35 +1490,38 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu
14951490
}
14961491
}
14971492

1498-
expr_ty interpolation = _PyAST_Interpolation(
1499-
expression, exprstr, convstr, format ? (expr_ty) format->result : NULL,
1500-
lineno, col_offset, end_lineno,
1501-
end_col_offset, arena
1502-
);
1503-
1504-
if (!debug) {
1505-
return interpolation;
1506-
}
1507-
15081493
/* Find the non whitespace token after the "=" */
15091494
int debug_end_line, debug_end_offset;
15101495
PyObject *debug_metadata;
1496+
constant exprstr;
15111497

15121498
if (conversion) {
15131499
debug_end_line = ((expr_ty) conversion->result)->lineno;
15141500
debug_end_offset = ((expr_ty) conversion->result)->col_offset;
1515-
debug_metadata = conversion->metadata;
1501+
debug_metadata = exprstr = conversion->metadata;
15161502
}
15171503
else if (format) {
15181504
debug_end_line = ((expr_ty) format->result)->lineno;
15191505
debug_end_offset = ((expr_ty) format->result)->col_offset + 1;
1520-
debug_metadata = format->metadata;
1506+
debug_metadata = exprstr = format->metadata;
15211507
}
15221508
else {
15231509
debug_end_line = end_lineno;
15241510
debug_end_offset = end_col_offset;
1525-
debug_metadata = closing_brace->metadata;
1511+
debug_metadata = exprstr = closing_brace->metadata;
15261512
}
1513+
1514+
assert(exprstr != NULL);
1515+
expr_ty interpolation = _PyAST_Interpolation(
1516+
expression, exprstr, convstr, format ? (expr_ty) format->result : NULL,
1517+
lineno, col_offset, end_lineno,
1518+
end_col_offset, arena
1519+
);
1520+
1521+
if (!debug) {
1522+
return interpolation;
1523+
}
1524+
15271525
expr_ty debug_text = _PyAST_Constant(debug_metadata, NULL, lineno, col_offset + 1, debug_end_line,
15281526
debug_end_offset - 1, p->arena);
15291527
if (!debug_text) {

‎Parser/lexer/lexer.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,12 @@ tok_backup(struct tok_state *tok, int c)
108108
}
109109

110110
static int
111-
set_fstring_expr(struct tok_state* tok, struct token *token, char c) {
111+
set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
112112
assert(token != NULL);
113113
assert(c == '}' || c == ':' || c == '!');
114114
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
115115

116-
if (!tok_mode->f_string_debug || token->metadata) {
116+
if (!(tok_mode->f_string_debug || tok_mode->tstring) || token->metadata) {
117117
return 0;
118118
}
119119
PyObject *res = NULL;
@@ -173,7 +173,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) {
173173
}
174174

175175
int
176-
_PyLexer_update_fstring_expr(struct tok_state *tok, char cur)
176+
_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
177177
{
178178
assert(tok->cur != NULL);
179179

@@ -1152,10 +1152,10 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
11521152
int cursor_in_format_with_debug =
11531153
cursor == 1 && (current_tok->f_string_debug || in_format_spec);
11541154
int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
1155-
if ((cursor_valid) && !_PyLexer_update_fstring_expr(tok, c)) {
1155+
if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
11561156
return MAKE_TOKEN(ENDMARKER);
11571157
}
1158-
if ((cursor_valid) && c != '{' && set_fstring_expr(tok, token, c)) {
1158+
if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) {
11591159
return MAKE_TOKEN(ERRORTOKEN);
11601160
}
11611161

@@ -1404,7 +1404,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
14041404
}
14051405

14061406
if (c == '{') {
1407-
if (!_PyLexer_update_fstring_expr(tok, c)) {
1407+
if (!_PyLexer_update_ftstring_expr(tok, c)) {
14081408
return MAKE_TOKEN(ENDMARKER);
14091409
}
14101410
int peek = tok_nextc(tok);

‎Parser/lexer/lexer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
#include "state.h"
55

6-
int _PyLexer_update_fstring_expr(struct tok_state *tok, char cur);
6+
int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
77

88
int _PyTokenizer_Get(struct tok_state *, struct token *);
99

‎Parser/tokenizer/file_tokenizer.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ tok_underflow_interactive(struct tok_state *tok) {
275275
return 0;
276276
}
277277

278-
if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) {
278+
if (tok->tok_mode_stack_index && !_PyLexer_update_ftstring_expr(tok, 0)) {
279279
return 0;
280280
}
281281
return 1;
@@ -322,7 +322,7 @@ tok_underflow_file(struct tok_state *tok) {
322322
tok->implicit_newline = 1;
323323
}
324324

325-
if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) {
325+
if (tok->tok_mode_stack_index && !_PyLexer_update_ftstring_expr(tok, 0)) {
326326
return 0;
327327
}
328328

‎Parser/tokenizer/readline_tokenizer.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ tok_underflow_readline(struct tok_state* tok) {
9090
tok->implicit_newline = 1;
9191
}
9292

93-
if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) {
93+
if (tok->tok_mode_stack_index && !_PyLexer_update_ftstring_expr(tok, 0)) {
9494
return 0;
9595
}
9696

0 commit comments

Comments
 (0)