Merge pull request RustPython#4384 from harupy/parse-formatted-value

Fix the location of `FormattedValue`

Author: youknowone

Lib/test/test_ast.py

@@ -2031,16 +2031,12 @@ def test_suites(self):
         self._check_content(s, try_stmt.body[0], 'raise RuntimeError')
         self._check_content(s, try_stmt.handlers[0].type, 'TypeError')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_fstring(self):
         s = 'x = f"abc {x + y} abc"'
         fstr = self._parse_value(s)
         binop = fstr.values[1].value
         self._check_content(s, binop, 'x + y')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_fstring_multi_line(self):
         s = dedent('''
             f"""Some multi-line text.

Lib/test/test_fstring.py

@@ -83,8 +83,6 @@ def __call__(self):
         # Make sure x was called.
         self.assertTrue(x.called)
 
-    # TODO: RUSTPYTHON - binop lineno
-    @unittest.expectedFailure 
     def test_ast_line_numbers(self):
         expr = """
 a = 10
@@ -116,8 +114,6 @@ def test_ast_line_numbers(self):
         self.assertEqual(binop.left.col_offset, 3)
         self.assertEqual(binop.right.col_offset, 7)
 
-    # TODO: RUSTPYTHON binops lineno and col_offset
-    @unittest.expectedFailure
     def test_ast_line_numbers_multiple_formattedvalues(self):
         expr = """
 f'no formatted values'
@@ -170,8 +166,6 @@ def test_ast_line_numbers_multiple_formattedvalues(self):
         self.assertEqual(binop2.left.col_offset, 23)
         self.assertEqual(binop2.right.col_offset, 27)
 
-    # TODO: RUSTPYTHON binops lineno and col_offset
-    @unittest.expectedFailure
     def test_ast_line_numbers_nested(self):
         expr = """
 a = 10
@@ -217,8 +211,6 @@ def test_ast_line_numbers_nested(self):
         self.assertEqual(call.lineno, 3)
         self.assertEqual(call.col_offset, 11)
 
-    # TODO: RUSTPYTHON binops lineno and col_offset
-    @unittest.expectedFailure
     def test_ast_line_numbers_duplicate_expression(self):
         expr = """
 a = 10
@@ -285,8 +277,6 @@ def test_ast_line_numbers_duplicate_expression(self):
         self.assertEqual(binop.left.col_offset, 23)
         self.assertEqual(binop.right.col_offset, 27)
 
-    # TODO: RUSTPYTHON err col_offset and missing end_* attributes
-    @unittest.expectedFailure
     def test_ast_numbers_fstring_with_formatting(self):
 
         t = ast.parse('f"Here is that pesky {xxx:.3f} again"')
@@ -310,8 +300,6 @@ def test_ast_numbers_fstring_with_formatting(self):
         self.assertEqual(name.col_offset, 22)
         self.assertEqual(name.end_col_offset, 25)
 
-    # TODO: RUSTPYTHON col_offset and binop lineno and col_offset
-    @unittest.expectedFailure
     def test_ast_line_numbers_multiline_fstring(self):
         # See bpo-30465 for details.
         expr = """
@@ -391,8 +379,6 @@ def test_ast_line_numbers_multiline_fstring(self):
         self.assertEqual(t.body[0].value.values[1].value.col_offset, 11)
         self.assertEqual(t.body[0].value.values[1].value.end_col_offset, 16)
 
-    # TODO: RUSTPYTHON lineno, col_offset, end*
-    @unittest.expectedFailure
     def test_ast_line_numbers_with_parentheses(self):
         expr = """
 x = (
@@ -470,8 +456,6 @@ def test_ast_compile_time_concat(self):
         exec(c)
         self.assertEqual(x[0], 'foo3')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_compile_time_concat_errors(self):
         self.assertAllRaise(SyntaxError,
                             'cannot mix bytes and nonbytes literals',
@@ -1083,8 +1067,6 @@ def test_del(self):
                              "del '' f''",
                              ])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_mismatched_braces(self):
         self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
                             ["f'{{}'",

compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap

@@ -1353,18 +1353,11 @@ OneOrMore<T>: Vec<T> = {
 };
 
 Constant: ast::Constant = {
-    <b:bytes+> => ast::Constant::Bytes(b.into_iter().flatten().collect()),
     <value:int> => ast::Constant::Int(value),
     <value:float> => ast::Constant::Float(value),
     <s:complex> => ast::Constant::Complex { real: s.0, imag: s.1 },
 };
 
-Bytes: Vec<u8> = {
-    <s:bytes+> => {
-        s.into_iter().flatten().collect::<Vec<u8>>()
-    },
-};
-
 Identifier: String = <s:name> => s;
 
 // Hook external lexer:
@@ -1462,8 +1455,11 @@ extern {
         int => lexer::Tok::Int { value: <BigInt> },
         float => lexer::Tok::Float { value: <f64> },
         complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String { value: <String>, kind: <StringKind> },
-        bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
+        string => lexer::Tok::String {
+            value: <String>,
+            kind: <StringKind>,
+            triple_quoted: <bool>
+        },
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,

compiler/parser/python.lalrpop

@@ -101,6 +101,15 @@ pub enum FStringErrorType {
     UnterminatedString,
 }
 
+impl FStringErrorType {
+    pub fn to_lexical_error(self, location: Location) -> LexicalError {
+        LexicalError {
+            error: LexicalErrorType::FStringError(self),
+            location,
+        }
+    }
+}
+
 impl fmt::Display for FStringErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {

compiler/parser/src/error.rs

@@ -1,3 +1,4 @@
+// We no longer need this file
 use self::FStringErrorType::*;
 use crate::{
     ast::{Constant, ConversionFlag, Expr, ExprKind, Location},