Skip to content

Commit dee1337

Browse files
youknowoneyouknowone
authored
Merge pull request RustPython#4379 from harupy/refactor-FStringParser
Refactor `FStringParser`
2 parents b7eb46a + af4a43d commit dee1337

File tree

1 file changed

+58
-66
lines changed

1 file changed

+58
-66
lines changed

‎compiler/parser/src/fstring.rs‎

Lines changed: 58 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -7,63 +7,72 @@ use crate::{
77
};
88
use std::{iter, mem, str};
99

10-
struct FStringParser {
10+
struct FStringParser<'a> {
11+
chars: iter::Peekable<str::Chars<'a>>,
1112
str_start: Location,
1213
str_end: Location,
1314
}
1415

15-
impl FStringParser {
16-
fn new(str_start: Location, str_end: Location) -> Self {
17-
Self { str_start, str_end }
16+
impl<'a> FStringParser<'a> {
17+
fn new(source: &'a str, str_start: Location, str_end: Location) -> Self {
18+
Self {
19+
chars: source.chars().peekable(),
20+
str_start,
21+
str_end,
22+
}
23+
}
24+
25+
fn next_char(&mut self) -> Option<char> {
26+
self.chars.next()
27+
}
28+
29+
fn peek(&mut self) -> Option<&char> {
30+
self.chars.peek()
1831
}
1932

2033
#[inline]
2134
fn expr(&self, node: ExprKind) -> Expr {
2235
Expr::new(self.str_start, self.str_end, node)
2336
}
2437

25-
fn parse_formatted_value<'a>(
26-
&mut self,
27-
mut chars: iter::Peekable<str::Chars<'a>>,
28-
nested: u8,
29-
) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
38+
fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
3039
let mut expression = String::new();
3140
let mut spec = None;
3241
let mut delims = Vec::new();
3342
let mut conversion = ConversionFlag::None;
3443
let mut self_documenting = false;
3544
let mut trailing_seq = String::new();
3645

37-
while let Some(ch) = chars.next() {
46+
while let Some(ch) = self.next_char() {
3847
match ch {
3948
// can be integrated better with the remainign code, but as a starting point ok
4049
// in general I would do here a tokenizing of the fstrings to omit this peeking.
41-
'!' if chars.peek() == Some(&'=') => {
50+
'!' if self.peek() == Some(&'=') => {
4251
expression.push_str("!=");
43-
chars.next();
52+
self.next_char();
4453
}
4554

46-
'=' if chars.peek() == Some(&'=') => {
55+
'=' if self.peek() == Some(&'=') => {
4756
expression.push_str("==");
48-
chars.next();
57+
self.next_char();
4958
}
5059

51-
'>' if chars.peek() == Some(&'=') => {
60+
'>' if self.peek() == Some(&'=') => {
5261
expression.push_str(">=");
53-
chars.next();
62+
self.next_char();
5463
}
5564

56-
'<' if chars.peek() == Some(&'=') => {
65+
'<' if self.peek() == Some(&'=') => {
5766
expression.push_str("<=");
58-
chars.next();
67+
self.next_char();
5968
}
6069

61-
'!' if delims.is_empty() && chars.peek() != Some(&'=') => {
70+
'!' if delims.is_empty() && self.peek() != Some(&'=') => {
6271
if expression.trim().is_empty() {
6372
return Err(EmptyExpression);
6473
}
6574

66-
conversion = match chars.next() {
75+
conversion = match self.next_char() {
6776
Some('s') => ConversionFlag::Str,
6877
Some('a') => ConversionFlag::Ascii,
6978
Some('r') => ConversionFlag::Repr,
@@ -83,7 +92,7 @@ impl FStringParser {
8392
}
8493
};
8594

86-
if let Some(&peek) = chars.peek() {
95+
if let Some(&peek) = self.peek() {
8796
if peek != '}' && peek != ':' {
8897
return Err(if expression.trim().is_empty() {
8998
EmptyExpression
@@ -102,17 +111,16 @@ impl FStringParser {
102111

103112
// match a python 3.8 self documenting expression
104113
// format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
105-
'=' if chars.peek() != Some(&'=') && delims.is_empty() => {
114+
'=' if self.peek() != Some(&'=') && delims.is_empty() => {
106115
self_documenting = true;
107116
}
108117

109118
':' if delims.is_empty() => {
110-
let (parsed_spec, remaining_chars) = self.parse_spec(chars, nested)?;
119+
let parsed_spec = self.parse_spec(nested)?;
111120

112121
spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
113122
values: parsed_spec,
114123
})));
115-
chars = remaining_chars;
116124
}
117125
'(' | '{' | '[' => {
118126
expression.push(ch);
@@ -195,21 +203,19 @@ impl FStringParser {
195203
}),
196204
]
197205
};
198-
return Ok((ret, chars));
206+
return Ok(ret);
199207
}
200208
'"' | '\'' => {
201209
expression.push(ch);
202-
let mut string_ended = false;
203-
for next in &mut chars {
204-
expression.push(next);
205-
if next == ch {
206-
string_ended = true;
210+
loop {
211+
let Some(c) = self.next_char() else {
212+
return Err(UnterminatedString);
213+
};
214+
expression.push(c);
215+
if c == ch {
207216
break;
208217
}
209218
}
210-
if !string_ended {
211-
return Err(UnterminatedString);
212-
}
213219
}
214220
' ' if self_documenting => {
215221
trailing_seq.push(ch);
@@ -231,14 +237,10 @@ impl FStringParser {
231237
})
232238
}
233239

234-
fn parse_spec<'a>(
235-
&mut self,
236-
mut chars: iter::Peekable<str::Chars<'a>>,
237-
nested: u8,
238-
) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
240+
fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
239241
let mut spec_constructor = Vec::new();
240242
let mut constant_piece = String::new();
241-
while let Some(&next) = chars.peek() {
243+
while let Some(&next) = self.peek() {
242244
match next {
243245
'{' => {
244246
if !constant_piece.is_empty() {
@@ -248,9 +250,8 @@ impl FStringParser {
248250
}));
249251
constant_piece.clear();
250252
}
251-
let (parsed_expr, remaining_chars) = self.parse(chars, nested + 1)?;
253+
let parsed_expr = self.parse(nested + 1)?;
252254
spec_constructor.extend(parsed_expr);
253-
chars = remaining_chars;
254255
continue;
255256
}
256257
'}' => {
@@ -260,7 +261,7 @@ impl FStringParser {
260261
constant_piece.push(next);
261262
}
262263
}
263-
chars.next();
264+
self.next_char();
264265
}
265266
if !constant_piece.is_empty() {
266267
spec_constructor.push(self.expr(ExprKind::Constant {
@@ -269,29 +270,25 @@ impl FStringParser {
269270
}));
270271
constant_piece.clear();
271272
}
272-
Ok((spec_constructor, chars))
273+
Ok(spec_constructor)
273274
}
274275

275-
fn parse<'a>(
276-
&mut self,
277-
mut chars: iter::Peekable<str::Chars<'a>>,
278-
nested: u8,
279-
) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
276+
fn parse(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
280277
if nested >= 2 {
281278
return Err(ExpressionNestedTooDeeply);
282279
}
283280

284281
let mut content = String::new();
285282
let mut values = vec![];
286283

287-
while let Some(&ch) = chars.peek() {
284+
while let Some(&ch) = self.peek() {
288285
match ch {
289286
'{' => {
290-
chars.next();
287+
self.next_char();
291288
if nested == 0 {
292-
match chars.peek() {
289+
match self.peek() {
293290
Some('{') => {
294-
chars.next();
291+
self.next_char();
295292
content.push('{');
296293
continue;
297294
}
@@ -306,26 +303,24 @@ impl FStringParser {
306303
}));
307304
}
308305

309-
let (parsed_values, remaining_chars) =
310-
self.parse_formatted_value(chars, nested)?;
306+
let parsed_values = self.parse_formatted_value(nested)?;
311307
values.extend(parsed_values);
312-
chars = remaining_chars;
313308
}
314309
'}' => {
315310
if nested > 0 {
316311
break;
317312
}
318-
chars.next();
319-
if let Some('}') = chars.peek() {
320-
chars.next();
313+
self.next_char();
314+
if let Some('}') = self.peek() {
315+
self.next_char();
321316
content.push('}');
322317
} else {
323318
return Err(SingleRbrace);
324319
}
325320
}
326321
_ => {
327322
content.push(ch);
328-
chars.next();
323+
self.next_char();
329324
}
330325
}
331326
}
@@ -337,7 +332,7 @@ impl FStringParser {
337332
}))
338333
}
339334

340-
Ok((values, chars))
335+
Ok(values)
341336
}
342337
}
343338

@@ -353,9 +348,8 @@ pub fn parse_located_fstring(
353348
start: Location,
354349
end: Location,
355350
) -> Result<Vec<Expr>, FStringError> {
356-
FStringParser::new(start, end)
357-
.parse(source.chars().peekable(), 0)
358-
.map(|(e, _)| e)
351+
FStringParser::new(source, start, end)
352+
.parse(0)
359353
.map_err(|error| FStringError {
360354
error,
361355
location: start,
@@ -367,9 +361,7 @@ mod tests {
367361
use super::*;
368362

369363
fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
370-
FStringParser::new(Location::default(), Location::default())
371-
.parse(source.chars().peekable(), 0)
372-
.map(|(e, _)| e)
364+
FStringParser::new(source, Location::default(), Location::default()).parse(0)
373365
}
374366

375367
#[test]

0 commit comments

Comments
 (0)