@@ -689,24 +689,12 @@ where
689689 fn consume_normal ( & mut self ) -> Result < ( ) , LexicalError > {
690690 // Check if we have some character:
691691 if let Some ( c) = self . window [ 0 ] {
692- // First check identifier:
693- if self . is_identifier_start ( c) {
694- let identifier = self . lex_identifier ( ) ?;
695- self . emit ( identifier) ;
696- } else if is_emoji_presentation ( c) {
697- let tok_start = self . get_pos ( ) ;
698- self . next_char ( ) ;
699- let tok_end = self . get_pos ( ) ;
700- self . emit ( (
701- tok_start,
702- Tok :: Name {
703- name : c. to_string ( ) ,
704- } ,
705- tok_end,
706- ) ) ;
707- } else {
708- self . consume_character ( c) ?;
709- }
692+ if self . is_identifier_start ( c) {
693+ let identifier = self . lex_identifier ( ) ?;
694+ self . emit ( identifier) ;
695+ } else {
696+ self . consume_character ( c) ?;
697+ }
710698 } else {
711699 // We reached end of file.
712700 let tok_pos = self . get_pos ( ) ;
@@ -1112,13 +1100,25 @@ where
11121100 } ) ;
11131101 }
11141102 }
1115-
11161103 _ => {
1117- let c = self . next_char ( ) ;
1118- return Err ( LexicalError {
1119- error : LexicalErrorType :: UnrecognizedToken { tok : c. unwrap ( ) } ,
1120- location : self . get_pos ( ) ,
1121- } ) ;
1104+ if is_emoji_presentation ( c) {
1105+ let tok_start = self . get_pos ( ) ;
1106+ self . next_char ( ) ;
1107+ let tok_end = self . get_pos ( ) ;
1108+ self . emit ( (
1109+ tok_start,
1110+ Tok :: Name {
1111+ name : c. to_string ( ) ,
1112+ } ,
1113+ tok_end,
1114+ ) ) ;
1115+ } else {
1116+ let c = self . next_char ( ) ;
1117+ return Err ( LexicalError {
1118+ error : LexicalErrorType :: UnrecognizedToken { tok : c. unwrap ( ) } ,
1119+ location : self . get_pos ( ) ,
1120+ } ) ;
1121+ }
11221122 } // Ignore all the rest..
11231123 }
11241124
0 commit comments