diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a2525e472..5d343f2c9 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1530,12 +1530,13 @@ impl<'a> Tokenizer<'a> { } Some('<') => self.consume_for_binop(chars, "<<", Token::ShiftLeft), Some('-') if self.dialect.supports_geometric_types() => { - chars.next(); // consume - match chars.peek() { - Some('>') => { - self.consume_for_binop(chars, "<->", Token::TwoWayArrow) - } - _ => self.start_binop_opt(chars, "<-", None), + // `<->` is a geometric operator, but `<-123` means `< -123`. + // If the sequence is not `<->`, keep `-` unconsumed so it can be tokenized as a unary minus. + if chars.peekable.clone().nth(1) == Some('>') { + chars.next(); // consume '-' + self.consume_for_binop(chars, "<->", Token::TwoWayArrow) + } else { + self.start_binop(chars, "<", Token::Lt) } } Some('^') if self.dialect.supports_geometric_types() => { @@ -2495,7 +2496,8 @@ fn take_char_from_hex_digits( mod tests { use super::*; use crate::dialect::{ - BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect, + BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, + RedshiftSqlDialect, SQLiteDialect, }; use crate::test_utils::{all_dialects_except, all_dialects_where}; use core::fmt::Debug; @@ -4152,4 +4154,40 @@ mod tests { ], ); } + + #[test] + fn tokenize_lt_followed_by_negative_number() { + let sql = "SELECT a <-4000"; + let dialect = RedshiftSqlDialect {}; + let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap(); + compare( + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::make_word("a", None), + Token::Whitespace(Whitespace::Space), + Token::Lt, + Token::Minus, + Token::Number("4000".to_string(), false), + ], + tokens, + ); + + // Ensure geometric `<->` is still recognized. + let tokens = Tokenizer::new(&dialect, "SELECT a <-> b") + .tokenize() + .unwrap(); + compare( + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::make_word("a", None), + Token::Whitespace(Whitespace::Space), + Token::TwoWayArrow, + Token::Whitespace(Whitespace::Space), + Token::make_word("b", None), + ], + tokens, + ); + } }