diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 0b31ae5..19d876b 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -386,38 +386,7 @@ impl<'a> Tokenizer<'a> { } } // numbers - '0'..='9' => { - let mut seen_decimal = false; - let mut s = peeking_take_while(chars, |ch| match ch { - '0'..='9' => true, - '.' if !seen_decimal => { - seen_decimal = true; - true - } - _ => false, - }); - // If in e-notation, parse the e-notation with special care given to negative exponents. - match chars.peek() { - Some('e') | Some('E') => { - s.push('E'); - // Consume the e-notation signifier. - chars.next(); - if let Some('-') = chars.peek() { - s.push('-'); - // Consume the negative sign. - chars.next(); - } - let e = peeking_take_while(chars, |ch| match ch { - '0'..='9' => true, - _ => false, - }); - s.push_str(&e); - } - _ => {} - } - - Ok(Some(Token::Number(s))) - } + '0'..='9' => self.tokenize_number(chars), // punctuation '(' => self.consume_and_return(chars, Token::LParen), ')' => self.consume_and_return(chars, Token::RParen), @@ -510,7 +479,22 @@ impl<'a> Tokenizer<'a> { } } '=' => self.consume_and_return(chars, Token::Eq), - '.' => self.consume_and_return(chars, Token::Period), + '.' => { + chars.next(); // consume '.' + match chars.peek() { + Some('0'..='9') => { + // Add the '.' back to the chars and parse as number. + let mut chars_w_leading_zero = ".".to_string(); + while let Some(token) = chars.next() { + chars_w_leading_zero.push(token); + } + let mut peekable = chars_w_leading_zero.chars().peekable(); + + self.tokenize_number(&mut peekable) + } + _ => Ok(Some(Token::Period)), + } + } '!' => { chars.next(); // consume match chars.peek() { @@ -652,6 +636,42 @@ impl<'a> Tokenizer<'a> { Ok(Some(Token::Parameter(n))) } + fn tokenize_number( + &self, + chars: &mut Peekable>, + ) -> Result, TokenizerError> { + let mut seen_decimal = false; + let mut s = peeking_take_while(chars, |ch| match ch { + '0'..='9' => true, + '.' if !seen_decimal => { + seen_decimal = true; + true + } + _ => false, + }); + // If in e-notation, parse the e-notation with special care given to negative exponents. + match chars.peek() { + Some('e') | Some('E') => { + s.push('E'); + // Consume the e-notation signifier. + chars.next(); + if let Some('-') = chars.peek() { + s.push('-'); + // Consume the negative sign. + chars.next(); + } + let e = peeking_take_while(chars, |ch| match ch { + '0'..='9' => true, + _ => false, + }); + s.push_str(&e); + } + _ => {} + } + + Ok(Some(Token::Number(s))) + } + fn consume_and_return( &self, chars: &mut Peekable>, diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index c22a8d1..374a43f 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -486,6 +486,20 @@ fn parse_number() { assert_eq!(expr, Expr::Value(Value::Number("1.0".into()))); } +#[test] +fn parse_numeric_begin_with_decimal() { + let expr = verified_expr(".1"); + + #[cfg(feature = "bigdecimal")] + assert_eq!( + expr, + Expr::Value(Value::Number(bigdecimal::BigDecimal::from(.1))) + ); + + #[cfg(not(feature = "bigdecimal"))] + assert_eq!(expr, Expr::Value(Value::Number(".1".into()))); +} + #[test] fn parse_approximate_numeric_literal() { let expr = verified_expr("1.0E2");