diff --git a/src/lexer.rs b/src/lexer.rs index 338325f..f99a4d5 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -374,7 +374,17 @@ where } // end for each byte match t { - None => None, + None => match (buf, state) { + (Some(b), Mode::Number) => Some(Token { + kind: TokenType::Number, + buf: Buffer::MultiByte(b), + }), + (None, Mode::Number) => Some(Token { + kind: TokenType::Number, + buf: Buffer::Span(Span { first, end: self.cursor }), + }), + _ => None, + }, Some(t) => { if self.cursor == last_cursor { None diff --git a/tests/lexer.rs b/tests/lexer.rs index 2b94eca..1120b80 100644 --- a/tests/lexer.rs +++ b/tests/lexer.rs @@ -169,6 +169,42 @@ fn other_backslash_escapes_in_string_value() { ); } +#[test] +fn isolated_value_pairs() { + for &(src, ref kind, first, end, buf) in &[ + (r#""v":12"#, TokenType::Number, 4, 6, "12"), + (r#""v":-12"#, TokenType::Number, 4, 7, "-12"), + (r#""v":"12""#, TokenType::String, 4, 8, r#""12""#), + (r#""v":true"#, TokenType::BooleanTrue, 4, 8, ""), + (r#""v":false"#, TokenType::BooleanFalse, 4, 9, ""), + (r#""v":null"#, TokenType::Null, 4, 8, ""), + ] { + let mut it = Lexer::new(src.bytes(), BufferType::Bytes(0)); + + assert_eq!( + it.by_ref().skip(2).next(), + Some(Token { + kind: kind.clone(), + buf: match &kind { + TokenType::Number => Buffer::MultiByte(buf.as_bytes().to_vec()), + TokenType::String => Buffer::MultiByte(buf.as_bytes().to_vec()), + _ => Buffer::Span(Span { first, end }), + } + }) + ); + + let mut it = Lexer::new(src.bytes(), BufferType::Span); + + assert_eq!( + it.by_ref().skip(2).next(), + Some(Token { + kind: kind.clone(), + buf: Buffer::Span(Span { first, end }), + }) + ); + } +} + #[test] fn special_values_closed_and_unclosed() { for &(src, ref kind, first, end) in &[