feat(lexer): null value lexing

* including tests for normal null values and invalid ones
Byron · May 6, 2015 · dc2f9a2 · dc2f9a2
1 parent d4782c8
commit dc2f9a2
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 2 deletions.
diff --git a/src/lib.rs b/src/lib.rs
@@ -85,21 +85,37 @@ impl<I> Iterator for Lexer<I>
         let mut first = 0;
         let prev_end = self.prev_end;
 
+        let mut nb = ['x', 'x', 'x', 'x']; // null buffer
+        let mut nbi = 0usize;                 // null buffer index
+
         for c in self.chars.by_ref() {
             lcid += 1;
 
             let mut set_cursor = || {
                 first = prev_end + lcid - 1;
             };
 
+            if nbi > 0 {
+                nb[nbi] = c;
+                if nbi == 3 {
+                    // we know nb[0] is 'n'
+                    if nb[1] == 'u' && nb[2] == 'l' && nb[3] == 'l' {
+                        t = TokenType::NullValue;
+                    }
+                    break;
+                } else {
+                    nbi += 1;
+                }
+            }
+
             match c {
                 '\\' => {
                     if in_str {
                         ign_next = true;
                         continue; // this is allowed only here !
                     } else {
                         // invalid
-                        assert_eq!(t, TokenType::Invalid);
+                        debug_assert!(t == TokenType::Invalid);
                         set_cursor();
                         break
                     }
@@ -124,6 +140,14 @@ impl<I> Iterator for Lexer<I>
                         break;
                     }
                 },
+                'n' => {
+                    if !in_str {
+                        debug_assert!(nbi == 0);
+                        nb[0] = c;
+                        nbi = 1;
+                        set_cursor();
+                    }
+                }
                 '"' => {
                     if !ign_next {
                         if in_str {
@@ -153,7 +177,7 @@ impl<I> Iterator for Lexer<I>
             if in_str {
                 t = TokenType::Invalid;
             }
-            
+
             self.prev_end = self.prev_end + lcid;
             Some(Token {
                 kind: t,

diff --git a/tests/lexer.rs b/tests/lexer.rs
@@ -63,6 +63,27 @@ fn unclosed_string_value() {
 }
 
 
+#[test]
+fn null_value() {
+    let src = r#"{"n":null}"#;
+    let mut it = Lexer::new(src.chars());
+
+    assert_eq!(it.by_ref().skip(3).next(), Some(Token { kind: TokenType::NullValue, 
+                                                        span: Span { first: 5,
+                                                                     end:  9 } }));
+}
+
+#[test]
+fn unclosed_null_value() {
+    let src = r#"{"n":nuxl}"#;
+    let mut it = Lexer::new(src.chars());
+
+    // invalid null characters cause it to be invalid
+    assert_eq!(it.by_ref().skip(3).next(), Some(Token { kind: TokenType::Invalid, 
+                                                        span: Span { first: 5,
+                                                                     end:  9 } }));
+}
+
 #[test]
 fn multi_line_strings() {
     // Add code here