fix(null-filter): handle consecutive null values

With this in place, we handle null value filtering pretty well, as
the tests indicate too.

However, we may still leave a trailing comma in non-null values which
could be a problem and thus shouldn't be done !

src/filter_null.rs

@@ -25,56 +25,63 @@ impl<I> Iterator for FilterNull<I> where I: Iterator<Item=Token>{
 
         let token = self.src.next();
         match token {
-            Some(first_str_candidate) => {
-                match first_str_candidate.kind {
-                    TokenType::String => {
-                        let first_str_token = first_str_candidate;
-                        match self.src.next() {
-                            Some(colon_candidate) => {
-                                match colon_candidate.kind {
-                                    TokenType::Colon => {
-                                        let colon = colon_candidate;
-                                        match self.src.next() {
-                                            Some(second_str_candidate) => {
-                                                match second_str_candidate.kind {
-                                                    TokenType::Null => {
-                                                        // WE HAVE A STR : STR triplete, and we forget it
-                                                        // This works by just not putting it onto the ringbuffer
-                                                        // See if there is a (optional) comma
-                                                        match self.src.next() {
-                                                            Some(comma_candidate) => {
-                                                                match comma_candidate.kind {
-                                                                    TokenType::Comma => self.src.next(),
-                                                                    _ => Some(comma_candidate)
-                                                                }
-                                                            },
-                                                            None => None,
+            Some(mut first_str_candidate) => {
+                loop {
+                    match first_str_candidate.kind {
+                        TokenType::String => {
+                            let first_str_token = first_str_candidate;
+                            match self.src.next() {
+                                Some(colon_candidate) => {
+                                    match colon_candidate.kind {
+                                        TokenType::Colon => {
+                                            let colon = colon_candidate;
+                                            match self.src.next() {
+                                                Some(second_str_candidate) => {
+                                                    match second_str_candidate.kind {
+                                                        TokenType::Null => {
+                                                            // WE HAVE A STR : STR triplete, and we forget it
+                                                            // This works by just not putting it onto the ringbuffer
+                                                            // See if there is a (optional) comma
+                                                            match self.src.next() {
+                                                                Some(comma_candidate) => {
+                                                                    first_str_candidate = 
+                                                                        match match comma_candidate.kind {
+                                                                            TokenType::Comma => self.src.next(),
+                                                                            _ => Some(comma_candidate)
+                                                                        } {
+                                                                            Some(t) => t,
+                                                                            None => return None,
+                                                                        };
+                                                                    continue;
+                                                                },
+                                                                None => return None,
+                                                            }
+                                                        },
+                                                        _ => {
+                                                            self.buf.push_back(colon);
+                                                            self.buf.push_back(second_str_candidate);
+                                                            return Some(first_str_token)
                                                         }
-                                                    },
-                                                    _ => {
-                                                        self.buf.push_back(colon);
-                                                        self.buf.push_back(second_str_candidate);
-                                                        Some(first_str_token)
                                                     }
+                                                },
+                                                None => {
+                                                    self.buf.push_back(colon);
+                                                    return Some(first_str_token)
                                                 }
-                                            },
-                                            None => {
-                                                self.buf.push_back(colon);
-                                                Some(first_str_token)
                                             }
+                                        },
+                                        _ => {
+                                            self.buf.push_back(colon_candidate);
+                                            return Some(first_str_token)
                                         }
-                                    },
-                                    _ => {
-                                        self.buf.push_back(colon_candidate);
-                                        Some(first_str_token)
-                                    }
-                                }// end is colon token
-                            },// end have token (colon?)
-                            None => Some(first_str_token),
-                        }// end match next token (colon?)
-                     }// end is string token,
-                     _ => Some(first_str_candidate),
-                 }// end match token kind (string?)
+                                    }// end is colon token
+                                },// end have token (colon?)
+                                None => return Some(first_str_token),
+                            }// end match next token (colon?)
+                         }// end is string token,
+                         _ => return Some(first_str_candidate),
+                     }// end match token kind (string?)
+                }// end inner str candidate loop
             },// end have token
             None => None,
         }

tests/filters.rs

@@ -4,7 +4,11 @@ use json_tools::{Lexer, FilterNull};
 
 #[test]
 fn filter_null_values() {
-    let src = r#"{"s":null, "v":true, "s":null }"#;
-    assert_eq!(Lexer::new(src.chars()).count(), 13);
-    assert_eq!(FilterNull::new(Lexer::new(src.chars())).count(), 6);
+    for &(src, count, fcount) in &[(r#"{"s":null, "s":true, "s":null }"#, 13, 6), //6 should be 5!
+                                   (r#"{"s":null, "s":null, "s":null }"#, 13, 2),
+                                   (r#"{"s":true, "s":null, "s":null }"#, 13, 6),
+                                   (r#"{"s":null, "s":null, "s":true }"#, 13, 5)] {
+        assert_eq!(Lexer::new(src.chars()).count(), count);
+        assert_eq!(FilterNull::new(Lexer::new(src.chars())).count(), fcount);    
+    }
 }