fix(null-filter): removed possible overflow

Previously it was possible to over-allocate memory by feeding us lots
of `,` characters. This was alleviated by allowing a one-token
look-ahead (implemented through put-back).

Also: slightly reduced match-case verbosity of lexer

src/filter_null.rs

@@ -3,14 +3,32 @@ use std::collections::VecDeque;
 
 pub struct FilterNull<I: Iterator<Item=Token>> {
     src: I,
+    // NOTE: We could remove the deck and keep the 3 slots we need as Option<Token>
+    // 0: optional comma
+    // 1: first string
+    // 2: colon
     buf: VecDeque<Token>,
+    next_token: Option<Token>
 }
 
 impl<I: Iterator<Item=Token>> FilterNull<I> {
     pub fn new(src: I) -> FilterNull<I> {
         FilterNull {
             src: src,
-            buf: VecDeque::with_capacity(2),
+            buf: VecDeque::with_capacity(3),
+            next_token: None
+        }
+    }
+
+    fn put_back(&mut self, t: Token) {
+        debug_assert!(self.next_token.is_none());
+        self.next_token = Some(t);
+    }
+
+    fn next_token(&mut self) -> Option<Token> {
+        match self.next_token.take() {
+            Some(t) => Some(t),
+            None => self.src.next()
         }
     }
 }
@@ -23,7 +41,7 @@ impl<I> Iterator for FilterNull<I> where I: Iterator<Item=Token>{
             return self.buf.pop_front()
         }
 
-        fn last_token(v: &mut VecDeque<Token>, t: Token) -> Option<Token> {
+        fn first_token(v: &mut VecDeque<Token>, t: Token) -> Option<Token> {
             // assume it's a comma
             if v.len() > 0 {
                 v.push_back(t);
@@ -33,19 +51,19 @@ impl<I> Iterator for FilterNull<I> where I: Iterator<Item=Token>{
             }
         }
 
-        let token = self.src.next();
+        let token = self.next_token();
         match token {
             Some(mut first_str_candidate) => {
                 loop {
                     match first_str_candidate.kind {
                         TokenType::String => {
                             let first_str_token = first_str_candidate;
-                            match self.src.next() {
+                            match self.next_token() {
                                 Some(colon_candidate) => {
                                     match colon_candidate.kind {
                                         TokenType::Colon => {
                                             let colon = colon_candidate;
-                                            match self.src.next() {
+                                            match self.next_token() {
                                                 Some(second_str_candidate) => {
                                                     match second_str_candidate.kind {
                                                         TokenType::Null => {
@@ -54,11 +72,11 @@ impl<I> Iterator for FilterNull<I> where I: Iterator<Item=Token>{
                                                             // See if there is a (optional) comma
                                                             // If self.buf has anything, it must be commas !
                                                             // Usually, it is only 0 or 1 !
-                                                            match self.src.next() {
+                                                            match self.next_token() {
                                                                 Some(comma_candidate) => {
                                                                     first_str_candidate = 
                                                                         match match comma_candidate.kind {
-                                                                            TokenType::Comma => self.src.next(),
+                                                                            TokenType::Comma => self.next_token(),
                                                                             _ => {
                                                                                 self.buf.pop_front();
                                                                                 Some(comma_candidate)
@@ -73,45 +91,50 @@ impl<I> Iterator for FilterNull<I> where I: Iterator<Item=Token>{
                                                             }
                                                         },
                                                         _ => {
-                                                            let res = last_token(&mut self.buf, first_str_token);
+                                                            let res = first_token(&mut self.buf, first_str_token);
                                                             self.buf.push_back(colon);
                                                             self.buf.push_back(second_str_candidate);
                                                             return res
                                                         }
                                                     }
                                                 },
                                                 None => {
-                                                    let res = last_token(&mut self.buf, first_str_token);
+                                                    let res = first_token(&mut self.buf, first_str_token);
                                                     self.buf.push_back(colon);
                                                     return res
                                                 }
                                             }
                                         },
                                         _ => {
-                                            let res = last_token(&mut self.buf, first_str_token);
+                                            let res = first_token(&mut self.buf, first_str_token);
                                             self.buf.push_back(colon_candidate);
                                             return res
                                         }
                                     }// end is colon token
                                 },// end have token (colon?)
-                                None => return last_token(&mut self.buf, first_str_token),
+                                None => return first_token(&mut self.buf, first_str_token),
                             }// end match next token (colon?)
                          }// end is string token,
                          TokenType::Comma => {
-                            match self.src.next() {
+                            // NOTE: in case of malformed ,,,,, sequences, we just consider
+                            // this a peek, return the previous comma, and put back this one
+                            if self.buf.len() > 0 {
+                                debug_assert_eq!(self.buf.len(), 1);
+                                self.put_back(first_str_candidate);
+                                return self.buf.pop_front()
+                            }
+                            match self.next_token() {
                                 None => return Some(first_str_candidate),
                                 Some(t) => {
                                     // keep it, it will be returned first in case we 
                                     // end up not having a match
-                                    // NOTE: will keep pushing back malformed ,,,,, sequences
-                                    // which could be used for DOS attacks. TODO: impl. put_back
                                     self.buf.push_back(first_str_candidate);
                                     first_str_candidate = t;
                                     continue
                                 }
                             }
                          },
-                         _ => return last_token(&mut self.buf, first_str_candidate),
+                         _ => return first_token(&mut self.buf, first_str_candidate),
                      }// end match token kind (string?)
                 }// end inner str candidate LOOP
             },// end have token

src/lexer.rs

@@ -170,16 +170,7 @@ impl<I> Iterator for Lexer<I>
                 },
                 Mode::Number => {
                     match c {
-                         '0'
-                        |'1'
-                        |'2'
-                        |'3'
-                        |'4'
-                        |'5'
-                        |'6'
-                        |'7'
-                        |'8'
-                        |'9'
+                         '0' ... '9'
                         |'-'
                         |'.' => {
                             continue;
@@ -229,16 +220,7 @@ impl<I> Iterator for Lexer<I>
                             state = Mode::Null([c, 'x', 'x', 'x'], 1);
                             set_cursor(self.cursor);
                         },
-                         '0'
-                        |'1'
-                        |'2'
-                        |'3'
-                        |'4'
-                        |'5'
-                        |'6'
-                        |'7'
-                        |'8'
-                        |'9'
+                         '0' ... '9'
                         |'-'
                         |'.'=> {
                             state = Mode::Number;