feat(lexer): datastructures and basic tests

The tests still fail as the actual lexer implementation is still to be
done.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "json-tools"
-version = "0.0.1"
+version = "0.1.0"
 authors = ["Sebastian Thiel <[email protected]>"]
 license = "MIT"
 description = "A collections of tools to handle json encoded data"

src/lib.rs

@@ -1,3 +1,84 @@
-#[test]
-fn it_works() {
+/// A lexer for utf-8 encoded json data
+pub struct Lexer<I> 
+                where I: Iterator<Item=char> {
+    chars: I,
+    cursor: u64,
 }
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+    /// All whitespace
+    WhiteSpace,
+
+    /// `{`
+    CurlyOpen,
+    /// `}`
+    CurlyClose,
+
+    /// `[`
+    BracketOpen,
+    /// `]`
+    BracketClose,
+
+    /// `:`
+    Colon,
+    /// `,`
+    Comma,
+
+
+    /// A json string , like `"foo"`
+    StringValue,
+    /// `true`
+    BooleanTrue,
+    /// `false`
+    BooleanFalse,
+    /// any json number, like `1.24123` or `123`
+    Number,
+    /// `null`
+    NullValue,
+
+    /// The type of the token could not be identified.
+    /// Should be removed if this lexer is ever to be feature complete
+    Invalid,
+}
+
+/// A pair of indices into the character stream returned by our source 
+/// iterator.
+/// It is an exclusive range.
+#[derive(Debug, PartialEq)]
+pub struct Span {
+    /// Index of the first the character
+    pub first: u64,
+    /// Index one past the last character
+    pub end: u64,
+}
+
+/// A lexical token, identifying its kind and span.
+#[derive(Debug, PartialEq)]
+pub struct Token {
+    /// The exact type of the token
+    pub kind: TokenType,
+    /// The span allows to reference back into the source character stream 
+    /// to obtain the string making up the token.
+    pub span: Span,
+}
+
+impl<I> Lexer<I> where I: Iterator<Item=char> {
+    /// Returns a new Lexer from a given character iterator.
+    pub fn new(chars: I) -> Lexer<I> {
+        Lexer {
+            chars: chars,
+            cursor: 0,
+        }
+    }
+}
+
+impl<I> Iterator for Lexer<I> 
+                    where I: Iterator<Item=char> {
+    type Item = Token;
+
+    /// Lex the underlying character stream to generate tokens
+    fn next(&mut self) -> Option<Token> {
+        None
+    }
+}

tests/lexer.rs

@@ -0,0 +1,25 @@
+extern crate json_tools;
+
+use json_tools::{Lexer, Token, Span, TokenType};
+
+#[test]
+fn unicode() {
+    let src = r#"{ "face": "😂" }"#;
+    let mut it = Lexer::new(src.chars());
+
+    assert_eq!(it.next(), Some(Token { kind: TokenType::BracketLeft, 
+                                       span: Span { first: 0,
+                                                    end:  1 } }));
+}
+
+
+#[test]
+fn string_escaping() {
+    // Add code here
+}
+
+
+#[test]
+fn multi_line_strings() {
+    // Add code here
+}