From 4f168d4b8384f7406bf860b682ee2d820febc7cc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 1 Feb 2020 22:32:48 +0800 Subject: [PATCH] simplification and modernization --- .clog.toml | 6 ---- Cargo.toml | 4 ++- benches/usage.rs | 32 ++++----------------- etc/sublime-text/json-tools.sublime-project | 30 ------------------- src/key_value_filter.rs | 8 +++--- src/lexer.rs | 19 ++++++------ src/lib.rs | 8 +++--- src/reader.rs | 10 ++++--- tests/filters.rs | 30 ++++--------------- tests/lexer.rs | 5 +--- 10 files changed, 37 insertions(+), 115 deletions(-) delete mode 100644 .clog.toml delete mode 100644 etc/sublime-text/json-tools.sublime-project diff --git a/.clog.toml b/.clog.toml deleted file mode 100644 index a27e3d1..0000000 --- a/.clog.toml +++ /dev/null @@ -1,6 +0,0 @@ -[clog] -repository = "https://github.com/Byron/json-tools" -from-latest-tag = true - -[sections] -Improvements = ["imp"] diff --git a/Cargo.toml b/Cargo.toml index aaaa2b7..0db119c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "json-tools" -version = "1.1.0" +edition = "2018" +version = "1.1.1" authors = ["Sebastian Thiel "] license = "MIT/Apache-2.0" description = "A zero-copy json-lexer, filters and serializer." repository = "https://github.com/Byron/json-tools" keywords = ["json", "tools", "lexer"] documentation = "https://docs.rs/crate/json-tools" +exclude = ["tests", "benches", "rustfmt.toml"] diff --git a/benches/usage.rs b/benches/usage.rs index 023b21a..207047b 100644 --- a/benches/usage.rs +++ b/benches/usage.rs @@ -181,10 +181,7 @@ fn span_lexer_throughput(b: &mut test::Bencher) { #[bench] fn span_lexer_span_token_reader_throughput(b: &mut test::Bencher) { b.iter(|| { - let mut r = TokenReader::new( - Lexer::new(NULL_RIDDEN.bytes(), BufferType::Span), - Some(NULL_RIDDEN), - ); + let mut r = TokenReader::new(Lexer::new(NULL_RIDDEN.bytes(), BufferType::Span), Some(NULL_RIDDEN)); io::copy(&mut r, &mut io::sink()).ok(); }); b.bytes = NULL_RIDDEN.len() as u64; @@ -193,10 +190,7 @@ fn span_lexer_span_token_reader_throughput(b: &mut test::Bencher) { #[bench] fn span_lexer_bytes_token_reader_throughput(b: &mut test::Bencher) { b.iter(|| { - let mut r = TokenReader::new( - Lexer::new(NULL_RIDDEN.bytes(), BufferType::Bytes(128)), - None, - ); + let mut r = TokenReader::new(Lexer::new(NULL_RIDDEN.bytes(), BufferType::Bytes(128)), None); io::copy(&mut r, &mut io::sink()).ok(); }); b.bytes = NULL_RIDDEN.len() as u64; @@ -206,10 +200,7 @@ fn span_lexer_bytes_token_reader_throughput(b: &mut test::Bencher) { fn bytes_token_producer_bytes_token_reader_throughput(b: &mut test::Bencher) { let mut ncb = 0u64; b.iter(|| { - let mut r = TokenReader::new( - KeyValueProducer::new(BufferType::Bytes(0)).take(30000), - None, - ); + let mut r = TokenReader::new(KeyValueProducer::new(BufferType::Bytes(0)).take(30000), None); ncb = io::copy(&mut r, &mut io::sink()).unwrap(); }); b.bytes = ncb; @@ -219,10 +210,7 @@ fn bytes_token_producer_bytes_token_reader_throughput(b: &mut test::Bencher) { fn span_token_producer_bytes_token_reader_throughput(b: &mut test::Bencher) { let mut ncb = 0u64; b.iter(|| { - let mut r = TokenReader::new( - KeyValueProducer::new(BufferType::Span).take(30000), - Some(KEY_VALUE_SRC), - ); + let mut r = TokenReader::new(KeyValueProducer::new(BufferType::Span).take(30000), Some(KEY_VALUE_SRC)); ncb = io::copy(&mut r, &mut io::sink()).unwrap(); }); b.bytes = ncb; @@ -242,10 +230,7 @@ fn bytes_lexer_throughput(b: &mut test::Bencher) { #[bench] fn span_filter_null_throughput(b: &mut test::Bencher) { b.iter(|| { - let f = FilterTypedKeyValuePairs::new( - Lexer::new(NULL_RIDDEN.bytes(), BufferType::Span), - TokenType::Null, - ); + let f = FilterTypedKeyValuePairs::new(Lexer::new(NULL_RIDDEN.bytes(), BufferType::Span), TokenType::Null); for t in f { test::black_box(t); } @@ -258,12 +243,7 @@ fn span_lexer_throughput_with_cursor(b: &mut test::Bencher) { use std::io::{Cursor, Read}; b.iter(|| { - let it = Lexer::new( - Cursor::new(NULL_RIDDEN.as_bytes()) - .bytes() - .filter_map(|r| r.ok()), - BufferType::Span, - ); + let it = Lexer::new(Cursor::new(NULL_RIDDEN.as_bytes()).bytes().filter_map(|r| r.ok()), BufferType::Span); for t in it { test::black_box(t); } diff --git a/etc/sublime-text/json-tools.sublime-project b/etc/sublime-text/json-tools.sublime-project deleted file mode 100644 index af4bf60..0000000 --- a/etc/sublime-text/json-tools.sublime-project +++ /dev/null @@ -1,30 +0,0 @@ -{ - "folders": - [ - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "Cargo.lock" - ], - "folder_exclude_patterns" : [ - "target", - ".git", - ], - }, - ], - "SublimeLinter": - { - "linters": - { - "rust": { - "@disable": false, - "args": [], - "crate-root": null, - "excludes": [], - "use-cargo": true, - "use-crate-root": true - }, - } - } -} diff --git a/src/key_value_filter.rs b/src/key_value_filter.rs index 94edf17..6a2adab 100644 --- a/src/key_value_filter.rs +++ b/src/key_value_filter.rs @@ -112,11 +112,11 @@ where let res = first_token(&mut self.buf, first_str_token); self.buf.push_back(colon_candidate); return res; - }// end is colon token - }// end have token (colon?) + } // end is colon token + } // end have token (colon?) None => return first_token(&mut self.buf, first_str_token), - }// end match next token (colon?) - }// end is string token, + } // end match next token (colon?) + } // end is string token, TokenType::Comma => { // NOTE: in case of malformed ,,,,, sequences, we just consider // this a peek, return the previous comma, and put back this one diff --git a/src/lexer.rs b/src/lexer.rs index 6ca1600..338325f 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -106,7 +106,7 @@ pub enum Buffer { #[derive(Debug, PartialEq, Clone)] pub enum BufferType { /// Use a `Buffer::MultiByte` were appropriate. Initialize it with the - /// given capcity (to obtain higher performance when pushing charcters) + /// given capacity (to obtain higher performance when pushing characters) Bytes(usize), Span, } @@ -121,7 +121,7 @@ where chars: chars.into_iter(), next_byte: None, cursor: 0, - buffer_type: buffer_type, + buffer_type, } } @@ -176,7 +176,7 @@ where { type Item = Token; - /// Lex the underlying bytte stream to generate tokens + /// Lex the underlying byte stream to generate tokens fn next(&mut self) -> Option { let mut t: Option = None; @@ -217,7 +217,7 @@ where } if *ign_digits > 0 { match c { - b'0'...b'9' | b'A'...b'F' | b'a'...b'f' => { + b'0'..=b'9' | b'A'..=b'F' | b'a'..=b'f' => { *ign_digits -= 1; continue; } @@ -257,7 +257,7 @@ where } } Mode::Number => match c { - b'0'...b'9' | b'-' | b'+' | b'.' | b'E' | b'e' => { + b'0'..=b'9' | b'-' | b'+' | b'.' | b'E' | b'e' => { if let Some(ref mut v) = buf { v.push(c); } @@ -325,7 +325,7 @@ where state = Mode::Null([c, b'x', b'x', b'x'], 1); set_cursor(self.cursor); } - b'0'...b'9' | b'-' | b'.' => { + b'0'..=b'9' | b'-' | b'.' => { state = Mode::Number; if let Some(ref mut v) = buf { v.push(c); @@ -381,12 +381,9 @@ where } else { let buf = match (&t, buf) { (&TokenType::String, Some(b)) | (&TokenType::Number, Some(b)) => Buffer::MultiByte(b), - _ => Buffer::Span(Span { - first: first, - end: self.cursor, - }), + _ => Buffer::Span(Span { first, end: self.cursor }), }; - Some(Token { kind: t, buf: buf }) + Some(Token { kind: t, buf }) } } } diff --git a/src/lib.rs b/src/lib.rs index 1ccfb8a..18ca312 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,10 @@ //! For usage examples, please have a look at the *tests* and *benchmarks*. -mod lexer; +mod iter_ext; mod key_value_filter; +mod lexer; mod reader; -mod iter_ext; -pub use lexer::{Buffer, BufferType, Lexer, Span, Token, TokenType}; +pub use iter_ext::IteratorExt; pub use key_value_filter::FilterTypedKeyValuePairs; +pub use lexer::{Buffer, BufferType, Lexer, Span, Token, TokenType}; pub use reader::TokenReader; -pub use iter_ext::IteratorExt; diff --git a/src/reader.rs b/src/reader.rs index 19ff8e7..780859c 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -1,6 +1,8 @@ -use std::io::{Read, Result, Write}; -use std::cmp; -use std::ptr; +use std::{ + cmp, + io::{Read, Result, Write}, + ptr, +}; use super::{Buffer, Token, TokenType}; @@ -89,7 +91,7 @@ impl<'a, I: IntoIterator> Read for TokenReader<'a, I> { if btc < bytes.len() { debug_assert!(bl == 0); - try!(self.buf.write_all(&bytes[btc..])); + self.buf.write_all(&bytes[btc..])?; } if bl == 0 { diff --git a/tests/filters.rs b/tests/filters.rs index ef6ef4d..7eea9ef 100644 --- a/tests/filters.rs +++ b/tests/filters.rs @@ -1,39 +1,19 @@ extern crate json_tools; -use std::io::{self, Cursor, Read}; use json_tools::{BufferType, FilterTypedKeyValuePairs, IteratorExt, Lexer, TokenReader, TokenType}; +use std::io::{self, Cursor, Read}; #[test] fn filter_null_values() { for &(src, want, count, fcount) in &[ - ( - r#"{ "s":null, "s":true, "s":null }"#, - r#"{"s":true}"#, - 13, - 5, - ), + (r#"{ "s":null, "s":true, "s":null }"#, r#"{"s":true}"#, 13, 5), (r#"{"s " : null, "s":null, "s":null }"#, r#"{}"#, 13, 2), (r#"{"s":true, "s":null, "s":null }"#, r#"{"s":true}"#, 13, 5), (r#"{"s":true, "s":null "s":null }"#, r#"{"s":true}"#, 12, 5), // invalid is fine - ( - r#"{"s":true,,,, "s":null, "s":null }"#, - r#"{"s":true,,,}"#, - 16, - 8, - ), + (r#"{"s":true,,,, "s":null, "s":null }"#, r#"{"s":true,,,}"#, 16, 8), (r#"{"s":null, "s":null, "s":true }"#, r#"{"s":true}"#, 13, 5), - ( - r#"{"s":true, "s":null, "s":true }"#, - r#"{"s":true,"s":true}"#, - 13, - 9, - ), - ( - r#"{"s":true, "s":null "s":true }"#, - r#"{"s":true"s":true}"#, - 12, - 8, - ), + (r#"{"s":true, "s":null, "s":true }"#, r#"{"s":true,"s":true}"#, 13, 9), + (r#"{"s":true, "s":null "s":true }"#, r#"{"s":true"s":true}"#, 12, 8), ] { assert_eq!(Lexer::new(src.bytes(), BufferType::Span).count(), count); let new_filter = |bt| FilterTypedKeyValuePairs::new(Lexer::new(src.bytes(), bt), TokenType::Null); diff --git a/tests/lexer.rs b/tests/lexer.rs index 14ffdbd..2b94eca 100644 --- a/tests/lexer.rs +++ b/tests/lexer.rs @@ -195,10 +195,7 @@ fn special_values_closed_and_unclosed() { Lexer::new(src.bytes(), BufferType::Span).skip(3).next(), Some(Token { kind: kind.clone(), - buf: Buffer::Span(Span { - first: first, - end: end, - }), + buf: Buffer::Span(Span { first: first, end: end }), }) ); }