Skip to content

Commit

Permalink
Refactor builder, destructure references when parsing
Browse files Browse the repository at this point in the history 
It doesn't seem as easy to compose/chain TokenStreams as it is to
collect a Vec<TokenTree> into a TokenStream. There is probably some
iterator API I could use instead.

Refs lambda-fairy#121
  • Loading branch information
@anxiousmodernman
anxiousmodernman committed Apr 9, 2018
1 parent e09ec27 commit cf7a170
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 40 deletions.
25 changes: 18 additions & 7 deletions maud_macros/src/build.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use proc_macro::{Delimiter, Group, Literal, Span, TokenStream, TokenTree};
use proc_macro::quote;
use proc_macro::token_stream;
use maud_htmlescape::Escaper;

//use proc_macro::token_stream::*;
use std::iter::FromIterator;
use maud_htmlescape::Escaper;

pub struct Builder {
output_ident: TokenTree,
Expand Down Expand Up @@ -36,12 +36,17 @@ impl Builder {

/// Reifies the `Builder` into a raw list of statements.
pub fn build(mut self) -> TokenStream {
// stmts: Vec<TokenStream>
let Builder { stmts, .. } = { self.flush(); self };
let mut i: token_stream::IntoIter;
for ts in stmts {
i.chain(ts);

// use a Group here?
let mut tts: Vec<TokenTree> = Vec::new();
for s in stmts.into_iter() {
let i = s.into_iter();
tts.extend(i);
}
TokenStream::from_iter(i.into_iter())

tts.into_iter().collect()
}

/// Pushes a statement, flushing the tail buffer in the process.
Expand Down Expand Up @@ -118,7 +123,13 @@ impl Builder {
// If the condition contains an opening brace `{`,
// wrap it in parentheses to avoid parse errors
if cond.clone().into_iter().any(|token| match token {
TokenTree::Group(Group { delimiter: Delimiter::Brace, .. }) => true,
TokenTree::Group(grp) => {
if grp.delimiter() == Delimiter::Brace {
true
} else {
false
}
},
_ => false,
}) {
let mut g = Group::new(Delimiter::Parenthesis, cond);
Expand Down
9 changes: 5 additions & 4 deletions maud_macros/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,11 @@ fn expand(input: TokenStream) -> TokenStream {
Ok(stmts) => stmts,
Err(e) => panic!(e),
};
quote!({
let tokens = quote!{
extern crate maud;
let mut $output_ident = String::with_capacity($size_hint as usize);
// let mut $output_ident = String::with_capacity($size_hint as usize);
$stmts
maud::PreEscaped($output_ident)
})
// maud::PreEscaped($output_ident)
};
tokens.into()
}
66 changes: 37 additions & 29 deletions maud_macros/src/parse.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,14 +158,14 @@ impl Parser {
self.element(&name, builder)?;
},
// Splice
TokenTree::Group(grp) if grp.delimiter() == Delimiter::Parenthesis => {
TokenTree::Group(ref grp) if grp.delimiter() == Delimiter::Parenthesis => {
self.advance();
// NOTE: the stream method returns a TokenStream without delimiters.
// Is this what we want?
builder.splice(grp.stream());
},
// Block
TokenTree::Group(grp) if grp.delimiter() == Delimiter::Brace => {
TokenTree::Group(ref grp) if grp.delimiter() == Delimiter::Brace => {
self.advance();
self.with_input(grp.stream()).markups(builder)?;
},
Expand All @@ -191,7 +191,7 @@ impl Parser {
fn if_expr(&mut self, builder: &mut Builder) -> ParseResult<()> {
loop {
match self.next() {
Some(TokenTree::Group(block)) if block.delimiter() == Delimiter::Brace => {
Some(TokenTree::Group(ref block)) if block.delimiter() == Delimiter::Brace => {
let block = self.block(block.stream(), block.span())?;
builder.push(block);
break;
Expand Down Expand Up @@ -220,6 +220,7 @@ impl Parser {
builder.push(TokenTree::Term(if_keyword));
self.if_expr(builder)?;
}
_ => () // NOTE: is this a bug?
}
self.else_if_expr(builder)
},
Expand All @@ -233,7 +234,7 @@ impl Parser {
fn while_expr(&mut self, builder: &mut Builder) -> ParseResult<()> {
loop {
match self.next() {
Some(TokenTree::Group(block)) if block.delimiter() == Delimiter::Brace => {
Some(TokenTree::Group(ref block)) if block.delimiter() == Delimiter::Brace => {
let block = self.block(block.stream(), block.span())?;
builder.push(block);
break;
Expand Down Expand Up @@ -262,7 +263,7 @@ impl Parser {
}
loop {
match self.next() {
Some(TokenTree::Group(block)) if block.delimiter() == Delimiter::Brace => {
Some(TokenTree::Group(ref block)) if block.delimiter() == Delimiter::Brace => {
let block = self.block(block.stream(), block.span())?;
builder.push(block);
break;
Expand All @@ -280,7 +281,7 @@ impl Parser {
fn match_expr(&mut self, builder: &mut Builder) -> ParseResult<()> {
loop {
match self.next() {
Some(TokenTree::Group(body)) if body.delimiter() == Delimiter::Brace => {
Some(TokenTree::Group(ref body)) if body.delimiter() == Delimiter::Brace => {
let body = self.with_input(body.stream()).match_arms()?;
builder.push(body);
break;
Expand All @@ -293,24 +294,22 @@ impl Parser {
}

fn match_arms(&mut self) -> ParseResult<TokenStream> {
let mut arms: token_stream::IntoIter;
let mut arms: Vec<TokenTree> = Vec::new();
while let Some(arm) = self.match_arm()? {
arms.chain(arm.into_iter());
arms.extend(arm);
}
Ok(TokenStream::from_iter(arms.into_iter()))
Ok(arms.into_iter().collect())
}

fn match_arm(&mut self) -> ParseResult<Option<TokenStream>> {
fn match_arm(&mut self) -> ParseResult<Option<Vec<TokenTree>>> {
let mut pat: Vec<TokenTree> = Vec::new();
loop {
match self.peek2() {
Some((
eq @ TokenTree::Op(o1),
Some(gt @ TokenTree::Op(o2)),
)) if o1.op() == '=' && o2.op() == '>' && o1.spacing() == Spacing::Joint => {
Some((TokenTree::Op(o1), Some(TokenTree::Op(o2))))
if o1.op() == '=' && o2.op() == '>' && o1.spacing() == Spacing::Joint => {
self.advance2();
pat.push(eq);
pat.push(gt);
pat.push(TokenTree::Op(o1));
pat.push(TokenTree::Op(o2));
break;
},
Some((token, _)) => {
Expand All @@ -327,7 +326,7 @@ impl Parser {
}
let body = match self.next() {
// $pat => { $stmts }
Some(TokenTree::Group(body)) if body.delimiter() == Delimiter::Brace => {
Some(TokenTree::Group(ref body)) if body.delimiter() == Delimiter::Brace => {
let body: TokenTree = self.block(body.stream(), body.span())?;
// Trailing commas are optional if the match arm is a braced block
if let Some(TokenTree::Op(o)) = self.peek() {
Expand Down Expand Up @@ -357,7 +356,8 @@ impl Parser {
},
None => return self.error("unexpected end of @match arm"),
};
Ok(Some(pat.into_iter().chain(iter::once(body)).collect()))
pat.push(body);
Ok(Some(pat))
}

/// Parses and renders a `@let` expression.
Expand All @@ -366,21 +366,29 @@ impl Parser {
fn let_expr(&mut self, builder: &mut Builder) -> ParseResult<()> {
loop {
match self.next() {
Some(token @ TokenTree::Op(o)) if o.op() == '=' => {
builder.push(token);
break;
Some(token) => {
match token {
TokenTree::Op(ref o) if o.op() == '=' => {
builder.push(token.clone());
break;
}
_ => builder.push(token),
}
},
Some(token) => builder.push(token),
None => return self.error("unexpected end of @let expression"),
}
}
loop {
match self.next() {
Some(token @ TokenTree::Op(o)) if o.op() == ';' => {
builder.push(token);
break;
Some(token) => {
match token {
TokenTree::Op(ref o) if o.op() == ';' => {
builder.push(token.clone());
break;
},
_ => builder.push(token),
}
},
Some(token) => builder.push(token),
None => return self.error("unexpected end of @let expression"),
}
}
Expand Down Expand Up @@ -421,7 +429,7 @@ impl Parser {
let token_after = attempt.next();
match (maybe_name, token_after) {
// Non-empty attribute
(Ok(name), Some(TokenTree::Op(o))) if o.op() == '=' => {
(Ok(ref name), Some(TokenTree::Op(ref o))) if o.op() == '=' => {
self.commit(attempt);
builder.attribute_start(&name);
{
Expand All @@ -433,7 +441,7 @@ impl Parser {
builder.attribute_end();
},
// Empty attribute
(Ok(name), Some(TokenTree::Op(o))) if o.op() == '?' => {
(Ok(ref name), Some(TokenTree::Op(ref o))) if o.op() == '?' => {
self.commit(attempt);
if let Some((cond, cond_span)) = self.attr_toggler() {
// Toggle the attribute based on a boolean expression
Expand Down Expand Up @@ -498,7 +506,7 @@ impl Parser {
/// Parses the `[cond]` syntax after an empty attribute or class shorthand.
fn attr_toggler(&mut self) -> Option<(TokenStream, Span)> {
match self.peek() {
Some(TokenTree::Group(grp)) if grp.delimiter() == Delimiter::Bracket => {
Some(TokenTree::Group(ref grp)) if grp.delimiter() == Delimiter::Bracket => {
self.advance();
Some((grp.stream(), grp.span()))
}
Expand Down

0 comments on commit cf7a170

Please sign in to comment.