Skip to content

Commit

Permalink
Rollup merge of #69801 - petrochenkov:nonorm, r=Centril
Browse files Browse the repository at this point in the history
rustc_parse: Remove `Parser::normalized(_prev)_token`

Perform the "normalization" (renamed to "uninterpolation") on the fly when necessary.

The final part of #69579 #69384 #69376 #69211 #69034 #69006.
r? @Centril
  • Loading branch information
Centril authored Mar 9, 2020
2 parents 2409e70 + 7a30bb1 commit 2677d59
Show file tree
Hide file tree
Showing 14 changed files with 200 additions and 191 deletions.
9 changes: 6 additions & 3 deletions src/librustc_ast/attr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,10 @@ impl MetaItem {
}

impl AttrItem {
pub fn span(&self) -> Span {
self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
}

pub fn meta(&self, span: Span) -> Option<MetaItem> {
Some(MetaItem {
path: self.path.clone(),
Expand Down Expand Up @@ -437,7 +441,7 @@ impl MetaItem {
I: Iterator<Item = TokenTree>,
{
// FIXME: Share code with `parse_path`.
let path = match tokens.next() {
let path = match tokens.next().map(TokenTree::uninterpolate) {
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span }))
| Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
let mut segments = if let token::Ident(name, _) = kind {
Expand All @@ -453,7 +457,7 @@ impl MetaItem {
};
loop {
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
tokens.next()
tokens.next().map(TokenTree::uninterpolate)
{
segments.push(PathSegment::from_ident(Ident::new(name, span)));
} else {
Expand All @@ -470,7 +474,6 @@ impl MetaItem {
Path { span, segments }
}
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(ref path) => path.clone(),
_ => return None,
Expand Down
93 changes: 69 additions & 24 deletions src/librustc_ast/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use rustc_macros::HashStable_Generic;
use rustc_span::symbol::kw;
use rustc_span::symbol::Symbol;
use rustc_span::{self, Span, DUMMY_SP};
use std::fmt;
use std::mem;
use std::borrow::Cow;
use std::{fmt, mem};

#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
#[derive(HashStable_Generic)]
Expand Down Expand Up @@ -225,8 +225,15 @@ pub enum TokenKind {
/* Literals */
Literal(Lit),

/* Name components */
/// Identifier token.
/// Do not forget about `NtIdent` when you want to match on identifiers.
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated identifiers in the same way.
Ident(ast::Name, /* is_raw */ bool),
/// Lifetime identifier token.
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated lifetime identifiers in the same way.
Lifetime(ast::Name),

Interpolated(Lrc<Nonterminal>),
Expand Down Expand Up @@ -328,6 +335,19 @@ impl Token {
mem::replace(self, Token::dummy())
}

/// For interpolated tokens, returns a span of the fragment to which the interpolated
/// token refers. For all other tokens this is just a regular span.
/// It is particularly important to use this for identifiers and lifetimes
/// for which spans affect name resolution and edition checks.
/// Note that keywords are also identifiers, so they should use this
/// if they keep spans or perform edition checks.
pub fn uninterpolated_span(&self) -> Span {
match &self.kind {
Interpolated(nt) => nt.span(),
_ => self.span,
}
}

pub fn is_op(&self) -> bool {
match self.kind {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
Expand All @@ -345,7 +365,7 @@ impl Token {

/// Returns `true` if the token can appear at the start of an expression.
pub fn can_begin_expr(&self) -> bool {
match self.kind {
match self.uninterpolate().kind {
Ident(name, is_raw) =>
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
OpenDelim(..) | // tuple, array or block
Expand All @@ -363,12 +383,10 @@ impl Token {
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
Interpolated(ref nt) => match **nt {
NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
NtLiteral(..) |
NtExpr(..) |
NtBlock(..) |
NtPath(..) |
NtLifetime(..) => true,
NtPath(..) => true,
_ => false,
},
_ => false,
Expand All @@ -377,7 +395,7 @@ impl Token {

/// Returns `true` if the token can appear at the start of a type.
pub fn can_begin_type(&self) -> bool {
match self.kind {
match self.uninterpolate().kind {
Ident(name, is_raw) =>
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
OpenDelim(Paren) | // tuple
Expand All @@ -391,8 +409,7 @@ impl Token {
Lt | BinOp(Shl) | // associated path
ModSep => true, // global path
Interpolated(ref nt) => match **nt {
NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
NtTy(..) | NtPath(..) | NtLifetime(..) => true,
NtTy(..) | NtPath(..) => true,
_ => false,
},
_ => false,
Expand Down Expand Up @@ -433,38 +450,48 @@ impl Token {
///
/// Keep this in sync with `Lit::from_token`.
pub fn can_begin_literal_or_bool(&self) -> bool {
match self.kind {
match self.uninterpolate().kind {
Literal(..) | BinOp(Minus) => true,
Ident(name, false) if name.is_bool_lit() => true,
Interpolated(ref nt) => match &**nt {
NtIdent(ident, false) if ident.name.is_bool_lit() => true,
NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
_ => false,
},
_ => false,
}
}

// A convenience function for matching on identifiers during parsing.
// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
// into the regular identifier or lifetime token it refers to,
// otherwise returns the original token.
pub fn uninterpolate(&self) -> Cow<'_, Token> {
match &self.kind {
Interpolated(nt) => match **nt {
NtIdent(ident, is_raw) => {
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
}
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
_ => Cow::Borrowed(self),
},
_ => Cow::Borrowed(self),
}
}

/// Returns an identifier if this token is an identifier.
pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
match self.kind {
Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
Interpolated(ref nt) => match **nt {
NtIdent(ident, is_raw) => Some((ident, is_raw)),
_ => None,
},
let token = self.uninterpolate();
match token.kind {
Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
_ => None,
}
}

/// Returns a lifetime identifier if this token is a lifetime.
pub fn lifetime(&self) -> Option<ast::Ident> {
match self.kind {
Lifetime(name) => Some(ast::Ident::new(name, self.span)),
Interpolated(ref nt) => match **nt {
NtLifetime(ident) => Some(ident),
_ => None,
},
let token = self.uninterpolate();
match token.kind {
Lifetime(name) => Some(ast::Ident::new(name, token.span)),
_ => None,
}
}
Expand Down Expand Up @@ -714,6 +741,24 @@ pub enum Nonterminal {
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Nonterminal, 40);

impl Nonterminal {
fn span(&self) -> Span {
match self {
NtItem(item) => item.span,
NtBlock(block) => block.span,
NtStmt(stmt) => stmt.span,
NtPat(pat) => pat.span,
NtExpr(expr) | NtLiteral(expr) => expr.span,
NtTy(ty) => ty.span,
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
NtMeta(attr_item) => attr_item.span(),
NtPath(path) => path.span,
NtVis(vis) => vis.span,
NtTT(tt) => tt.span(),
}
}
}

impl PartialEq for Nonterminal {
fn eq(&self, rhs: &Self) -> bool {
match (self, rhs) {
Expand Down
7 changes: 7 additions & 0 deletions src/librustc_ast/tokenstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ impl TokenTree {
pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree {
TokenTree::token(token::CloseDelim(delim), span.close)
}

pub fn uninterpolate(self) -> TokenTree {
match self {
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
tt => tt,
}
}
}

impl<CTX> HashStable<CTX> for TokenStream
Expand Down
15 changes: 4 additions & 11 deletions src/librustc_ast/util/literal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,23 +191,16 @@ impl Lit {
///
/// Keep this in sync with `Token::can_begin_literal_or_bool`.
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
let lit = match token.kind {
let lit = match token.uninterpolate().kind {
token::Ident(name, false) if name.is_bool_lit() => {
token::Lit::new(token::Bool, name, None)
}
token::Literal(lit) => lit,
token::Interpolated(ref nt) => {
match &**nt {
token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
let lit = token::Lit::new(token::Bool, ident.name, None);
return Lit::from_lit_token(lit, ident.span);
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
if let ast::ExprKind::Lit(lit) = &expr.kind {
return Ok(lit.clone());
}
token::NtExpr(expr) | token::NtLiteral(expr) => {
if let ast::ExprKind::Lit(lit) = &expr.kind {
return Ok(lit.clone());
}
}
_ => {}
}
return Err(LitError::NotLiteral);
}
Expand Down
65 changes: 32 additions & 33 deletions src/librustc_builtin_macros/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,44 +156,43 @@ fn parse_args<'a>(
if p.token == token::Eof {
break;
} // accept trailing commas
if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
named = true;
let name = if let token::Ident(name, _) = p.normalized_token.kind {
match p.token.ident() {
Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => {
named = true;
p.bump();
name
} else {
unreachable!();
};
p.expect(&token::Eq)?;
let e = p.parse_expr()?;
if let Some(prev) = names.get(&ident.name) {
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", ident))
.span_label(args[*prev].span, "previously here")
.span_label(e.span, "duplicate argument")
.emit();
continue;
}

p.expect(&token::Eq)?;
let e = p.parse_expr()?;
if let Some(prev) = names.get(&name) {
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name))
.span_label(args[*prev].span, "previously here")
.span_label(e.span, "duplicate argument")
.emit();
continue;
// Resolve names into slots early.
// Since all the positional args are already seen at this point
// if the input is valid, we can simply append to the positional
// args. And remember the names.
let slot = args.len();
names.insert(ident.name, slot);
args.push(e);
}

// Resolve names into slots early.
// Since all the positional args are already seen at this point
// if the input is valid, we can simply append to the positional
// args. And remember the names.
let slot = args.len();
names.insert(name, slot);
args.push(e);
} else {
let e = p.parse_expr()?;
if named {
let mut err = ecx
.struct_span_err(e.span, "positional arguments cannot follow named arguments");
err.span_label(e.span, "positional arguments must be before named arguments");
for pos in names.values() {
err.span_label(args[*pos].span, "named argument");
_ => {
let e = p.parse_expr()?;
if named {
let mut err = ecx.struct_span_err(
e.span,
"positional arguments cannot follow named arguments",
);
err.span_label(e.span, "positional arguments must be before named arguments");
for pos in names.values() {
err.span_label(args[*pos].span, "named argument");
}
err.emit();
}
err.emit();
args.push(e);
}
args.push(e);
}
}
Ok((fmtstr, args, names))
Expand Down
19 changes: 5 additions & 14 deletions src/librustc_expand/mbe/macro_parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -750,17 +750,8 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na

/// The token is an identifier, but not `_`.
/// We prohibit passing `_` to macros expecting `ident` for now.
fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
match token.kind {
token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
token::Interpolated(ref nt) => match **nt {
token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
Some((ident.name, is_raw))
}
_ => None,
},
_ => None,
}
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
}

/// Checks whether a non-terminal may begin with a particular token.
Expand All @@ -783,7 +774,7 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
&& !token.is_keyword(kw::Let)
}
sym::ty => token.can_begin_type(),
sym::ident => get_macro_name(token).is_some(),
sym::ident => get_macro_ident(token).is_some(),
sym::literal => token.can_begin_literal_or_bool(),
sym::vis => match token.kind {
// The follow-set of :vis + "priv" keyword + interpolated
Expand Down Expand Up @@ -888,9 +879,9 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
sym::ty => token::NtTy(p.parse_ty()?),
// this could be handled like a token, since it is one
sym::ident => {
if let Some((name, is_raw)) = get_macro_name(&p.token) {
if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
p.bump();
token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
token::NtIdent(ident, is_raw)
} else {
let token_str = pprust::token_to_string(&p.token);
let msg = &format!("expected ident, found {}", &token_str);
Expand Down
5 changes: 2 additions & 3 deletions src/librustc_parse/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#![feature(crate_visibility_modifier)]

use rustc_ast::ast;
use rustc_ast::token::{self, Nonterminal, Token};
use rustc_ast::token::{self, Nonterminal};
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
use rustc_ast_pretty::pprust;
use rustc_data_structures::sync::Lrc;
Expand Down Expand Up @@ -171,8 +171,7 @@ fn maybe_source_file_to_parser(
let mut parser = stream_to_parser(sess, stream, None);
parser.unclosed_delims = unclosed_delims;
if parser.token == token::Eof {
let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
parser.set_token(Token::new(token::Eof, span));
parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
}

Ok(parser)
Expand Down
Loading

0 comments on commit 2677d59

Please sign in to comment.