Auto merge of #60679 - petrochenkov:lit2, r=matklad

Keep original literal tokens in AST

The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR.

The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes).
However, the size of `ast::Expr` stays the same.

The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](#59706).

Fixes a part of #43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc)
Fixes a part of #60495 (everything except for proc macro API doesn't need escaping anymore)
This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357).

cc @matklad

src/librustc/hir/lowering.rs

@@ -4100,7 +4100,7 @@ impl<'a> LoweringContext<'a> {
                 let ohs = P(self.lower_expr(ohs));
                 hir::ExprKind::Unary(op, ohs)
             }
-            ExprKind::Lit(ref l) => hir::ExprKind::Lit((*l).clone()),
+            ExprKind::Lit(ref l) => hir::ExprKind::Lit(respan(l.span, l.node.clone())),
             ExprKind::Cast(ref expr, ref ty) => {
                 let expr = P(self.lower_expr(expr));
                 hir::ExprKind::Cast(expr, self.lower_ty(ty, ImplTraitContext::disallowed()))

src/librustc/hir/mod.rs

@@ -20,7 +20,7 @@ use syntax_pos::{Span, DUMMY_SP, symbol::InternedString};
 use syntax::source_map::Spanned;
 use rustc_target::spec::abi::Abi;
 use syntax::ast::{self, CrateSugar, Ident, Name, NodeId, AsmDialect};
-use syntax::ast::{Attribute, Label, Lit, StrStyle, FloatTy, IntTy, UintTy};
+use syntax::ast::{Attribute, Label, LitKind, StrStyle, FloatTy, IntTy, UintTy};
 use syntax::attr::{InlineAttr, OptimizeAttr};
 use syntax::ext::hygiene::SyntaxContext;
 use syntax::ptr::P;
@@ -1331,6 +1331,9 @@ impl BodyOwnerKind {
     }
 }
 
+/// A literal.
+pub type Lit = Spanned<LitKind>;
+
 /// A constant (expression) that's not an item or associated item,
 /// but needs its own `DefId` for type-checking, const-eval, etc.
 /// These are usually found nested inside types (e.g., array lengths)

src/librustc/hir/print.rs

@@ -5,7 +5,7 @@ use syntax::parse::ParseSess;
 use syntax::parse::lexer::comments;
 use syntax::print::pp::{self, Breaks};
 use syntax::print::pp::Breaks::{Consistent, Inconsistent};
-use syntax::print::pprust::PrintState;
+use syntax::print::pprust::{self, PrintState};
 use syntax::ptr::P;
 use syntax::symbol::keywords;
 use syntax::util::parser::{self, AssocOp, Fixity};
@@ -18,7 +18,6 @@ use crate::hir::{GenericParam, GenericParamKind, GenericArg};
 use std::borrow::Cow;
 use std::cell::Cell;
 use std::io::{self, Write, Read};
-use std::iter::Peekable;
 use std::vec;
 
 pub enum AnnNode<'a> {
@@ -76,7 +75,6 @@ pub struct State<'a> {
     pub s: pp::Printer<'a>,
     cm: Option<&'a SourceMap>,
     comments: Option<Vec<comments::Comment>>,
-    literals: Peekable<vec::IntoIter<comments::Literal>>,
     cur_cmnt: usize,
     boxes: Vec<pp::Breaks>,
     ann: &'a (dyn PpAnn + 'a),
@@ -98,14 +96,6 @@ impl<'a> PrintState<'a> for State<'a> {
     fn cur_cmnt(&mut self) -> &mut usize {
         &mut self.cur_cmnt
     }
-
-    fn cur_lit(&mut self) -> Option<&comments::Literal> {
-        self.literals.peek()
-    }
-
-    fn bump_lit(&mut self) -> Option<comments::Literal> {
-        self.literals.next()
-    }
 }
 
 #[allow(non_upper_case_globals)]
@@ -116,18 +106,16 @@ pub const default_columns: usize = 78;
 
 
 /// Requires you to pass an input filename and reader so that
-/// it can scan the input text for comments and literals to
-/// copy forward.
+/// it can scan the input text for comments to copy forward.
 pub fn print_crate<'a>(cm: &'a SourceMap,
                        sess: &ParseSess,
                        krate: &hir::Crate,
                        filename: FileName,
                        input: &mut dyn Read,
                        out: Box<dyn Write + 'a>,
-                       ann: &'a dyn PpAnn,
-                       is_expanded: bool)
+                       ann: &'a dyn PpAnn)
                        -> io::Result<()> {
-    let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
+    let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
 
     // When printing the AST, we sometimes need to inject `#[no_std]` here.
     // Since you can't compile the HIR, it's not necessary.
@@ -143,36 +131,21 @@ impl<'a> State<'a> {
                           filename: FileName,
                           input: &mut dyn Read,
                           out: Box<dyn Write + 'a>,
-                          ann: &'a dyn PpAnn,
-                          is_expanded: bool)
+                          ann: &'a dyn PpAnn)
                           -> State<'a> {
-        let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
-
-        State::new(cm,
-                   out,
-                   ann,
-                   Some(cmnts),
-                   // If the code is post expansion, don't use the table of
-                   // literals, since it doesn't correspond with the literals
-                   // in the AST anymore.
-                   if is_expanded {
-                       None
-                   } else {
-                       Some(lits)
-                   })
+        let comments = comments::gather_comments(sess, filename, input);
+        State::new(cm, out, ann, Some(comments))
     }
 
     pub fn new(cm: &'a SourceMap,
                out: Box<dyn Write + 'a>,
                ann: &'a dyn PpAnn,
-               comments: Option<Vec<comments::Comment>>,
-               literals: Option<Vec<comments::Literal>>)
+               comments: Option<Vec<comments::Comment>>)
                -> State<'a> {
         State {
             s: pp::mk_printer(out, default_columns),
             cm: Some(cm),
             comments,
-            literals: literals.unwrap_or_default().into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -189,7 +162,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
             s: pp::mk_printer(Box::new(&mut wr), default_columns),
             cm: None,
             comments: None,
-            literals: vec![].into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -1276,6 +1248,12 @@ impl<'a> State<'a> {
         self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
     }
 
+    fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
+        self.maybe_print_comment(lit.span.lo())?;
+        let (token, suffix) = lit.node.to_lit_token();
+        self.writer().word(pprust::literal_to_string(token, suffix))
+    }
+
     pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
         self.maybe_print_comment(expr.span.lo())?;
         self.print_outer_attributes(&expr.attrs)?;

src/librustc/ich/impls_syntax.rs

@@ -162,7 +162,13 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
     Unsuffixed
 });
 
-impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
+impl_stable_hash_for!(struct ::syntax::ast::Lit {
+    node,
+    token,
+    suffix,
+    span
+});
+
 impl_stable_hash_for!(enum ::syntax::ast::LitKind {
     Str(value, style),
     Err(value),
@@ -175,6 +181,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitKind {
     Bool(value)
 });
 
+impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
+
 impl_stable_hash_for!(enum ::syntax::ast::IntTy { Isize, I8, I16, I32, I64, I128 });
 impl_stable_hash_for!(enum ::syntax::ast::UintTy { Usize, U8, U16, U32, U64, U128 });
 impl_stable_hash_for!(enum ::syntax::ast::FloatTy { F32, F64 });
@@ -280,6 +288,19 @@ for tokenstream::TokenStream {
     }
 }
 
+impl_stable_hash_for!(enum token::Lit {
+    Bool(val),
+    Byte(val),
+    Char(val),
+    Err(val),
+    Integer(val),
+    Float(val),
+    Str_(val),
+    ByteStr(val),
+    StrRaw(val, n),
+    ByteStrRaw(val, n)
+});
+
 fn hash_token<'a, 'gcx, W: StableHasherResult>(
     token: &token::Token,
     hcx: &mut StableHashingContext<'a>,
@@ -327,22 +348,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
         token::Token::CloseDelim(delim_token) => {
             std_hash::Hash::hash(&delim_token, hasher);
         }
-        token::Token::Literal(ref lit, ref opt_name) => {
-            mem::discriminant(lit).hash_stable(hcx, hasher);
-            match *lit {
-                token::Lit::Byte(val) |
-                token::Lit::Char(val) |
-                token::Lit::Err(val) |
-                token::Lit::Integer(val) |
-                token::Lit::Float(val) |
-                token::Lit::Str_(val) |
-                token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
-                token::Lit::StrRaw(val, n) |
-                token::Lit::ByteStrRaw(val, n) => {
-                    val.hash_stable(hcx, hasher);
-                    n.hash_stable(hcx, hasher);
-                }
-            };
+        token::Token::Literal(lit, opt_name) => {
+            lit.hash_stable(hcx, hasher);
             opt_name.hash_stable(hcx, hasher);
         }
 

src/librustc_driver/pretty.rs

@@ -805,8 +805,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                             src_name,
                                             &mut rdr,
                                             box out,
-                                            annotation.pp_ann(),
-                                            true)
+                                            annotation.pp_ann())
                 })
             }
 
@@ -829,8 +828,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                                                          src_name,
                                                                          &mut rdr,
                                                                          box out,
-                                                                         annotation.pp_ann(),
-                                                                         true);
+                                                                         annotation.pp_ann());
                     for node_id in uii.all_matching_node_ids(hir_map) {
                         let node = hir_map.get(node_id);
                         pp_state.print_node(node)?;

src/librustc_lint/types.rs

@@ -62,7 +62,7 @@ impl TypeLimits {
 /// Returns `true` iff the lint was overridden.
 fn lint_overflowing_range_endpoint<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     lit_val: u128,
     max: u128,
     expr: &'tcx hir::Expr,
@@ -132,7 +132,7 @@ fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) {
     }
 }
 
-fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &ast::Lit) -> Option<String> {
+fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &hir::Lit) -> Option<String> {
     let src = cx.sess().source_map().span_to_snippet(lit.span).ok()?;
     let firstch = src.chars().next()?;
 
@@ -249,7 +249,7 @@ fn lint_int_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     type_limits: &TypeLimits,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     t: ast::IntTy,
     v: u128,
 ) {
@@ -301,7 +301,7 @@ fn lint_int_literal<'a, 'tcx>(
 fn lint_uint_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     t: ast::UintTy,
 ) {
     let uint_type = if let ast::UintTy::Usize = t {
@@ -363,7 +363,7 @@ fn lint_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     type_limits: &TypeLimits,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
 ) {
     match cx.tables.node_type(e.hir_id).sty {
         ty::Int(t) => {

src/librustc_typeck/check/mod.rs

@@ -3083,7 +3083,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
 
     // AST fragment checking
     fn check_lit(&self,
-                 lit: &ast::Lit,
+                 lit: &hir::Lit,
                  expected: Expectation<'tcx>)
                  -> Ty<'tcx>
     {

src/librustdoc/clean/cfg.rs

@@ -414,10 +414,11 @@ impl<'a> fmt::Display for Html<'a> {
 mod test {
     use super::Cfg;
 
-    use syntax::symbol::Symbol;
+    use syntax_pos::DUMMY_SP;
     use syntax::ast::*;
+    use syntax::attr;
     use syntax::source_map::dummy_spanned;
-    use syntax_pos::DUMMY_SP;
+    use syntax::symbol::Symbol;
     use syntax::with_globals;
 
     fn word_cfg(s: &str) -> Cfg {
@@ -592,14 +593,10 @@ mod test {
             let mi = dummy_meta_item_word("all");
             assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
 
-            let mi = MetaItem {
-                path: Path::from_ident(Ident::from_str("all")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
-                    Symbol::intern("done"),
-                    StrStyle::Cooked,
-                ))),
-                span: DUMMY_SP,
-            };
+            let mi = attr::mk_name_value_item_str(
+                Ident::from_str("all"),
+                dummy_spanned(Symbol::intern("done"))
+            );
             assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
 
             let mi = dummy_meta_item_list!(all, [a, b]);
@@ -627,11 +624,12 @@ mod test {
     #[test]
     fn test_parse_err() {
         with_globals(|| {
-            let mi = MetaItem {
-                path: Path::from_ident(Ident::from_str("foo")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
-                span: DUMMY_SP,
-            };
+            let mi = attr::mk_name_value_item(
+                DUMMY_SP,
+                Ident::from_str("foo"),
+                LitKind::Bool(false),
+                DUMMY_SP,
+            );
             assert!(Cfg::parse(&mi).is_err());
 
             let mi = dummy_meta_item_list!(not, [a, b]);

src/librustdoc/html/highlight.rs

@@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {
 
                     // Number literals.
                     token::Integer(..) | token::Float(..) => Class::Number,
+
+                    token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
                 }
             }
 

src/libsyntax/ast.rs

@@ -6,6 +6,7 @@ pub use crate::symbol::{Ident, Symbol as Name};
 pub use crate::util::parser::ExprPrecedence;
 
 use crate::ext::hygiene::{Mark, SyntaxContext};
+use crate::parse::token;
 use crate::print::pprust;
 use crate::ptr::P;
 use crate::source_map::{dummy_spanned, respan, Spanned};
@@ -1350,8 +1351,19 @@ pub enum StrStyle {
     Raw(u16),
 }
 
-/// A literal.
-pub type Lit = Spanned<LitKind>;
+/// An AST literal.
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+pub struct Lit {
+    /// The original literal token as written in source code.
+    pub token: token::Lit,
+    /// The original literal suffix as written in source code.
+    pub suffix: Option<Symbol>,
+    /// The "semantic" representation of the literal lowered from the original tokens.
+    /// Strings are unescaped, hexadecimal forms are eliminated, etc.
+    /// FIXME: Remove this and only create the semantic representation during lowering to HIR.
+    pub node: LitKind,
+    pub span: Span,
+}
 
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq)]
 pub enum LitIntType {