Auto merge of #64570 - tmandry:rollup-3pg02lj, r=tmandry

Rollup of 4 pull requests

Successful merges:

 - #64486 (Print out more information for `-Zunpretty=expanded,hygiene`)
 - #64503 (rename Allocation::retag -> with_tags_and_extra)
 - #64516 (update Nomicon and Reference)
 - #64528 (Load proc macro metadata in the correct order.)

Failed merges:

r? @ghost

src/librustc/mir/interpret/allocation.rs

@@ -130,9 +130,9 @@ impl<Tag> Allocation<Tag> {
     }
 }
 
-impl Allocation<()> {
+impl Allocation<(), ()> {
     /// Add Tag and Extra fields
-    pub fn retag<T, E>(
+    pub fn with_tags_and_extra<T, E>(
         self,
         mut tagger: impl FnMut(AllocId) -> T,
         extra: E,

src/librustc_driver/pretty.rs

@@ -326,6 +326,7 @@ impl<'hir> pprust::PpAnn for IdentifiedAnnotation<'hir> {
     }
     fn post(&self, s: &mut pprust::State<'_>, node: pprust::AnnNode<'_>) {
         match node {
+            pprust::AnnNode::Crate(_) |
             pprust::AnnNode::Ident(_) |
             pprust::AnnNode::Name(_) => {},
 
@@ -431,14 +432,18 @@ impl<'a> pprust::PpAnn for HygieneAnnotation<'a> {
         match node {
             pprust::AnnNode::Ident(&ast::Ident { name, span }) => {
                 s.s.space();
-                // FIXME #16420: this doesn't display the connections
-                // between syntax contexts
                 s.synth_comment(format!("{}{:?}", name.as_u32(), span.ctxt()))
             }
             pprust::AnnNode::Name(&name) => {
                 s.s.space();
                 s.synth_comment(name.as_u32().to_string())
             }
+            pprust::AnnNode::Crate(_) => {
+                s.s.hardbreak();
+                let verbose = self.sess.verbose();
+                s.synth_comment(syntax_pos::hygiene::debug_hygiene_data(verbose));
+                s.s.hardbreak_if_not_bol();
+            }
             _ => {}
         }
     }

src/librustc_metadata/decoder.rs

@@ -489,7 +489,11 @@ impl<'a, 'tcx> CrateMetadata {
 
     fn raw_proc_macro(&self, id: DefIndex) -> &ProcMacro {
         // DefIndex's in root.proc_macro_data have a one-to-one correspondence
-        // with items in 'raw_proc_macros'
+        // with items in 'raw_proc_macros'.
+        // NOTE: If you update the order of macros in 'proc_macro_data' for any reason,
+        // you must also update src/libsyntax_ext/proc_macro_harness.rs
+        // Failing to do so will result in incorrect data being associated
+        // with proc macros when deserialized.
         let pos = self.root.proc_macro_data.unwrap().decode(self).position(|i| i == id).unwrap();
         &self.raw_proc_macros.unwrap()[pos]
     }

src/libsyntax/ast.rs

@@ -2387,7 +2387,7 @@ pub enum ItemKind {
     ),
     /// A macro invocation.
     ///
-    /// E.g., `macro_rules! foo { .. }` or `foo!(..)`.
+    /// E.g., `foo!(..)`.
     Mac(Mac),
 
     /// A macro definition.

src/libsyntax/print/pprust.rs

@@ -35,6 +35,7 @@ pub enum AnnNode<'a> {
     SubItem(ast::NodeId),
     Expr(&'a ast::Expr),
     Pat(&'a ast::Pat),
+    Crate(&'a ast::Crate),
 }
 
 pub trait PpAnn {
@@ -140,6 +141,7 @@ pub fn print_crate<'a>(cm: &'a SourceMap,
 
     s.print_mod(&krate.module, &krate.attrs);
     s.print_remaining_comments();
+    s.ann.post(&mut s, AnnNode::Crate(krate));
     s.s.eof()
 }
 
@@ -1369,8 +1371,12 @@ impl<'a> State<'a> {
                 }
             }
             ast::ItemKind::MacroDef(ref macro_def) => {
-                let (kw, has_bang) =
-                    if macro_def.legacy { ("macro_rules", true) } else { ("macro", false) };
+                let (kw, has_bang) = if macro_def.legacy {
+                    ("macro_rules", true)
+                } else {
+                    self.print_visibility(&item.vis);
+                    ("macro", false)
+                };
                 self.print_mac_common(
                     Some(MacHeader::Keyword(kw)),
                     has_bang,

src/libsyntax_ext/proc_macro_harness.rs

@@ -20,15 +20,24 @@ struct ProcMacroDerive {
     attrs: Vec<ast::Name>,
 }
 
+enum ProcMacroDefType {
+    Attr,
+    Bang
+}
+
 struct ProcMacroDef {
     function_name: Ident,
     span: Span,
+    def_type: ProcMacroDefType
+}
+
+enum ProcMacro {
+    Derive(ProcMacroDerive),
+    Def(ProcMacroDef)
 }
 
 struct CollectProcMacros<'a> {
-    derives: Vec<ProcMacroDerive>,
-    attr_macros: Vec<ProcMacroDef>,
-    bang_macros: Vec<ProcMacroDef>,
+    macros: Vec<ProcMacro>,
     in_root: bool,
     handler: &'a errors::Handler,
     is_proc_macro_crate: bool,
@@ -46,22 +55,22 @@ pub fn inject(sess: &ParseSess,
     let ecfg = ExpansionConfig::default("proc_macro".to_string());
     let mut cx = ExtCtxt::new(sess, ecfg, resolver);
 
-    let (derives, attr_macros, bang_macros) = {
-        let mut collect = CollectProcMacros {
-            derives: Vec::new(),
-            attr_macros: Vec::new(),
-            bang_macros: Vec::new(),
-            in_root: true,
-            handler,
-            is_proc_macro_crate,
-            is_test_crate,
-        };
-        if has_proc_macro_decls || is_proc_macro_crate {
-            visit::walk_crate(&mut collect, &krate);
-        }
-        (collect.derives, collect.attr_macros, collect.bang_macros)
+    let mut collect = CollectProcMacros {
+        macros: Vec::new(),
+        in_root: true,
+        handler,
+        is_proc_macro_crate,
+        is_test_crate,
     };
 
+    if has_proc_macro_decls || is_proc_macro_crate {
+        visit::walk_crate(&mut collect, &krate);
+    }
+    // NOTE: If you change the order of macros in this vec
+    // for any reason, you must also update 'raw_proc_macro'
+    // in src/librustc_metadata/decoder.rs
+    let macros = collect.macros;
+
     if !is_proc_macro_crate {
         return krate
     }
@@ -74,7 +83,7 @@ pub fn inject(sess: &ParseSess,
         return krate;
     }
 
-    krate.module.items.push(mk_decls(&mut cx, &derives, &attr_macros, &bang_macros));
+    krate.module.items.push(mk_decls(&mut cx, &macros));
 
     krate
 }
@@ -161,12 +170,12 @@ impl<'a> CollectProcMacros<'a> {
         };
 
         if self.in_root && item.vis.node.is_pub() {
-            self.derives.push(ProcMacroDerive {
+            self.macros.push(ProcMacro::Derive(ProcMacroDerive {
                 span: item.span,
                 trait_name: trait_ident.name,
                 function_name: item.ident,
                 attrs: proc_attrs,
-            });
+            }));
         } else {
             let msg = if !self.in_root {
                 "functions tagged with `#[proc_macro_derive]` must \
@@ -180,10 +189,11 @@ impl<'a> CollectProcMacros<'a> {
 
     fn collect_attr_proc_macro(&mut self, item: &'a ast::Item) {
         if self.in_root && item.vis.node.is_pub() {
-            self.attr_macros.push(ProcMacroDef {
+            self.macros.push(ProcMacro::Def(ProcMacroDef {
                 span: item.span,
                 function_name: item.ident,
-            });
+                def_type: ProcMacroDefType::Attr
+            }));
         } else {
             let msg = if !self.in_root {
                 "functions tagged with `#[proc_macro_attribute]` must \
@@ -197,10 +207,11 @@ impl<'a> CollectProcMacros<'a> {
 
     fn collect_bang_proc_macro(&mut self, item: &'a ast::Item) {
         if self.in_root && item.vis.node.is_pub() {
-            self.bang_macros.push(ProcMacroDef {
+            self.macros.push(ProcMacro::Def(ProcMacroDef {
                 span: item.span,
                 function_name: item.ident,
-            });
+                def_type: ProcMacroDefType::Bang
+            }));
         } else {
             let msg = if !self.in_root {
                 "functions tagged with `#[proc_macro]` must \
@@ -322,9 +333,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
 //      }
 fn mk_decls(
     cx: &mut ExtCtxt<'_>,
-    custom_derives: &[ProcMacroDerive],
-    custom_attrs: &[ProcMacroDef],
-    custom_macros: &[ProcMacroDef],
+    macros: &[ProcMacro],
 ) -> P<ast::Item> {
     let expn_id = cx.resolver.expansion_for_ast_pass(
         DUMMY_SP,
@@ -354,26 +363,32 @@ fn mk_decls(
         let proc_macro_ty_method_path = |method| cx.expr_path(cx.path(span, vec![
             proc_macro, bridge, client, proc_macro_ty, method,
         ]));
-        custom_derives.iter().map(|cd| {
-            cx.expr_call(span, proc_macro_ty_method_path(custom_derive), vec![
-                cx.expr_str(cd.span, cd.trait_name),
-                cx.expr_vec_slice(
-                    span,
-                    cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::<Vec<_>>()
-                ),
-                local_path(cd.span, cd.function_name),
-            ])
-        }).chain(custom_attrs.iter().map(|ca| {
-            cx.expr_call(span, proc_macro_ty_method_path(attr), vec![
-                cx.expr_str(ca.span, ca.function_name.name),
-                local_path(ca.span, ca.function_name),
-            ])
-        })).chain(custom_macros.iter().map(|cm| {
-            cx.expr_call(span, proc_macro_ty_method_path(bang), vec![
-                cx.expr_str(cm.span, cm.function_name.name),
-                local_path(cm.span, cm.function_name),
-            ])
-        })).collect()
+        macros.iter().map(|m| {
+            match m {
+                ProcMacro::Derive(cd) => {
+                    cx.expr_call(span, proc_macro_ty_method_path(custom_derive), vec![
+                        cx.expr_str(cd.span, cd.trait_name),
+                        cx.expr_vec_slice(
+                            span,
+                            cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::<Vec<_>>()
+                        ),
+                        local_path(cd.span, cd.function_name),
+                    ])
+                },
+                ProcMacro::Def(ca) => {
+                    let ident = match ca.def_type {
+                        ProcMacroDefType::Attr => attr,
+                        ProcMacroDefType::Bang => bang
+                    };
+
+                    cx.expr_call(span, proc_macro_ty_method_path(ident), vec![
+                        cx.expr_str(ca.span, ca.function_name.name),
+                        local_path(ca.span, ca.function_name),
+                    ])
+
+                }
+            }
+        }).collect()
     };
 
     let decls_static = cx.item_static(

src/libsyntax_pos/hygiene.rs

@@ -343,6 +343,38 @@ pub fn update_dollar_crate_names(mut get_name: impl FnMut(SyntaxContext) -> Symb
     }))
 }
 
+pub fn debug_hygiene_data(verbose: bool) -> String {
+    HygieneData::with(|data| {
+        if verbose {
+            format!("{:#?}", data)
+        } else {
+            let mut s = String::from("");
+            s.push_str("Expansions:");
+            data.expn_data.iter().enumerate().for_each(|(id, expn_info)| {
+                let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID");
+                s.push_str(&format!(
+                    "\n{}: parent: {:?}, call_site_ctxt: {:?}, kind: {:?}",
+                    id,
+                    expn_info.parent,
+                    expn_info.call_site.ctxt(),
+                    expn_info.kind,
+                ));
+            });
+            s.push_str("\n\nSyntaxContexts:");
+            data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| {
+                s.push_str(&format!(
+                    "\n#{}: parent: {:?}, outer_mark: ({:?}, {:?})",
+                    id,
+                    ctxt.parent,
+                    ctxt.outer_expn,
+                    ctxt.outer_transparency,
+                ));
+            });
+            s
+        }
+    })
+}
+
 impl SyntaxContext {
     #[inline]
     pub const fn root() -> Self {

src/test/pretty/macro.rs

@@ -2,6 +2,6 @@
 
 #![feature(decl_macro)]
 
-macro mac { ($ arg : expr) => { $ arg + $ arg } }
+pub(crate) macro mac { ($ arg : expr) => { $ arg + $ arg } }
 
 fn main() { }

src/test/rustdoc/inline_cross/proc_macro.rs

@@ -10,8 +10,19 @@ extern crate some_macros;
 // @has proc_macro/macro.some_proc_macro.html
 // @has proc_macro/attr.some_proc_attr.html
 // @has proc_macro/derive.SomeDerive.html
-pub use some_macros::{some_proc_macro, some_proc_attr, SomeDerive};
+
+// @has proc_macro/macro.some_proc_macro.html
+// @has - 'a proc-macro that swallows its input and does nothing.'
+pub use some_macros::some_proc_macro;
 
 // @has proc_macro/macro.reexported_macro.html
 // @has - 'Doc comment from the original crate'
 pub use some_macros::reexported_macro;
+
+// @has proc_macro/attr.some_proc_attr.html
+// @has - 'a proc-macro attribute that passes its item through verbatim.'
+pub use some_macros::some_proc_attr;
+
+// @has proc_macro/derive.SomeDerive.html
+// @has - 'a derive attribute that adds nothing to its input.'
+pub use some_macros::SomeDerive;

src/test/ui/hygiene/unpretty-debug.stdout

@@ -13,3 +13,13 @@ macro_rules! foo /* 0#0 */ { ($ x : ident) => { y + $ x } }
 fn bar /* 0#0 */() { let x /* 0#0 */ = 1; y /* 0#1 */ + x /* 0#0 */ }
 
 fn y /* 0#0 */() { }
+
+/*
+Expansions:
+0: parent: ExpnId(0), call_site_ctxt: #0, kind: Root
+1: parent: ExpnId(0), call_site_ctxt: #0, kind: Macro(Bang, foo)
+
+SyntaxContexts:
+#0: parent: #0, outer_mark: (ExpnId(0), Opaque)
+#1: parent: #0, outer_mark: (ExpnId(1), SemiTransparent)
+*/