doc: doc comments and intra-doc links for core::compiler

src/cargo/core/compiler/compilation.rs

@@ -1,3 +1,5 @@
+//! Type definitions for the result of a compilation.
+
 use std::collections::{BTreeSet, HashMap};
 use std::env;
 use std::ffi::{OsStr, OsString};

src/cargo/core/compiler/compile_kind.rs

@@ -1,3 +1,5 @@
+//! Type definitions for cross-compilation.
+
 use crate::core::Target;
 use crate::util::errors::CargoResult;
 use crate::util::interning::InternedString;

src/cargo/core/compiler/context/mod.rs

@@ -161,11 +161,6 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         }
 
         for unit in &self.bcx.roots {
-            // Build up a list of pending jobs, each of which represent
-            // compiling a particular package. No actual work is executed as
-            // part of this, that's all done next as part of the `execute`
-            // function which will run everything in order with proper
-            // parallelism.
             let force_rebuild = self.bcx.build_config.force_rebuild;
             super::compile(&mut self, &mut queue, &mut plan, unit, exec, force_rebuild)?;
         }

src/cargo/core/compiler/future_incompat.rs

@@ -1,4 +1,37 @@
-//! Support for future-incompatible warning reporting.
+//! Support for [future-incompatible warning reporting][1].
+//!
+//! Here is an overview of how Cargo handles future-incompatible reports.
+//!
+//! ## Receive reports from the compiler
+//!
+//! When receiving a compiler message during a build, if it is effectively
+//! a [`FutureIncompatReport`], Cargo gathers and forwards it as a
+//! `Message::FutureIncompatReport` to the main thread.
+//!
+//! To have the correct layout of strucutures for deserializing a report
+//! emitted by the compiler, most of structure definitions, for example
+//! [`FutureIncompatReport`], are copied either partially or entirely from
+//! [compiler/rustc_errors/src/json.rs][2] in rust-lang/rust repository.
+//!
+//! ## Persist reports on disk
+//!
+//! When a build comes to an end, by calling [`save_and_display_report`]
+//! Cargo saves the report on disk, and displays it directly if requested
+//! via command line or configuration. The information of the on-disk file can
+//! be found in [`FUTURE_INCOMPAT_FILE`].
+//!
+//! During the persistent process, Cargo will attempt to query the source of
+//! each package emitting the report, for the sake of providing an upgrade
+//! information as a solution to fix the incompatibility.
+//!
+//! ## Display reports to users
+//!
+//! Users can run `cargo report future-incompat` to retrieve a report. This is
+//! done by [`OnDiskReports::load`]. Cargo simply prints reports to the
+//! standard output.
+//!
+//! [1]: https://doc.rust-lang.org/nightly/cargo/reference/future-incompat-report.html
+//! [2]: https://github.com/rust-lang/rust/blob/9bb6e60d1f1360234aae90c97964c0fa5524f141/compiler/rustc_errors/src/json.rs#L312-L315
 
 use crate::core::compiler::BuildContext;
 use crate::core::{Dependency, PackageId, QueryKind, Workspace};

src/cargo/core/compiler/links.rs

@@ -4,16 +4,20 @@ use crate::core::{PackageId, Resolve};
 use crate::util::errors::CargoResult;
 use std::collections::{HashMap, HashSet};
 
-/// Validate `links` field does not conflict between packages.
+/// Validates [`package.links`] field in the manifest file does not conflict
+/// between packages.
+///
+/// NOTE: This is the *old* links validator. Links are usually validated in the
+/// resolver. However, the `links` field was added to the index in early 2018
+/// (see [rust-lang/cargo#4978]). However, `links` has been around since 2014,
+/// so there are still many crates in the index that don't have `links`
+/// properly set in the index (over 600 at the time of this writing in 2019).
+/// This can probably be removed at some point in the future, though it might
+/// be worth considering fixing the index.
+///
+/// [rust-lang/cargo#4978]: https://github.com/rust-lang/cargo/pull/4978
+/// [`package.links`]: https://doc.rust-lang.org/nightly/cargo/reference/build-scripts.html#the-links-manifest-key
 pub fn validate_links(resolve: &Resolve, unit_graph: &UnitGraph) -> CargoResult<()> {
-    // NOTE: This is the *old* links validator. Links are usually validated in
-    // the resolver. However, the `links` field was added to the index in
-    // early 2018 (see https://github.com/rust-lang/cargo/pull/4978). However,
-    // `links` has been around since 2014, so there are still many crates in
-    // the index that don't have `links` properly set in the index (over 600
-    // at the time of this writing in 2019). This can probably be removed at
-    // some point in the future, though it might be worth considering fixing
-    // the index.
     let mut validated: HashSet<PackageId> = HashSet::new();
     let mut links: HashMap<String, PackageId> = HashMap::new();
     let mut units: Vec<_> = unit_graph.keys().collect();

src/cargo/core/compiler/mod.rs

@@ -1,3 +1,36 @@
+//! # Interact with the compiler
+//!
+//! If you consider [`ops::cargo_compile::compile`] as a `rustc` driver but on
+//! Cargo side, this module is kinda the `rustc_interface` for that merits.
+//! It contains all the interaction between Cargo and the rustc compiler,
+//! from preparing the context for the entire build process, to scheduling
+//! and executing each unit of work (e.g. running `rustc`), to managing and
+//! caching the output artifact of a build.
+//!
+//! However, it hasn't yet exposed a clear definition of each phase or session,
+//! like what rustc has done[^1]. Also, no one knows if Cargo really needs that.
+//! To be pragmatic, here we list a handful of items you may want to learn:
+//!
+//! * [`BuildContext`] is a static context containg all information you need
+//!   before a build gets started.
+//! * [`Context`] is the center of the world, coordinating a running build and
+//!   collecting information from it.
+//! * [`custom_build`] is the home of build script executions and output parsing.
+//! * [`fingerprint`] not only defines but also executes a set of rules to
+//!   determine if a re-compile is needed.
+//! * [`job_queue`] is where the parallelism, job scheduling, and communication
+//!   machinary happen between Cargo and the compiler.
+//! * [`layout`] defines and manages output artifacts of a build in the filesystem.
+//! * [`unit_dependencies`] is for building a dependency graph for compilation
+//!   from a result of dependency resolution.
+//! * [`Unit`] contains sufficient information to build something, usually
+//!   turning into a compiler invocation in a later phase.
+//!
+//! [^1]: Maybe [`-Zbuild-plan`](https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-plan)
+//!   was designed to serve that purpose but still [in flux](https://github.com/rust-lang/cargo/issues/7614).
+//!
+//! [`ops::cargo_compile::compile`]: crate::ops::compile
+
 pub mod artifact;
 mod build_config;
 mod build_context;
@@ -67,18 +100,31 @@ use rustfix::diagnostics::Applicability;
 
 const RUSTDOC_CRATE_VERSION_FLAG: &str = "--crate-version";
 
+// TODO: Rename this to `ExtraLinkArgFor` or else, and move to compiler/custom_build.rs?
+/// Represents one of the instruction from `cargo:rustc-link-arg-*` build script
+/// instruction family.
+///
+/// In other words, indicates targets that custom linker arguments applies to.
 #[derive(Clone, Hash, Debug, PartialEq, Eq)]
 pub enum LinkType {
+    /// Represents `cargo:rustc-link-arg=FLAG`.
     All,
+    /// Represents `cargo:rustc-cdylib-link-arg=FLAG`.
     Cdylib,
+    /// Represents `cargo:rustc-link-arg-bins=FLAG`.
     Bin,
+    /// Represents `cargo:rustc-link-arg-bin=BIN=FLAG`.
     SingleBin(String),
+    /// Represents `cargo:rustc-link-arg-tests=FLAG`.
     Test,
+    /// Represents `cargo:rustc-link-arg-benches=FLAG`.
     Bench,
+    /// Represents `cargo:rustc-link-arg-examples=FLAG`.
     Example,
 }
 
 impl LinkType {
+    /// Checks if this link type applies to a given [`Target`].
     pub fn applies_to(&self, target: &Target) -> bool {
         match self {
             LinkType::All => true,
@@ -140,6 +186,15 @@ impl Executor for DefaultExecutor {
     }
 }
 
+/// Builds up and enqueue a list of pending jobs onto the `job` queue.
+///
+/// Starting from the `unit`, this function recursively calls itself to build
+/// all jobs for dependencies of the `unit`. Each of these jobs represents
+/// compiling a particular package.
+///
+/// Note that **no actual work is executed as part of this**, that's all done
+/// next as part of [`JobQueue::execute`] function which will run everything
+/// in order with proper parallelism.
 fn compile<'cfg>(
     cx: &mut Context<'_, 'cfg>,
     jobs: &mut JobQueue<'cfg>,
@@ -230,6 +285,7 @@ fn make_failed_scrape_diagnostic(
     )
 }
 
+/// Creates a unit of work invoking `rustc` for building the `unit`.
 fn rustc(cx: &mut Context<'_, '_>, unit: &Unit, exec: &Arc<dyn Executor>) -> CargoResult<Work> {
     let mut rustc = prepare_rustc(cx, &unit.target.rustc_crate_types(), unit)?;
     let build_plan = cx.bcx.build_config.build_plan;
@@ -638,6 +694,8 @@ where
     search_path
 }
 
+// TODO: do we really need this as a separate function?
+// Maybe we should reorganize `rustc` fn to make it more traceable and readable.
 fn prepare_rustc(
     cx: &mut Context<'_, '_>,
     crate_types: &[CrateType],
@@ -672,6 +730,7 @@ fn prepare_rustc(
     Ok(base)
 }
 
+/// Creates a unit of work invoking `rustdoc` for documenting the `unit`.
 fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
     let bcx = cx.bcx;
     // script_metadata is not needed here, it is only for tests.
@@ -857,6 +916,9 @@ fn append_crate_version_flag(unit: &Unit, rustdoc: &mut ProcessBuilder) {
         .arg(unit.pkg.version().to_string());
 }
 
+/// Adds [`--cap-lints`] to the command to execute.
+///
+/// [`--cap-lints`]: https://doc.rust-lang.org/nightly/rustc/lints/levels.html#capping-lints
 fn add_cap_lints(bcx: &BuildContext<'_, '_>, unit: &Unit, cmd: &mut ProcessBuilder) {
     // If this is an upstream dep we don't want warnings from, turn off all
     // lints.
@@ -870,7 +932,9 @@ fn add_cap_lints(bcx: &BuildContext<'_, '_>, unit: &Unit, cmd: &mut ProcessBuild
     }
 }
 
-/// Forward -Zallow-features if it is set for cargo.
+/// Forwards [`-Zallow-features`] if it is set for cargo.
+///
+/// [`-Zallow-features`]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#allow-features
 fn add_allow_features(cx: &Context<'_, '_>, cmd: &mut ProcessBuilder) {
     if let Some(allow) = &cx.bcx.config.cli_unstable().allow_features {
         let mut arg = String::from("-Zallow-features=");
@@ -879,14 +943,16 @@ fn add_allow_features(cx: &Context<'_, '_>, cmd: &mut ProcessBuilder) {
     }
 }
 
-/// Add error-format flags to the command.
+/// Adds [`--error-format`] to the command to execute.
 ///
 /// Cargo always uses JSON output. This has several benefits, such as being
 /// easier to parse, handles changing formats (for replaying cached messages),
 /// ensures atomic output (so messages aren't interleaved), allows for
 /// intercepting messages like rmeta artifacts, etc. rustc includes a
 /// "rendered" field in the JSON message with the message properly formatted,
 /// which Cargo will extract and display to the user.
+///
+/// [`--error-format`]: https://doc.rust-lang.org/nightly/rustc/command-line-arguments.html#--error-format-control-how-errors-are-produced
 fn add_error_format_and_color(cx: &Context<'_, '_>, cmd: &mut ProcessBuilder) {
     cmd.arg("--error-format=json");
     let mut json = String::from("--json=diagnostic-rendered-ansi,artifacts,future-incompat");
@@ -905,6 +971,7 @@ fn add_error_format_and_color(cx: &Context<'_, '_>, cmd: &mut ProcessBuilder) {
     }
 }
 
+/// Adds essential rustc flags and environment variables to the command to execute.
 fn build_base_args(
     cx: &mut Context<'_, '_>,
     cmd: &mut ProcessBuilder,
@@ -1124,7 +1191,7 @@ fn build_base_args(
     Ok(())
 }
 
-/// All active features for the unit passed as --cfg
+/// All active features for the unit passed as `--cfg features=<feature-name>`.
 fn features_args(unit: &Unit) -> Vec<OsString> {
     let mut args = Vec::with_capacity(unit.features.len() * 2);
 
@@ -1136,7 +1203,10 @@ fn features_args(unit: &Unit) -> Vec<OsString> {
     args
 }
 
-/// Generate the --check-cfg arguments for the unit
+/// Generates the `--check-cfg` arguments for the `unit`.
+/// See unstable feature [`check-cfg`].
+///
+/// [`check-cfg`]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#check-cfg
 fn check_cfg_args(cx: &Context<'_, '_>, unit: &Unit) -> Vec<OsString> {
     if let Some((features, well_known_names, well_known_values, _output)) =
         cx.bcx.config.cli_unstable().check_cfg
@@ -1176,6 +1246,7 @@ fn check_cfg_args(cx: &Context<'_, '_>, unit: &Unit) -> Vec<OsString> {
     }
 }
 
+/// Adds LTO related codegen flags.
 fn lto_args(cx: &Context<'_, '_>, unit: &Unit) -> Vec<OsString> {
     let mut result = Vec::new();
     let mut push = |arg: &str| {
@@ -1196,6 +1267,11 @@ fn lto_args(cx: &Context<'_, '_>, unit: &Unit) -> Vec<OsString> {
     result
 }
 
+/// Adds dependency-relevant rustc flags and environment variables
+/// to the command to execute, such as [`-L`] and [`--extern`].
+///
+/// [`-L`]: https://doc.rust-lang.org/nightly/rustc/command-line-arguments.html#-l-add-a-directory-to-the-library-search-path
+/// [`--extern`]: https://doc.rust-lang.org/nightly/rustc/command-line-arguments.html#--extern-specify-where-an-external-library-is-located
 fn build_deps_args(
     cmd: &mut ProcessBuilder,
     cx: &mut Context<'_, '_>,
@@ -1267,7 +1343,9 @@ fn build_deps_args(
     Ok(())
 }
 
-/// Add custom flags from the output a of build-script to a `ProcessBuilder`
+/// Adds extra rustc flags and environment variables collected from the output
+/// of a build-script to the command to execute, include custom environment
+/// variables and `cfg`.
 fn add_custom_flags(
     cmd: &mut ProcessBuilder,
     build_script_outputs: &BuildScriptOutputs,
@@ -1377,6 +1455,8 @@ fn envify(s: &str) -> String {
         .collect()
 }
 
+/// Configuration of the display of messages emitted by the compiler,
+/// e.g. diagnostics, warnings, errors, and message caching.
 struct OutputOptions {
     /// What format we're emitting from Cargo itself.
     format: MessageFormat,
@@ -1395,7 +1475,9 @@ struct OutputOptions {
     /// cache will be filled with diagnostics from dependencies. When the
     /// cache is replayed without `-vv`, we don't want to show them.
     show_diagnostics: bool,
+    /// Tracks the number of warnings we've seen so far.
     warnings_seen: usize,
+    /// Tracks the number of errors we've seen so far.
     errors_seen: usize,
 }
 
@@ -1677,6 +1759,9 @@ fn on_stderr_line_inner(
     Ok(true)
 }
 
+/// Creates a unit of work that replays the cached compiler message.
+///
+/// Usually used when a job is fresh and doesn't need to recompile.
 fn replay_output_cache(
     package_id: PackageId,
     manifest_path: PathBuf,

src/cargo/core/compiler/timings.rs

@@ -16,6 +16,13 @@ use std::io::{BufWriter, Write};
 use std::thread::available_parallelism;
 use std::time::{Duration, Instant, SystemTime};
 
+/// Tracking information for the entire build.
+///
+/// Methods on this structure are generally called from the main thread of a
+/// running [`JobQueue`] instance (`DrainState` in specific) when the queue
+/// receives messages from spawned off threads.
+///
+/// [`JobQueue`]: super::JobQueue
 pub struct Timings<'cfg> {
     config: &'cfg Config,
     /// Whether or not timings should be captured.
@@ -253,12 +260,12 @@ impl<'cfg> Timings<'cfg> {
         self.concurrency.push(c);
     }
 
-    /// Mark that a fresh unit was encountered.
+    /// Mark that a fresh unit was encountered. (No re-compile needed)
     pub fn add_fresh(&mut self) {
         self.total_fresh += 1;
     }
 
-    /// Mark that a dirty unit was encountered.
+    /// Mark that a dirty unit was encountered. (Re-compile needed)
     pub fn add_dirty(&mut self) {
         self.total_dirty += 1;
     }
@@ -456,6 +463,8 @@ impl<'cfg> Timings<'cfg> {
         Ok(())
     }
 
+    /// Write timing data in JavaScript. Primarily for `timings.js` to put data
+    /// in a `<script>` HTML element to draw graphs.
     fn write_js_data(&self, f: &mut impl Write) -> CargoResult<()> {
         // Create a map to link indices of unlocked units.
         let unit_map: HashMap<Unit, usize> = self

src/cargo/core/compiler/unit.rs

@@ -1,3 +1,5 @@
+//! Types and impls for [`Unit`].
+
 use crate::core::compiler::unit_dependencies::IsArtifact;
 use crate::core::compiler::{CompileKind, CompileMode, CompileTarget, CrateType};
 use crate::core::manifest::{Target, TargetKind};
@@ -108,6 +110,9 @@ impl UnitInner {
 }
 
 impl Unit {
+    /// Gets the unique key for [`-Zbuild-plan`].
+    ///
+    /// [`-Zbuild-plan`]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-plan
     pub fn buildkey(&self) -> String {
         format!("{}-{}", self.pkg.name(), short_hash(self))
     }