diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 29d7bcc425bea..65d635c0bd69f 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -212,7 +212,7 @@ impl Step for GenerateCopyright { let license_metadata = builder.ensure(CollectLicenseMetadata); // Temporary location, it will be moved to the proper one once it's accurate. - let dest = builder.out.join("COPYRIGHT.md"); + let dest = builder.out.join("COPYRIGHT.html"); let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); cmd.env("LICENSE_METADATA", &license_metadata); diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs index 721a6b1c6e627..eda53c73c0af6 100644 --- a/src/tools/generate-copyright/src/cargo_metadata.rs +++ b/src/tools/generate-copyright/src/cargo_metadata.rs @@ -1,6 +1,6 @@ //! Gets metadata about a workspace from Cargo -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::path::Path; @@ -23,13 +23,18 @@ pub enum Error { RunningVendor, } -/// Describes one of our dependencies +/// Uniquely describes a package on crates.io #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Dependency { +pub struct Package { /// The name of the package pub name: String, /// The version number pub version: String, +} + +/// Extra data about a package +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct PackageMetadata { /// The license it is under pub license: String, /// The list of authors from the package metadata @@ -40,20 +45,44 @@ pub struct Dependency { pub notices: BTreeMap, } -/// Use `cargo` to get a list of dependencies and their license data. +/// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data. /// /// This will involve running `cargo vendor` into `${BUILD}/vendor` so we can /// grab the license files. /// /// Any dependency with a path beginning with `root_path` is ignored, as we /// assume `reuse` has covered it already. -pub fn get( +pub fn get_metadata_and_notices( cargo: &Path, dest: &Path, root_path: &Path, manifest_paths: &[&Path], -) -> Result, Error> { - let mut temp_set = BTreeSet::new(); +) -> Result, Error> { + let mut output = get_metadata(cargo, root_path, manifest_paths)?; + + // Now do a cargo-vendor and grab everything + let vendor_path = dest.join("vendor"); + println!("Vendoring deps into {}...", vendor_path.display()); + run_cargo_vendor(cargo, &vendor_path, manifest_paths)?; + + // Now for each dependency we found, go and grab any important looking files + for (package, metadata) in output.iter_mut() { + load_important_files(package, metadata, &vendor_path)?; + } + + Ok(output) +} + +/// Use `cargo metadata` to get a list of dependencies and their license data. +/// +/// Any dependency with a path beginning with `root_path` is ignored, as we +/// assume `reuse` has covered it already. +pub fn get_metadata( + cargo: &Path, + root_path: &Path, + manifest_paths: &[&Path], +) -> Result, Error> { + let mut output = BTreeMap::new(); // Look at the metadata for each manifest for manifest_path in manifest_paths { if manifest_path.file_name() != Some(OsStr::new("Cargo.toml")) { @@ -71,7 +100,7 @@ pub fn get( .and_then(|v| v.as_str()) .map(Path::new) .ok_or_else(|| Error::MissingJsonElement("package.manifest_path"))?; - if manifest_path.starts_with(&root_path) { + if manifest_path.starts_with(root_path) { // it's an in-tree dependency and reuse covers it continue; } @@ -93,28 +122,14 @@ pub fn get( .ok_or_else(|| Error::MissingJsonElement("package.authors"))?; let authors: Vec = authors_list.iter().filter_map(|v| v.as_str()).map(|s| s.to_owned()).collect(); - temp_set.insert(Dependency { - name: name.to_owned(), - version: version.to_owned(), - license: license.to_owned(), - authors, - notices: BTreeMap::new(), - }); + let package = Package { name: name.to_owned(), version: version.to_owned() }; + output.insert( + package.clone(), + PackageMetadata { license: license.to_owned(), authors, notices: BTreeMap::new() }, + ); } } - // Now do a cargo-vendor and grab everything - let vendor_path = dest.join("vendor"); - println!("Vendoring deps into {}...", vendor_path.display()); - run_cargo_vendor(cargo, &vendor_path, manifest_paths)?; - - // Now for each dependency we found, go and grab any important looking files - let mut output = BTreeSet::new(); - for mut dep in temp_set { - load_important_files(&mut dep, &vendor_path)?; - output.insert(dep); - } - Ok(output) } @@ -128,7 +143,7 @@ fn get_metadata_json(cargo: &Path, manifest_path: &Path) -> Result Resu } vendor_command.arg(dest); - let vendor_status = vendor_command.status().map_err(|e| Error::LaunchingVendor(e))?; + let vendor_status = vendor_command.status().map_err(Error::LaunchingVendor)?; if !vendor_status.success() { return Err(Error::RunningVendor); @@ -164,8 +179,12 @@ fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[&Path]) -> Resu /// /// Maybe one-day Cargo.toml will contain enough information that we don't need /// to do this manual scraping. -fn load_important_files(dep: &mut Dependency, vendor_root: &Path) -> Result<(), Error> { - let name_version = format!("{}-{}", dep.name, dep.version); +fn load_important_files( + package: &Package, + dep: &mut PackageMetadata, + vendor_root: &Path, +) -> Result<(), Error> { + let name_version = format!("{}-{}", package.name, package.version); println!("Scraping notices for {}...", name_version); let dep_vendor_path = vendor_root.join(name_version); for entry in std::fs::read_dir(dep_vendor_path)? { diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index 6191cd158bc9b..efccba0651e4d 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::io::Write; use std::path::{Path, PathBuf}; @@ -5,6 +6,33 @@ use anyhow::Error; mod cargo_metadata; +static TOP_BOILERPLATE: &str = r##" + + + + + Copyright notices for The Rust Toolchain + + + +

Copyright notices for The Rust Toolchain

+ +

This file describes the copyright and licensing information for the source +code within The Rust Project git tree, and the third-party dependencies used +when building the Rust toolchain (including the Rust Standard Library).

+ +

Table of Contents

+ +"##; + +static BOTTOM_BOILERPLATE: &str = r#" + + +"#; + /// The entry point to the binary. /// /// You should probably let `bootstrap` execute this program instead of running it directly. @@ -26,43 +54,28 @@ fn main() -> Result<(), Error> { Path::new("./library/std/Cargo.toml"), ]; let collected_cargo_metadata = - cargo_metadata::get(&cargo, &out_dir, &root_path, &workspace_paths)?; + cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?; + + let stdlib_set = + cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?; let mut buffer = Vec::new(); - writeln!(buffer, "# COPYRIGHT for Rust")?; - writeln!(buffer)?; - writeln!( - buffer, - "This file describes the copyright and licensing information for the source code within The Rust Project git tree, and the third-party dependencies used when building the Rust toolchain (including the Rust Standard Library)" - )?; - writeln!(buffer)?; - writeln!(buffer, "## Table of Contents")?; - writeln!(buffer)?; - writeln!(buffer, "* [In-tree files](#in-tree-files)")?; - writeln!(buffer, "* [Out-of-tree files](#out-of-tree-files)")?; - // writeln!(buffer, "* [License Texts](#license-texts)")?; - writeln!(buffer)?; - - writeln!(buffer, "## In-tree files")?; - writeln!(buffer)?; + writeln!(buffer, "{}", TOP_BOILERPLATE)?; + writeln!( buffer, - "The following licenses cover the in-tree source files that were used in this release:" + r#"

In-tree files

The following licenses cover the in-tree source files that were used in this release:

"# )?; - writeln!(buffer)?; - render_tree_recursive(&collected_tree_metadata.files, &mut buffer, 0)?; - - writeln!(buffer)?; + render_tree_recursive(&collected_tree_metadata.files, &mut buffer)?; - writeln!(buffer, "## Out-of-tree files")?; - writeln!(buffer)?; writeln!( buffer, - "The following licenses cover the out-of-tree crates that were used in this release:" + r#"

Out-of-tree dependencies

The following licenses cover the out-of-tree crates that were used in this release:

"# )?; - writeln!(buffer)?; - render_deps(collected_cargo_metadata.iter(), &mut buffer)?; + render_deps(&collected_cargo_metadata, &stdlib_set, &mut buffer)?; + + writeln!(buffer, "{}", BOTTOM_BOILERPLATE)?; std::fs::write(&dest_file, &buffer)?; @@ -71,56 +84,51 @@ fn main() -> Result<(), Error> { /// Recursively draw the tree of files/folders we found on disk and their licenses, as /// markdown, into the given Vec. -fn render_tree_recursive(node: &Node, buffer: &mut Vec, depth: usize) -> Result<(), Error> { - let prefix = std::iter::repeat("> ").take(depth + 1).collect::(); - +fn render_tree_recursive(node: &Node, buffer: &mut Vec) -> Result<(), Error> { + writeln!(buffer, r#"
"#)?; match node { Node::Root { children } => { for child in children { - render_tree_recursive(child, buffer, depth)?; + render_tree_recursive(child, buffer)?; } } Node::Directory { name, children, license } => { - render_tree_license(&prefix, std::iter::once(name), license.as_ref(), buffer)?; + render_tree_license(std::iter::once(name), license.as_ref(), buffer)?; if !children.is_empty() { - writeln!(buffer, "{prefix}")?; - writeln!(buffer, "{prefix}*Exceptions:*")?; + writeln!(buffer, "

Exceptions:

")?; for child in children { - writeln!(buffer, "{prefix}")?; - render_tree_recursive(child, buffer, depth + 1)?; + render_tree_recursive(child, buffer)?; } } } Node::Group { files, directories, license } => { - render_tree_license( - &prefix, - directories.iter().chain(files.iter()), - Some(license), - buffer, - )?; + render_tree_license(directories.iter().chain(files.iter()), Some(license), buffer)?; } Node::File { name, license } => { - render_tree_license(&prefix, std::iter::once(name), Some(license), buffer)?; + render_tree_license(std::iter::once(name), Some(license), buffer)?; } } + writeln!(buffer, "
")?; Ok(()) } /// Draw a series of sibling files/folders, as markdown, into the given Vec. fn render_tree_license<'a>( - prefix: &str, names: impl Iterator, license: Option<&License>, buffer: &mut Vec, ) -> Result<(), Error> { + writeln!(buffer, "

File/Directory: ")?; for name in names { - writeln!(buffer, "{prefix}**`{name}`** ")?; + writeln!(buffer, "{name}")?; } + writeln!(buffer, "

")?; + if let Some(license) = license { - writeln!(buffer, "{prefix}License: `{}`", license.spdx)?; + writeln!(buffer, "

License: {}

", license.spdx)?; for copyright in license.copyright.iter() { - writeln!(buffer, "{prefix}Copyright: {copyright}")?; + writeln!(buffer, "

Copyright: {copyright}

")?; } } @@ -128,36 +136,48 @@ fn render_tree_license<'a>( } /// Render a list of out-of-tree dependencies as markdown into the given Vec. -fn render_deps<'a, 'b>( - deps: impl Iterator, - buffer: &'b mut Vec, +fn render_deps( + all_deps: &BTreeMap, + stdlib_set: &BTreeMap, + buffer: &mut Vec, ) -> Result<(), Error> { - for dep in deps { - let authors_list = dep.authors.join(", ").replace("<", "\\<").replace(">", "\\>"); - let url = format!("https://crates.io/crates/{}/{}", dep.name, dep.version); + for (package, metadata) in all_deps { + let authors_list = if metadata.authors.is_empty() { + "None Specified".to_owned() + } else { + metadata.authors.join(", ") + }; + let url = format!("https://crates.io/crates/{}/{}", package.name, package.version); writeln!(buffer)?; writeln!( buffer, - "### [{name} {version}]({url})", - name = dep.name, - version = dep.version, - url = url, + r#"

📦 {name}-{version}

"#, + name = package.name, + version = package.version, )?; - writeln!(buffer)?; - writeln!(buffer, "* Authors: {}", authors_list)?; - writeln!(buffer, "* License: {}", dep.license)?; - for (name, contents) in &dep.notices { - writeln!(buffer)?; - writeln!(buffer, "#### {}", name.to_string_lossy())?; - writeln!(buffer)?; - writeln!(buffer, "
Click to expand")?; - writeln!(buffer)?; - writeln!(buffer, "```")?; - writeln!(buffer, "{}", contents)?; - writeln!(buffer, "```")?; - writeln!(buffer)?; - writeln!(buffer, "
")?; + writeln!(buffer, r#"

URL: {url}

"#,)?; + writeln!( + buffer, + "

In libstd: {}

", + if stdlib_set.contains_key(package) { "Yes" } else { "No" } + )?; + writeln!(buffer, "

Authors: {}

", escape_html(&authors_list))?; + writeln!(buffer, "

License: {}

", escape_html(&metadata.license))?; + writeln!(buffer, "

Notices: ")?; + if metadata.notices.is_empty() { + writeln!(buffer, "None")?; + } else { + for (name, contents) in &metadata.notices { + writeln!( + buffer, + "

{}", + name.to_string_lossy() + )?; + writeln!(buffer, "
\n{}\n
", contents)?; + writeln!(buffer, "
")?; + } } + writeln!(buffer, "

")?; } Ok(()) } @@ -192,3 +212,13 @@ fn env_path(var: &str) -> Result { anyhow::bail!("missing environment variable {var}") } } + +/// Escapes any invalid HTML characters +fn escape_html(input: &str) -> String { + static MAPPING: [(char, &str); 3] = [('&', "&"), ('<', "<"), ('>', ">")]; + let mut output = input.to_owned(); + for (ch, s) in &MAPPING { + output = output.replace(*ch, s); + } + output +}