From 985be4f4a9f8ba442cb44ea60b1e22688fba7c31 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 15 Feb 2017 15:57:06 -0800 Subject: [PATCH] rustbuild: Add support for compiling Cargo This commit adds support to rustbuild for compiling Cargo as part of the release process. Previously rustbuild would simply download a Cargo snapshot and repackage it. With this change we should be able to turn off artifacts from the rust-lang/cargo repository and purely rely on the artifacts Cargo produces here. The infrastructure added here is intended to be extensible to other components, such as the RLS. It won't exactly be a one-line addition, but the addition of Cargo didn't require too much hooplah anyway. The process for release Cargo will now look like: * The rust-lang/rust repository has a Cargo submodule which is used to build a Cargo to pair with the rust-lang/rust release * Periodically we'll update the cargo submodule as necessary on rust-lang/rust's master branch * When branching beta we'll create a new branch of Cargo (as we do today), and the first commit to the beta branch will be to update the Cargo submodule to this exact revision. * When branching stable, we'll ensure that the Cargo submodule is updated and then make a stable release. Backports to Cargo will look like: * Send a PR to cargo's master branch * Send a PR to cargo's release branch (e.g. rust-1.16.0) * Send a PR to rust-lang/rust's beta branch updating the submodule * Eventually send a PR to rust-lang/rust's master branch updating the submodule For reference, the process to add a new component to the rust-lang/rust release would look like: * Add `$foo` as a submodule in `src/tools` * Add a `tool-$foo` step which compiles `$foo` with the specified compiler, likely mirroring what Cargo does. * Add a `dist-$foo` step which uses `src/tools/$foo` and the `tool-$foo` output to create a rust-installer package for `$foo` likely mirroring what Cargo does. * Update the `dist-extended` step with a new dependency on `dist-$foo` * Update `src/tools/build-manifest` for the new component. --- configure | 1 + src/bootstrap/channel.rs | 95 ++++++++++---------- src/bootstrap/compile.rs | 34 ++++++-- src/bootstrap/config.rs | 4 + src/bootstrap/config.toml.example | 5 ++ src/bootstrap/dist.rs | 125 +++++++++++++++------------ src/bootstrap/doc.rs | 9 +- src/bootstrap/install.rs | 4 +- src/bootstrap/lib.rs | 114 +++++++++++++++++++----- src/bootstrap/metadata.rs | 2 + src/bootstrap/native.rs | 101 ++++++++++++++++++++++ src/bootstrap/step.rs | 11 ++- src/build_helper/lib.rs | 18 ++++ src/ci/run.sh | 1 + src/tools/build-manifest/src/main.rs | 19 ++-- src/tools/tidy/src/cargo.rs | 2 +- src/tools/tidy/src/deps.rs | 2 + src/tools/tidy/src/main.rs | 1 + 18 files changed, 400 insertions(+), 148 deletions(-) diff --git a/configure b/configure index 70952438a3559..be8628de62832 100755 --- a/configure +++ b/configure @@ -651,6 +651,7 @@ opt locked-deps 0 "force Cargo.lock to be up to date" opt vendor 0 "enable usage of vendored Rust crates" opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)" opt dist-src 1 "when building tarballs enables building a source tarball" +opt cargo-openssl-static 0 "static openssl in cargo" # Optimization and debugging options. These may be overridden by the release channel, etc. opt_nosave optimize 1 "build optimized rust code" diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index 81e745bc76c9e..427306fdc4f0d 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -15,6 +15,7 @@ //! `package_vers`, and otherwise indicating to the compiler what it should //! print out as part of its version information. +use std::path::Path; use std::process::Command; use build_helper::output; @@ -22,65 +23,69 @@ use build_helper::output; use Build; // The version number -const CFG_RELEASE_NUM: &'static str = "1.17.0"; +pub const CFG_RELEASE_NUM: &'static str = "1.17.0"; // An optional number to put after the label, e.g. '.2' -> '-beta.2' // Be sure to make this starts with a dot to conform to semver pre-release // versions (section 9) -const CFG_PRERELEASE_VERSION: &'static str = ".1"; +pub const CFG_PRERELEASE_VERSION: &'static str = ".1"; -pub fn collect(build: &mut Build) { - build.release_num = CFG_RELEASE_NUM.to_string(); - build.prerelease_version = CFG_RELEASE_NUM.to_string(); +pub struct GitInfo { + inner: Option, +} - // Depending on the channel, passed in `./configure --release-channel`, - // determine various properties of the build. - match &build.config.channel[..] { - "stable" => { - build.release = CFG_RELEASE_NUM.to_string(); - build.package_vers = build.release.clone(); - build.unstable_features = false; - } - "beta" => { - build.release = format!("{}-beta{}", CFG_RELEASE_NUM, - CFG_PRERELEASE_VERSION); - build.package_vers = "beta".to_string(); - build.unstable_features = false; - } - "nightly" => { - build.release = format!("{}-nightly", CFG_RELEASE_NUM); - build.package_vers = "nightly".to_string(); - build.unstable_features = true; - } - _ => { - build.release = format!("{}-dev", CFG_RELEASE_NUM); - build.package_vers = build.release.clone(); - build.unstable_features = true; - } - } - build.version = build.release.clone(); +struct Info { + commit_date: String, + sha: String, + short_sha: String, +} - // If we have a git directory, add in some various SHA information of what - // commit this compiler was compiled from. - if build.src.join(".git").is_dir() { - let ver_date = output(Command::new("git").current_dir(&build.src) +impl GitInfo { + pub fn new(dir: &Path) -> GitInfo { + if !dir.join(".git").exists() { + return GitInfo { inner: None } + } + let ver_date = output(Command::new("git").current_dir(dir) .arg("log").arg("-1") .arg("--date=short") .arg("--pretty=format:%cd")); - let ver_hash = output(Command::new("git").current_dir(&build.src) + let ver_hash = output(Command::new("git").current_dir(dir) .arg("rev-parse").arg("HEAD")); let short_ver_hash = output(Command::new("git") - .current_dir(&build.src) + .current_dir(dir) .arg("rev-parse") .arg("--short=9") .arg("HEAD")); - let ver_date = ver_date.trim().to_string(); - let ver_hash = ver_hash.trim().to_string(); - let short_ver_hash = short_ver_hash.trim().to_string(); - build.version.push_str(&format!(" ({} {})", short_ver_hash, - ver_date)); - build.ver_date = Some(ver_date.to_string()); - build.ver_hash = Some(ver_hash); - build.short_ver_hash = Some(short_ver_hash); + GitInfo { + inner: Some(Info { + commit_date: ver_date.trim().to_string(), + sha: ver_hash.trim().to_string(), + short_sha: short_ver_hash.trim().to_string(), + }), + } + } + + pub fn sha(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.sha[..]) + } + + pub fn sha_short(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.short_sha[..]) + } + + pub fn commit_date(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.commit_date[..]) + } + + pub fn version(&self, build: &Build, num: &str) -> String { + let mut version = build.release(num); + if let Some(ref inner) = self.inner { + version.push_str(" ("); + version.push_str(&inner.short_sha); + version.push_str(" "); + version.push_str(&inner.commit_date); + version.push_str(")"); + } + return version } } diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index cea8b13366657..46d8d4b4aab2d 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -24,6 +24,7 @@ use std::process::Command; use build_helper::{output, mtime, up_to_date}; use filetime::FileTime; +use channel::GitInfo; use util::{exe, libdir, is_dylib, copy}; use {Build, Compiler, Mode}; @@ -210,9 +211,9 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) { // Set some configuration variables picked up by build scripts and // the compiler alike - cargo.env("CFG_RELEASE", &build.release) + cargo.env("CFG_RELEASE", build.rust_release()) .env("CFG_RELEASE_CHANNEL", &build.config.channel) - .env("CFG_VERSION", &build.version) + .env("CFG_VERSION", build.rust_version()) .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(PathBuf::new())); if compiler.stage == 0 { @@ -229,13 +230,13 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) { cargo.env_remove("RUSTC_DEBUGINFO_LINES"); } - if let Some(ref ver_date) = build.ver_date { + if let Some(ref ver_date) = build.rust_info.commit_date() { cargo.env("CFG_VER_DATE", ver_date); } - if let Some(ref ver_hash) = build.ver_hash { + if let Some(ref ver_hash) = build.rust_info.sha() { cargo.env("CFG_VER_HASH", ver_hash); } - if !build.unstable_features { + if !build.unstable_features() { cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); } // Flag that rust llvm is in use @@ -416,13 +417,32 @@ pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) { // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target)); let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build"); - cargo.arg("--manifest-path") - .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool))); + let dir = build.src.join("src/tools").join(tool); + cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); // We don't want to build tools dynamically as they'll be running across // stages and such and it's just easier if they're not dynamically linked. cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); + if let Some(dir) = build.openssl_install_dir(target) { + cargo.env("OPENSSL_STATIC", "1"); + cargo.env("OPENSSL_DIR", dir); + cargo.env("LIBZ_SYS_STATIC", "1"); + } + + cargo.env("CFG_RELEASE_CHANNEL", &build.config.channel); + + let info = GitInfo::new(&dir); + if let Some(sha) = info.sha() { + cargo.env("CFG_COMMIT_HASH", sha); + } + if let Some(sha_short) = info.sha_short() { + cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); + } + if let Some(date) = info.commit_date() { + cargo.env("CFG_COMMIT_DATE", date); + } + build.run(&mut cargo); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 8308dc3202fff..438ce6103d624 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -106,6 +106,7 @@ pub struct Config { pub gdb: Option, pub python: Option, pub configure_args: Vec, + pub openssl_static: bool, } /// Per-target configuration stored in the global configuration structure. @@ -155,6 +156,7 @@ struct Build { extended: Option, verbose: Option, sanitizers: Option, + openssl_static: Option, } /// TOML representation of various global install decisions. @@ -305,6 +307,7 @@ impl Config { set(&mut config.extended, build.extended); set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); + set(&mut config.openssl_static, build.openssl_static); if let Some(ref install) = toml.install { config.prefix = install.prefix.clone().map(PathBuf::from); @@ -453,6 +456,7 @@ impl Config { ("EXTENDED", self.extended), ("SANITIZERS", self.sanitizers), ("DIST_SRC", self.rust_dist_src), + ("CARGO_OPENSSL_STATIC", self.openssl_static), } match key { diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index f95e890f346ee..30763e38a336f 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -134,6 +134,11 @@ # Build the sanitizer runtimes #sanitizers = false +# Indicates whether the OpenSSL linked into Cargo will be statically linked or +# not. If static linkage is specified then the build system will download a +# known-good version of OpenSSL, compile it, and link it to Cargo. +#openssl-static = false + # ============================================================================= # General install configuration options # ============================================================================= diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c468d4896a614..740d6625119ac 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -33,19 +33,12 @@ const SH_CMD: &'static str = "sh"; const SH_CMD: &'static str = "bash"; use {Build, Compiler, Mode}; +use channel; use util::{cp_r, libdir, is_dylib, cp_filtered, copy}; -pub fn package_vers(build: &Build) -> &str { - match &build.config.channel[..] { - "stable" => &build.release, - "beta" => "beta", - "nightly" => "nightly", - _ => &build.release, - } -} - fn pkgname(build: &Build, component: &str) -> String { - format!("{}-{}", component, package_vers(build)) + assert!(component.starts_with("rust")); // does not work with cargo + format!("{}-{}", component, build.rust_package_vers()) } fn distdir(build: &Build) -> PathBuf { @@ -93,7 +86,7 @@ pub fn docs(build: &Build, stage: u32, host: &str) { // As part of this step, *also* copy the docs directory to a directory which // buildbot typically uploads. if host == build.config.build { - let dst = distdir(build).join("doc").join(&build.package_vers); + let dst = distdir(build).join("doc").join(build.rust_package_vers()); t!(fs::create_dir_all(&dst)); cp_r(&src, &dst); } @@ -162,7 +155,7 @@ pub fn rustc(build: &Build, stage: u32, host: &str) { cp("LICENSE-MIT"); cp("README.md"); // tiny morsel of metadata is used by rust-packaging - let version = &build.version; + let version = build.rust_version(); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); // On MinGW we've got a few runtime DLL dependencies that we need to @@ -312,7 +305,7 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) { } pub fn rust_src_location(build: &Build) -> PathBuf { - let plain_name = format!("rustc-{}-src", package_vers(build)); + let plain_name = format!("rustc-{}-src", build.rust_package_vers()); distdir(build).join(&format!("{}.tar.gz", plain_name)) } @@ -477,14 +470,14 @@ pub fn rust_src(build: &Build) { build.run(&mut cmd); // Rename directory, so that root folder of tarball has the correct name - let plain_name = format!("rustc-{}-src", package_vers(build)); + let plain_name = format!("rustc-{}-src", build.rust_package_vers()); let plain_dst_src = tmpdir(build).join(&plain_name); let _ = fs::remove_dir_all(&plain_dst_src); t!(fs::create_dir_all(&plain_dst_src)); cp_r(&dst_src, &plain_dst_src); // Create the version file - write_file(&plain_dst_src.join("version"), build.version.as_bytes()); + write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); // Create plain source tarball let mut cmd = Command::new("tar"); @@ -536,43 +529,62 @@ fn write_file(path: &Path, data: &[u8]) { t!(vf.write_all(data)); } -// FIXME(#38531) eventually this should package up a Cargo that we just compiled -// and tested locally, but for now we're downloading cargo -// artifacts from their compiled location. pub fn cargo(build: &Build, stage: u32, target: &str) { println!("Dist cargo stage{} ({})", stage, target); + let compiler = Compiler::new(stage, &build.config.build); - let branch = match &build.config.channel[..] { - "stable" | - "beta" => format!("rust-{}", build.release_num), - _ => "master".to_string(), - }; + let src = build.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + let release_num = &build.crates["cargo"].version; + let name = format!("cargo-{}", build.package_vers(release_num)); + let version = build.cargo_info.version(build, release_num); - let dst = tmpdir(build).join("cargo"); - let _ = fs::remove_dir_all(&dst); - build.run(Command::new("git") - .arg("clone") - .arg("--depth").arg("1") - .arg("--branch").arg(&branch) - .arg("https://github.com/rust-lang/cargo") - .current_dir(dst.parent().unwrap())); - let sha = output(Command::new("git") - .arg("rev-parse") - .arg("HEAD") - .current_dir(&dst)); - let sha = sha.trim(); - println!("\tgot cargo sha: {}", sha); - - let input = format!("https://s3.amazonaws.com/rust-lang-ci/cargo-builds\ - /{}/cargo-nightly-{}.tar.gz", sha, target); - let output = distdir(build).join(format!("cargo-nightly-{}.tar.gz", target)); - println!("\tdownloading {}", input); - let mut curl = Command::new("curl"); - curl.arg("-f") - .arg("-o").arg(&output) - .arg(&input) - .arg("--retry").arg("3"); - build.run(&mut curl); + let tmp = tmpdir(build); + let image = tmp.join("cargo-image"); + drop(fs::remove_dir_all(&image)); + t!(fs::create_dir_all(&image)); + + // Prepare the image directory + t!(fs::create_dir_all(image.join("share/zsh/site-functions"))); + t!(fs::create_dir_all(image.join("etc/bash_completions.d"))); + install(&build.tool(&compiler, "cargo"), &image.join("bin"), 0o755); + for man in t!(etc.join("man").read_dir()) { + let man = t!(man); + install(&man.path(), &image.join("share/man/man1"), 0o644); + } + install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644); + copy(&etc.join("cargo.bashcomp.sh"), + &image.join("etc/bash_completions.d/cargo")); + let doc = image.join("share/doc/cargo"); + install(&src.join("README.md"), &doc, 0o644); + install(&src.join("LICENSE-MIT"), &doc, 0o644); + install(&src.join("LICENSE-APACHE"), &doc, 0o644); + install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644); + + // Prepare the overlay + let overlay = tmp.join("cargo-overlay"); + drop(fs::remove_dir_all(&overlay)); + t!(fs::create_dir_all(&overlay)); + install(&src.join("README.md"), &overlay, 0o644); + install(&src.join("LICENSE-MIT"), &overlay, 0o644); + install(&src.join("LICENSE-APACHE"), &overlay, 0o644); + install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644); + t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); + + // Generate the installer tarball + let mut cmd = Command::new("sh"); + cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh"))) + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=Rust-is-ready-to-roll.") + .arg(format!("--image-dir={}", sanitize_sh(&image))) + .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build)))) + .arg(format!("--output-dir={}", sanitize_sh(&distdir(build)))) + .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay))) + .arg(format!("--package-name={}-{}", name, target)) + .arg("--component-name=cargo") + .arg("--legacy-manifest-dirs=rustlib,cargo"); + build.run(&mut cmd); } /// Creates a combined installer for the specified target in the provided stage. @@ -603,7 +615,7 @@ pub fn extended(build: &Build, stage: u32, target: &str) { install(&build.src.join("COPYRIGHT"), &overlay, 0o644); install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644); install(&build.src.join("LICENSE-MIT"), &overlay, 0o644); - let version = &build.version; + let version = build.rust_version(); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); install(&etc.join("README.md"), &overlay, 0o644); @@ -876,16 +888,16 @@ pub fn extended(build: &Build, stage: u32, target: &str) { } fn add_env(build: &Build, cmd: &mut Command, target: &str) { - let mut parts = build.release_num.split('.'); - cmd.env("CFG_RELEASE_INFO", &build.version) - .env("CFG_RELEASE_NUM", &build.release_num) - .env("CFG_RELEASE", &build.release) - .env("CFG_PRERELEASE_VERSION", &build.prerelease_version) + let mut parts = channel::CFG_RELEASE_NUM.split('.'); + cmd.env("CFG_RELEASE_INFO", build.rust_version()) + .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM) + .env("CFG_RELEASE", build.rust_release()) + .env("CFG_PRERELEASE_VERSION", channel::CFG_PRERELEASE_VERSION) .env("CFG_VER_MAJOR", parts.next().unwrap()) .env("CFG_VER_MINOR", parts.next().unwrap()) .env("CFG_VER_PATCH", parts.next().unwrap()) .env("CFG_VER_BUILD", "0") // just needed to build - .env("CFG_PACKAGE_VERS", package_vers(build)) + .env("CFG_PACKAGE_VERS", build.rust_package_vers()) .env("CFG_PACKAGE_NAME", pkgname(build, "rust")) .env("CFG_BUILD", target) .env("CFG_CHANNEL", &build.config.channel); @@ -925,7 +937,8 @@ pub fn hash_and_sign(build: &Build) { cmd.arg(sign); cmd.arg(distdir(build)); cmd.arg(today.trim()); - cmd.arg(package_vers(build)); + cmd.arg(build.rust_package_vers()); + cmd.arg(build.cargo_info.version(build, &build.crates["cargo"].version)); cmd.arg(addr); t!(fs::create_dir_all(distdir(build))); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 5fd6570e1610c..d19e5b1b88456 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -77,12 +77,9 @@ pub fn standalone(build: &Build, target: &str) { if !up_to_date(&version_input, &version_info) { let mut info = String::new(); t!(t!(File::open(&version_input)).read_to_string(&mut info)); - let blank = String::new(); - let short = build.short_ver_hash.as_ref().unwrap_or(&blank); - let hash = build.ver_hash.as_ref().unwrap_or(&blank); - let info = info.replace("VERSION", &build.release) - .replace("SHORT_HASH", short) - .replace("STAMP", hash); + let info = info.replace("VERSION", &build.rust_release()) + .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or("")) + .replace("STAMP", build.rust_info.sha().unwrap_or("")); t!(t!(File::create(&version_info)).write_all(info.as_bytes())); } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index efc460f35838c..ba8442ebd8c37 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf, Component}; use std::process::Command; use Build; -use dist::{package_vers, sanitize_sh, tmpdir}; +use dist::{sanitize_sh, tmpdir}; /// Installs everything. pub fn install(build: &Build, stage: u32, host: &str) { @@ -59,7 +59,7 @@ pub fn install(build: &Build, stage: u32, host: &str) { fn install_sh(build: &Build, package: &str, name: &str, stage: u32, host: &str, prefix: &Path, docdir: &Path, libdir: &Path, mandir: &Path, empty_dir: &Path) { println!("Install {} stage{} ({})", package, stage, host); - let package_name = format!("{}-{}-{}", name, package_vers(build), host); + let package_name = format!("{}-{}-{}", name, build.rust_package_vers(), host); let mut cmd = Command::new("sh"); cmd.current_dir(empty_dir) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2b34142b3b0f3..b432ef18054c7 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -84,7 +84,7 @@ use std::fs::{self, File}; use std::path::{Component, PathBuf, Path}; use std::process::Command; -use build_helper::{run_silent, output, mtime}; +use build_helper::{run_silent, run_suppressed, output, mtime}; use util::{exe, libdir, add_lib_path}; @@ -148,16 +148,9 @@ pub struct Build { rustc: PathBuf, src: PathBuf, out: PathBuf, - release: String, - unstable_features: bool, - ver_hash: Option, - short_ver_hash: Option, - ver_date: Option, - version: String, - package_vers: String, + rust_info: channel::GitInfo, + cargo_info: channel::GitInfo, local_rebuild: bool, - release_num: String, - prerelease_version: String, // Probed tools at runtime lldb_version: Option, @@ -173,6 +166,7 @@ pub struct Build { #[derive(Debug)] struct Crate { name: String, + version: String, deps: Vec, path: PathBuf, doc_step: String, @@ -236,6 +230,8 @@ impl Build { } None => false, }; + let rust_info = channel::GitInfo::new(&src); + let cargo_info = channel::GitInfo::new(&src.join("src/tools/cargo")); Build { flags: flags, @@ -245,22 +241,15 @@ impl Build { src: src, out: out, - release: String::new(), - unstable_features: false, - ver_hash: None, - short_ver_hash: None, - ver_date: None, - version: String::new(), + rust_info: rust_info, + cargo_info: cargo_info, local_rebuild: local_rebuild, - package_vers: String::new(), cc: HashMap::new(), cxx: HashMap::new(), crates: HashMap::new(), lldb_version: None, lldb_python_dir: None, is_sudo: is_sudo, - release_num: String::new(), - prerelease_version: String::new(), } } @@ -278,15 +267,14 @@ impl Build { cc::find(self); self.verbose("running sanity check"); sanity::check(self); - self.verbose("collecting channel variables"); - channel::collect(self); // If local-rust is the same major.minor as the current version, then force a local-rebuild let local_version_verbose = output( Command::new(&self.rustc).arg("--version").arg("--verbose")); let local_release = local_version_verbose .lines().filter(|x| x.starts_with("release:")) .next().unwrap().trim_left_matches("release:").trim(); - if local_release.split('.').take(2).eq(self.release.split('.').take(2)) { + let my_version = channel::CFG_RELEASE_NUM; + if local_release.split('.').take(2).eq(my_version.split('.').take(2)) { self.verbose(&format!("auto-detected local-rebuild {}", local_release)); self.local_rebuild = true; } @@ -797,6 +785,12 @@ impl Build { run_silent(cmd) } + /// Runs a command, printing out nice contextual information if it fails. + fn run_quiet(&self, cmd: &mut Command) { + self.verbose(&format!("running: {:?}", cmd)); + run_suppressed(cmd) + } + /// Prints a message if this build is configured in verbose mode. fn verbose(&self, msg: &str) { if self.flags.verbose() || self.config.verbose() { @@ -914,6 +908,82 @@ impl Build { compiler.stage >= 2 && self.config.host.iter().any(|h| h == target) } + + /// Returns the directory that OpenSSL artifacts are compiled into if + /// configured to do so. + fn openssl_dir(&self, target: &str) -> Option { + // OpenSSL not used on Windows + if target.contains("windows") { + None + } else if self.config.openssl_static { + Some(self.out.join(target).join("openssl")) + } else { + None + } + } + + /// Returns the directory that OpenSSL artifacts are installed into if + /// configured as such. + fn openssl_install_dir(&self, target: &str) -> Option { + self.openssl_dir(target).map(|p| p.join("install")) + } + + /// Given `num` in the form "a.b.c" return a "release string" which + /// describes the release version number. + /// + /// For example on nightly this returns "a.b.c-nightly", on beta it returns + /// "a.b.c-beta.1" and on stable it just returns "a.b.c". + fn release(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => format!("{}-beta{}", num, channel::CFG_PRERELEASE_VERSION), + "nightly" => format!("{}-nightly", num), + _ => format!("{}-dev", num), + } + } + + /// Returns the value of `release` above for Rust itself. + fn rust_release(&self) -> String { + self.release(channel::CFG_RELEASE_NUM) + } + + /// Returns the "package version" for a component given the `num` release + /// number. + /// + /// The package version is typically what shows up in the names of tarballs. + /// For channels like beta/nightly it's just the channel name, otherwise + /// it's the `num` provided. + fn package_vers(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => "beta".to_string(), + "nightly" => "nightly".to_string(), + _ => format!("{}-dev", num), + } + } + + /// Returns the value of `package_vers` above for Rust itself. + fn rust_package_vers(&self) -> String { + self.package_vers(channel::CFG_RELEASE_NUM) + } + + /// Returns the `version` string associated with this compiler for Rust + /// itself. + /// + /// Note that this is a descriptive string which includes the commit date, + /// sha, version, etc. + fn rust_version(&self) -> String { + self.rust_info.version(self, channel::CFG_RELEASE_NUM) + } + + /// Returns whether unstable features should be enabled for the compiler + /// we're building. + fn unstable_features(&self) -> bool { + match &self.config.channel[..] { + "stable" | "beta" => false, + "nightly" | _ => true, + } + } } impl<'a> Compiler<'a> { diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs index 5ab542b6a2489..5918fe41e7c8b 100644 --- a/src/bootstrap/metadata.rs +++ b/src/bootstrap/metadata.rs @@ -27,6 +27,7 @@ struct Output { struct Package { id: String, name: String, + version: String, source: Option, manifest_path: String, } @@ -72,6 +73,7 @@ fn build_krate(build: &mut Build, krate: &str) { test_step: format!("test-crate-{}", package.name), bench_step: format!("bench-crate-{}", package.name), name: package.name, + version: package.version, deps: Vec::new(), path: path, }); diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index f16fc2092f616..f8113858fdc5d 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -191,3 +191,104 @@ pub fn test_helpers(build: &Build, target: &str) { .file(build.src.join("src/rt/rust_test_helpers.c")) .compile("librust_test_helpers.a"); } +const OPENSSL_VERS: &'static str = "1.0.2k"; +const OPENSSL_SHA256: &'static str = + "6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0"; + +pub fn openssl(build: &Build, target: &str) { + let out = match build.openssl_dir(target) { + Some(dir) => dir, + None => return, + }; + + let stamp = out.join(".stamp"); + let mut contents = String::new(); + drop(File::open(&stamp).and_then(|mut f| f.read_to_string(&mut contents))); + if contents == OPENSSL_VERS { + return + } + t!(fs::create_dir_all(&out)); + + let name = format!("openssl-{}.tar.gz", OPENSSL_VERS); + let tarball = out.join(&name); + if !tarball.exists() { + let tmp = tarball.with_extension("tmp"); + build.run(Command::new("curl") + .arg("-o").arg(&tmp) + .arg(format!("https://www.openssl.org/source/{}", name))); + let mut shasum = if target.contains("apple") { + let mut cmd = Command::new("shasum"); + cmd.arg("-a").arg("256"); + cmd + } else { + Command::new("sha256sum") + }; + let output = output(&mut shasum.arg(&tmp)); + let found = output.split_whitespace().next().unwrap(); + if found != OPENSSL_SHA256 { + panic!("downloaded openssl sha256 different\n\ + expected: {}\n\ + found: {}\n", OPENSSL_SHA256, found); + } + t!(fs::rename(&tmp, &tarball)); + } + let obj = out.join(format!("openssl-{}", OPENSSL_VERS)); + let dst = build.openssl_install_dir(target).unwrap(); + drop(fs::remove_dir_all(&obj)); + drop(fs::remove_dir_all(&dst)); + build.run(Command::new("tar").arg("xf").arg(&tarball).current_dir(&out)); + + let mut configure = if target == "i686-unknown-linux-gnu" { + let mut cmd = Command::new("setarch"); + cmd.arg("i386").arg(obj.join("Configure")); + cmd + } else { + Command::new(obj.join("Configure")) + }; + + configure.arg(format!("--prefix={}", dst.display())); + configure.arg("no-dso"); + configure.arg("no-ssl2"); + configure.arg("no-ssl3"); + configure.arg("no-comp"); + + let os = match target { + "aarch64-unknown-linux-gnu" => "linux-aarch64", + "arm-unknown-linux-gnueabi" => "linux-armv4", + "arm-unknown-linux-gnueabihf" => "linux-armv4", + "armv7-unknown-linux-gnueabihf" => "linux-armv4", + "i686-apple-darwin" => "darwin-i386-cc", + "i686-unknown-freebsd" => "BSD-x86-elf", + "i686-unknown-linux-gnu" => "linux-elf", + "i686-unknown-linux-musl" => "linux-elf", + "mips-unknown-linux-gnu" => "linux-mips32", + "mips64-unknown-linux-gnuabi64" => "linux64-mips64", + "mips64el-unknown-linux-gnuabi64" => "linux64-mips64", + "mipsel-unknown-linux-gnu" => "linux-mips32", + "powerpc-unknown-linux-gnu" => "linux-ppc", + "powerpc64-unknown-linux-gnu" => "linux-ppc64", + "powerpc64le-unknown-linux-gnu" => "linux-ppc64le", + "s390x-unknown-linux-gnu" => "linux64-s390x", + "x86_64-apple-darwin" => "darwin64-x86_64-cc", + "x86_64-unknown-freebsd" => "BSD-x86_64", + "x86_64-unknown-linux-gnu" => "linux-x86_64", + "x86_64-unknown-linux-musl" => "linux-x86_64", + "x86_64-unknown-netbsd" => "BSD-x86_64", + _ => panic!("don't know how to configure OpenSSL for {}", target), + }; + configure.arg(os); + configure.arg("-fPIC"); + if target.contains("i686") { + configure.arg("-m32"); + } + configure.current_dir(&obj); + println!("Configuring openssl for {}", target); + build.run_quiet(&mut configure); + println!("Building openssl for {}", target); + build.run_quiet(Command::new("make").current_dir(&obj)); + println!("Installing openssl for {}", target); + build.run_quiet(Command::new("make").arg("install").current_dir(&obj)); + + let mut f = t!(File::create(&stamp)); + t!(f.write_all(OPENSSL_VERS.as_bytes())); +} diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index dcd3407e174aa..b49158d3e8887 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -478,9 +478,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|s| s.name("dist-src")) .run(move |_| check::distcheck(build)); - rules.build("test-helpers", "src/rt/rust_test_helpers.c") .run(move |s| native::test_helpers(build, s.target)); + rules.build("openssl", "path/to/nowhere") + .run(move |s| native::openssl(build, s.target)); // Some test suites are run inside emulators, and most of our test binaries // are linked dynamically which means we need to ship the standard library @@ -547,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build("tool-qemu-test-client", "src/tools/qemu-test-client") .dep(|s| s.name("libstd")) .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-client")); + rules.build("tool-cargo", "src/tools/cargo") + .dep(|s| s.name("libstd")) + .dep(|s| s.stage(0).host(s.target).name("openssl")) + .run(move |s| compile::tool(build, s.stage, s.target, "cargo")); // ======================================================================== // Documentation targets @@ -673,6 +678,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.dist("dist-cargo", "cargo") .host(true) .only_host_build(true) + .dep(|s| s.name("tool-cargo")) .run(move |s| dist::cargo(build, s.stage, s.target)); rules.dist("dist-extended", "extended") .default(build.config.extended) @@ -1180,6 +1186,7 @@ mod tests { build_step: "build-crate-std".to_string(), test_step: "test-std".to_string(), bench_step: "bench-std".to_string(), + version: String::new(), }); build.crates.insert("test".to_string(), ::Crate { name: "test".to_string(), @@ -1189,10 +1196,12 @@ mod tests { build_step: "build-crate-test".to_string(), test_step: "test-test".to_string(), bench_step: "bench-test".to_string(), + version: String::new(), }); build.crates.insert("rustc-main".to_string(), ::Crate { name: "rustc-main".to_string(), deps: Vec::new(), + version: String::new(), path: cwd.join("src/rustc-main"), doc_step: "doc-rustc-main".to_string(), build_step: "build-crate-rustc-main".to_string(), diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index 3dfd293808286..08f1f31c2d74b 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -53,6 +53,24 @@ pub fn run_silent(cmd: &mut Command) { } } +pub fn run_suppressed(cmd: &mut Command) { + let output = match cmd.output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", + cmd, e)), + }; + if !output.status.success() { + fail(&format!("command did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n", + cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr))); + } +} + pub fn gnu_target(target: &str) -> String { match target { "i686-pc-windows-msvc" => "i686-pc-win32".to_string(), diff --git a/src/ci/run.sh b/src/ci/run.sh index c4b2556599366..c52ebc7a46779 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -24,6 +24,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" +RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-openssl-static" if [ "$DIST_SRC" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 6d40dcacdba43..18c752df30053 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -131,7 +131,8 @@ macro_rules! t { } struct Builder { - channel: String, + rust_release: String, + cargo_release: String, input: PathBuf, output: PathBuf, gpg_passphrase: String, @@ -147,13 +148,15 @@ fn main() { let input = PathBuf::from(args.next().unwrap()); let output = PathBuf::from(args.next().unwrap()); let date = args.next().unwrap(); - let channel = args.next().unwrap(); + let rust_release = args.next().unwrap(); + let cargo_release = args.next().unwrap(); let s3_address = args.next().unwrap(); let mut passphrase = String::new(); t!(io::stdin().read_to_string(&mut passphrase)); Builder { - channel: channel, + rust_release: rust_release, + cargo_release: cargo_release, input: input, output: output, gpg_passphrase: passphrase, @@ -184,10 +187,10 @@ impl Builder { manifest.insert("pkg".to_string(), toml::encode(&pkg)); let manifest = toml::Value::Table(manifest).to_string(); - let filename = format!("channel-rust-{}.toml", self.channel); + let filename = format!("channel-rust-{}.toml", self.rust_release); self.write_manifest(&manifest, &filename); - if self.channel != "beta" && self.channel != "nightly" { + if self.rust_release != "beta" && self.rust_release != "nightly" { self.write_manifest(&manifest, "channel-rust-stable.toml"); } } @@ -336,11 +339,11 @@ impl Builder { fn filename(&self, component: &str, target: &str) -> String { if component == "rust-src" { - format!("rust-src-{}.tar.gz", self.channel) + format!("rust-src-{}.tar.gz", self.rust_release) } else if component == "cargo" { - format!("cargo-nightly-{}.tar.gz", target) + format!("cargo-{}-{}.tar.gz", self.cargo_release, target) } else { - format!("{}-{}-{}.tar.gz", component, self.channel, target) + format!("{}-{}-{}.tar.gz", component, self.rust_release, target) } } diff --git a/src/tools/tidy/src/cargo.rs b/src/tools/tidy/src/cargo.rs index 11acb64743a7a..053f0bbe3b81d 100644 --- a/src/tools/tidy/src/cargo.rs +++ b/src/tools/tidy/src/cargo.rs @@ -20,7 +20,7 @@ use std::fs::File; use std::path::Path; pub fn check(path: &Path, bad: &mut bool) { - if path.ends_with("vendor") { + if !super::filter_dirs(path) { return } for entry in t!(path.read_dir(), path).map(|e| t!(e)) { diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index e1c44a20e9756..3bf396db4d39d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -16,6 +16,7 @@ use std::path::Path; static LICENSES: &'static [&'static str] = &[ "MIT/Apache-2.0", + "MIT / Apache-2.0", "Apache-2.0/MIT", "MIT OR Apache-2.0", "MIT", @@ -25,6 +26,7 @@ static LICENSES: &'static [&'static str] = &[ /// These MPL licensed projects are acceptable, but only these. static EXCEPTIONS: &'static [&'static str] = &[ "mdbook", + "openssl", "pest", "thread-id", ]; diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index 1bb7919c1d35b..2af891b5b8562 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -70,6 +70,7 @@ fn filter_dirs(path: &Path) -> bool { "src/rustllvm", "src/rust-installer", "src/liblibc", + "src/tools/cargo", "src/vendor", ]; skip.iter().any(|p| path.ends_with(p))