Joshua's CI playground

Cargo.lock

@@ -408,9 +408,8 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.58"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518"
+version = "1.0.59"
+source = "git+https://github.com/alexcrichton/cc-rs?rev=192a84d9313210e09f2af1d7a8c27f70bd6a0f6e#192a84d9313210e09f2af1d7a8c27f70bd6a0f6e"
 dependencies = [
  "jobserver",
 ]

Cargo.toml

@@ -30,6 +30,7 @@ members = [
   "src/tools/expand-yaml-anchors",
 ]
 exclude = [
+  "silicon",
   "build",
   # HACK(eddyb) This hardcodes the fact that our CI uses `/checkout/obj`.
   "obj",
@@ -94,6 +95,8 @@ rustc-std-workspace-core = { path = 'library/rustc-std-workspace-core' }
 rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
 rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
 
+cc = { git = 'https://github.com/alexcrichton/cc-rs', rev = '192a84d9313210e09f2af1d7a8c27f70bd6a0f6e' }
+
 # This crate's integration with libstd is a bit wonky, so we use a submodule
 # instead of a crates.io dependency. Make sure everything else in the repo is
 # also using the submodule, however, so we can avoid duplicate copies of the

silicon/.gitignore

@@ -0,0 +1,4 @@
+/cc-rs
+/libc
+Makefile
+build

silicon/README.md

@@ -0,0 +1,198 @@
+# Setup
+
+1. Download and install Xcode 12 beta, then set it as your default
+
+    ```
+    xcode-select -s /Applications/Xcode-beta.app/Contents/Developer/
+    ```
+
+1. Install CMake
+
+    Install this however you like. A simple way is to download it and add it to your path;
+
+    ```
+    curl -L 'https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0-Darwin-x86_64.tar.gz' -o cmake.tgz
+    tar -xf cmake.tgz
+    export PATH="${PATH}:${PWD}/cmake-3.18.0-Darwin-x86_64/CMake.app/Contents/bin"
+    ```
+
+- Clone [this fork of rust][fork], checkout the "silicon" branch.
+
+[fork]: https://github.com/shepmaster/rust
+
+# Cross-compile the compiler for the DTK
+
+The current Rust compiler doesn't know how to target the DTK. We use
+some advanced build system configuration to build an enhanced version
+that can.
+
+This section can be simplified when an appropriate target
+specification is added to the standard Rust bootstrapping compiler.
+
+## Compiling on x86_64-apple-darwin (a.k.a. not the DTK)
+
+1. `cd silicon/cross`
+
+1. Configure the compiler
+
+    ```
+    ../../configure
+    ```
+
+1. Cross-compile the compiler
+
+    ```
+    SDKROOT=$(xcrun -sdk macosx11.0 --show-sdk-path) \
+    MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.0 --show-sdk-platform-version) \
+    DESTDIR=/tmp/crossed \
+    ../../x.py install -i --stage 1 --host aarch64-apple-darwin --target aarch64-apple-darwin,x86_64-apple-darwin \
+    src/librustc library/std
+    ```
+
+1. Copy the cross-compiler to the DTK
+
+```
+rsync -avz /tmp/crossed/ dtk:crossed/
+```
+
+## Compiling on aarch64-apple-darwin (a.k.a. the DTK)
+
+1. Configure the compiler
+
+    ```
+    ../../configure \
+    --build=x86_64-apple-darwin
+    ```
+
+1. Cross-compile the compiler
+
+    ```
+    SDKROOT=$(xcrun -sdk macosx11.0 --show-sdk-path) \
+    MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.0 --show-sdk-platform-version) \
+    DESTDIR=~/crossed \
+    ../../x.py install -i --stage 1 --host aarch64-apple-darwin --target aarch64-apple-darwin,x86_64-apple-darwin \
+    src/librustc library/std
+    ```
+
+# Next steps
+
+At this point, you have a compiler that should be suffient for many goals.
+
+## Build a native binary
+
+On the DTK, you can now use native Rust:
+
+```
+% echo 'fn main() { println!("Hello, DTK!") }' > hello.rs
+
+% ~/crossed/usr/local/bin/rustc hello.rs
+
+% ./hello
+Hello, DTK!
+
+% file hello
+hello: Mach-O 64-bit executable arm64
+```
+
+## Installing rustup
+
+Rustup doesn't know that `x86_64` binaries can run on `aarch64`, so we
+have to hack the install script so it thinks that the machine is
+`x86_64`.
+
+1. Download the installer script
+
+    ```
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh
+    ```
+
+1. Apply this patch to force it to think we are x86_64:
+
+    ```diff
+    --- /tmp/rustup.sh  2020-07-05 07:49:10.000000000 -0700
+    +++ /tmp/rustup-hacked.sh   2020-07-04 17:27:08.000000000 -0700
+    @@ -188,6 +188,8 @@
+             fi
+         fi
+
+    +    _cputype=x86_64
+    +
+         case "$_ostype" in
+
+             Android)
+    ```
+
+1. Install rustup
+
+    ```
+    bash /tmp/rustup.sh --default-toolchain none
+    ```
+
+1. Create a toolchain and make it the default:
+
+    ```
+    rustup toolchain link native ~/crossed/usr/local/
+    rustup default native
+    ```
+
+## Build the compiler on the DTK using the native toolchain
+
+Re-run the setup steps on the DTK, if you haven't already.
+
+1. `cd silicon/pure-native`
+
+1. Get an x86_64 version of Cargo to use during bootstrapping
+
+    ```
+    curl 'https://static.rust-lang.org/dist/2020-06-16/cargo-beta-x86_64-apple-darwin.tar.xz' -o cargo.tar.xz
+    tar -xf cargo.tar.xz
+    ```
+
+1. Configure the compiler
+
+    ```
+    ../../configure \
+    --set build.cargo="${PWD}/cargo-beta-x86_64-apple-darwin/cargo/bin/cargo" \
+    --set build.rustc="${HOME}/crossed/usr/local/bin/rustc" \
+    --set build.rustfmt="${HOME}/crossed/usr/local/bin/rustfmt"
+    ```
+
+1. Build and install the compiler
+
+    ```
+    DESTDIR=~/native-build \
+    ../../x.py install -i --stage 1 --host aarch64-apple-darwin --target aarch64-apple-darwin,x86_64-apple-darwin \
+    src/librustc library/std cargo
+    ```
+
+# Miscellaneous notes
+
+- `SDKROOT` - used to indicate to `rustc` what SDK whould be
+  used. Point this at the `MacOSX11` target inside your `Xcode.app` /
+  `Xcode-beta.app`
+
+- `MACOSX_DEPLOYMENT_TARGET` - used to indicate to `rustc` what
+  version of macOS to target.
+
+- We build LLVM *3 times* and use incremental builds for faster
+  iteration times. This can easily take up **30 GiB** of
+  space. Building can take several hours on an 2.9 GHz 6-Core Intel
+  Core i9.
+
+- Rough build times
+  - 2.9 GHz 6-Core Intel Core i9
+    - bootstrap - `Build completed successfully in 0:30:32`
+    - cross - `Build completed successfully in 1:04:22`
+  - DTK
+    - pure-native - `Build completed successfully in 0:39:10`
+
+# Troubleshooting
+
+- Failure while running `x.py`? Add `-vvvvvvv`.
+
+- "archive member 'something.o' with length xxxx is not mach-o or llvm bitcode file 'some-tmp-path/libbacktrace_sys-e02ed1fd10e8b80d.rlib' for architecture arm64"?
+  - Find the rlib: `find build -name 'libbacktrace_sys*'`
+  - Remove it: `rm build/aarch64-apple-darwin/stage0-std/aarch64-apple-darwin/release/deps/libbacktrace_sys-e02ed1fd10e8b80d.r{meta,lib}`
+  - Look at the objects it created in `build/aarch64-apple-darwin/stage0-std/aarch64-apple-darwin/release/build/backtrace-sys-fec595dd32504c90/`
+  - Delete those too: `rm -r build/aarch64-apple-darwin/stage0-std/aarch64-apple-darwin/release/build/backtrace-sys-fec595dd32504c90`
+  - Rebuild with verbose

silicon/cross/.gitignore

@@ -0,0 +1 @@
+config.toml

silicon/pure-native/.gitignore

@@ -0,0 +1,3 @@
+cargo-beta-x86_64-apple-darwin/
+cargo.tar.xz
+config.toml

src/bootstrap/config.rs

@@ -590,12 +590,12 @@ impl Config {
         set(&mut config.print_step_timings, build.print_step_timings);
 
         // See https://github.com/rust-lang/compiler-team/issues/326
-        config.stage = match config.cmd {
+        config.stage = match dbg!(&config.cmd) {
             Subcommand::Doc { .. } => flags.stage.or(build.doc_stage).unwrap_or(0),
             Subcommand::Build { .. } => flags.stage.or(build.build_stage).unwrap_or(1),
             Subcommand::Test { .. } => flags.stage.or(build.test_stage).unwrap_or(1),
             Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2),
-            Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2),
+            Subcommand::Dist { .. } => dbg!(dbg!(flags.stage).or(build.dist_stage).unwrap_or(2)),
             Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2),
             // These are all bootstrap tools, which don't depend on the compiler.
             // The stage we pass shouldn't matter, but use 0 just in case.

src/bootstrap/doc.rs

@@ -752,6 +752,7 @@ impl Step for RustcBook {
         let out_listing = out_base.join("src/lints");
         builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base);
         builder.info(&format!("Generating lint docs ({})", self.target));
+
         let rustc = builder.rustc(self.compiler);
         // The tool runs `rustc` for extracting output examples, so it needs a
         // functional sysroot.
@@ -762,7 +763,8 @@ impl Step for RustcBook {
         cmd.arg("--out");
         cmd.arg(&out_listing);
         cmd.arg("--rustc");
-        cmd.arg(rustc);
+        cmd.arg(&rustc);
+        cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg());
         if builder.config.verbose() {
             cmd.arg("--verbose");
         }

src/bootstrap/flags.rs

@@ -40,6 +40,7 @@ pub struct Flags {
     pub llvm_skip_rebuild: Option<bool>,
 }
 
+#[derive(Debug)]
 pub enum Subcommand {
     Build {
         paths: Vec<PathBuf>,
@@ -98,7 +99,6 @@ impl Default for Subcommand {
 
 impl Flags {
     pub fn parse(args: &[String]) -> Flags {
-        let mut extra_help = String::new();
         let mut subcommand_help = String::from(
             "\
 Usage: x.py <subcommand> [options] [<paths>...]
@@ -170,16 +170,6 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
             "VALUE",
         );
 
-        // fn usage()
-        let usage =
-            |exit_code: i32, opts: &Options, subcommand_help: &str, extra_help: &str| -> ! {
-                println!("{}", opts.usage(subcommand_help));
-                if !extra_help.is_empty() {
-                    println!("{}", extra_help);
-                }
-                process::exit(exit_code);
-            };
-
         // We can't use getopt to parse the options until we have completed specifying which
         // options are valid, but under the current implementation, some options are conditional on
         // the subcommand. Therefore we must manually identify the subcommand first, so that we can
@@ -263,12 +253,38 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
             _ => {}
         };
 
+        // fn usage()
+        let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! {
+            let mut extra_help = String::new();
+
+            // All subcommands except `clean` can have an optional "Available paths" section
+            if verbose {
+                let config = Config::parse(&["build".to_string()]);
+                let build = Build::new(config);
+
+                let maybe_rules_help = Builder::get_help(&build, subcommand.as_str());
+                extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str());
+            } else if !(subcommand.as_str() == "clean" || subcommand.as_str() == "fmt") {
+                extra_help.push_str(
+                    format!("Run `./x.py {} -h -v` to see a list of available paths.", subcommand)
+                        .as_str(),
+                );
+            }
+
+            println!("{}", opts.usage(subcommand_help));
+            if !extra_help.is_empty() {
+                println!("{}", extra_help);
+            }
+            process::exit(exit_code);
+        };
+
         // Done specifying what options are possible, so do the getopts parsing
         let matches = opts.parse(&args[..]).unwrap_or_else(|e| {
             // Invalid argument/option format
             println!("\n{}\n", e);
-            usage(1, &opts, &subcommand_help, &extra_help);
+            usage(1, &opts, false, &subcommand_help);
         });
+
         // Extra sanity check to make sure we didn't hit this crazy corner case:
         //
         //     ./x.py --frobulate clean build
@@ -436,24 +452,11 @@ Arguments:
         let paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>();
 
         let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from);
-
-        // All subcommands except `clean` can have an optional "Available paths" section
-        if matches.opt_present("verbose") {
-            let config = Config::parse(&["build".to_string()]);
-            let build = Build::new(config);
-
-            let maybe_rules_help = Builder::get_help(&build, subcommand.as_str());
-            extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str());
-        } else if !(subcommand.as_str() == "clean" || subcommand.as_str() == "fmt") {
-            extra_help.push_str(
-                format!("Run `./x.py {} -h -v` to see a list of available paths.", subcommand)
-                    .as_str(),
-            );
-        }
+        let verbose = matches.opt_present("verbose");
 
         // User passed in -h/--help?
         if matches.opt_present("help") {
-            usage(0, &opts, &subcommand_help, &extra_help);
+            usage(0, &opts, verbose, &subcommand_help);
         }
 
         let cmd = match subcommand.as_str() {
@@ -483,7 +486,7 @@ Arguments:
             "clean" => {
                 if !paths.is_empty() {
                     println!("\nclean does not take a path argument\n");
-                    usage(1, &opts, &subcommand_help, &extra_help);
+                    usage(1, &opts, verbose, &subcommand_help);
                 }
 
                 Subcommand::Clean { all: matches.opt_present("all") }
@@ -494,12 +497,12 @@ Arguments:
             "run" | "r" => {
                 if paths.is_empty() {
                     println!("\nrun requires at least a path!\n");
-                    usage(1, &opts, &subcommand_help, &extra_help);
+                    usage(1, &opts, verbose, &subcommand_help);
                 }
                 Subcommand::Run { paths }
             }
             _ => {
-                usage(1, &opts, &subcommand_help, &extra_help);
+                usage(1, &opts, verbose, &subcommand_help);
             }
         };