Auto merge of rust-lang#123557 - GuillaumeGomez:rollup-3af7urf, r=Gui…

…llaumeGomez

Rollup of 4 pull requests

Successful merges:

 - rust-lang#123541 (remove miri-test-libstd hacks that are no longer needed)
 - rust-lang#123552 (Add missing -Zquery-dep-graph to the spike-neg incr comp tests)
 - rust-lang#123553 (Miri subtree update)
 - rust-lang#123554 (Simplify/cleanup `search-result-color.goml`)

r? `@ghost`
`@rustbot` modify labels: rollup

library/alloc/src/lib.rs

@@ -56,11 +56,6 @@
 //! [`Rc`]: rc
 //! [`RefCell`]: core::cell
 
-// To run alloc tests without x.py without ending up with two copies of alloc, Miri needs to be
-// able to "empty" this crate. See <https://github.com/rust-lang/miri-test-libstd/issues/4>.
-// rustc itself never sets the feature, so this line has no effect there.
-#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))]
-//
 #![allow(unused_attributes)]
 #![stable(feature = "alloc", since = "1.36.0")]
 #![doc(
@@ -71,7 +66,6 @@
 #![doc(cfg_hide(
     not(test),
     not(any(test, bootstrap)),
-    any(not(feature = "miri-test-libstd"), test, doctest),
     no_global_oom_handling,
     not(no_global_oom_handling),
     not(no_rc),

library/core/src/lib.rs

@@ -57,10 +57,7 @@
 //
 // This cfg won't affect doc tests.
 #![cfg(not(test))]
-// To run core tests without x.py without ending up with two copies of core, Miri needs to be
-// able to "empty" this crate. See <https://github.com/rust-lang/miri-test-libstd/issues/4>.
-// rustc itself never sets the feature, so this line has no effect there.
-#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))]
+//
 #![stable(feature = "core", since = "1.6.0")]
 #![doc(
     html_playground_url = "https://play.rust-lang.org/",
@@ -71,7 +68,6 @@
 #![doc(rust_logo)]
 #![doc(cfg_hide(
     not(test),
-    any(not(feature = "miri-test-libstd"), test, doctest),
     no_fp_fmt_parse,
     target_pointer_width = "16",
     target_pointer_width = "32",

library/std/src/lib.rs

@@ -212,13 +212,7 @@
 //! [rust-discord]: https://discord.gg/rust-lang
 //! [array]: prim@array
 //! [slice]: prim@slice
-// To run std tests without x.py without ending up with two copies of std, Miri needs to be
-// able to "empty" this crate. See <https://github.com/rust-lang/miri-test-libstd/issues/4>.
-// rustc itself never sets the feature, so this line has no effect there.
-#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))]
-// miri-test-libstd also prefers to make std use the sysroot versions of the dependencies.
-#![cfg_attr(feature = "miri-test-libstd", feature(rustc_private))]
-//
+
 #![cfg_attr(not(feature = "restricted-std"), stable(feature = "rust1", since = "1.0.0"))]
 #![cfg_attr(feature = "restricted-std", unstable(feature = "restricted_std", issue = "none"))]
 #![cfg_attr(not(bootstrap), rustc_preserve_ub_checks)]

src/tools/miri/.github/workflows/ci.yml

@@ -24,8 +24,8 @@ jobs:
         include:
           - os: ubuntu-latest
             host_target: x86_64-unknown-linux-gnu
-          - os: macos-latest
-            host_target: x86_64-apple-darwin
+          - os: macos-14
+            host_target: aarch64-apple-darwin
           - os: windows-latest
             host_target: i686-pc-windows-msvc
     runs-on: ${{ matrix.os }}
@@ -49,15 +49,16 @@ jobs:
         with:
           path: |
             # Taken from <https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci>.
-            ~/.cargo/bin
+            # Cache package/registry information
             ~/.cargo/registry/index
             ~/.cargo/registry/cache
             ~/.cargo/git/db
-            # contains package information of crates installed via `cargo install`.
+            # Cache installed binaries
+            ~/.cargo/bin
             ~/.cargo/.crates.toml
             ~/.cargo/.crates2.json
-          key: ${{ runner.os }}-cargo-reset20230315-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-reset20230315
+          key: cargo-${{ runner.os }}-reset20240331-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: cargo-${{ runner.os }}-reset20240331
 
       - name: Install rustup-toolchain-install-master
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
@@ -98,15 +99,16 @@ jobs:
         with:
           path: |
             # Taken from <https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci>.
-            ~/.cargo/bin
+            # Cache package/registry information
             ~/.cargo/registry/index
             ~/.cargo/registry/cache
             ~/.cargo/git/db
-            # contains package information of crates installed via `cargo install`.
+            # Cache installed binaries
+            ~/.cargo/bin
             ~/.cargo/.crates.toml
             ~/.cargo/.crates2.json
-          key: ${{ runner.os }}-cargo-reset20230315-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-reset20230315
+          key: cargo-${{ runner.os }}-reset20240331-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: cargo-${{ runner.os }}-reset20240331
 
       - name: Install rustup-toolchain-install-master
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
@@ -130,6 +132,10 @@ jobs:
         run: ./miri fmt --check
       - name: clippy
         run: ./miri clippy -- -D warnings
+      - name: clippy (no features)
+        run: ./miri clippy --no-default-features -- -D warnings
+      - name: clippy (all features)
+        run: ./miri clippy --all-features -- -D warnings
       - name: rustdoc
         run: RUSTDOCFLAGS="-Dwarnings" ./miri cargo doc --document-private-items
 
@@ -189,7 +195,7 @@ jobs:
         with:
           fetch-depth: 256 # get a bit more of the history
       - name: install josh-proxy
-        run: RUSTFLAGS="--cap-lints warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
+        run: RUSTFLAGS="--cap-lints warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r23.12.04
       - name: setup bot git name and email
         run: |
           git config --global user.name 'The Miri Cronjob Bot'

src/tools/miri/CONTRIBUTING.md

@@ -241,18 +241,20 @@ We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit chan
 rustc and Miri repositories. You can install it as follows:
 
 ```sh
-cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
+RUSTFLAGS="--cap-lints=warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r23.12.04
 ```
 
 Josh will automatically be started and stopped by `./miri`.
 
 ### Importing changes from the rustc repo
 
+*Note: this usually happens automatically, so these steps rarely have to be done by hand.*
+
 We assume we start on an up-to-date master branch in the Miri repo.
 
 ```sh
 # Fetch and merge rustc side of the history. Takes ca 5 min the first time.
-# This will also update the 'rustc-version' file.
+# This will also update the `rustc-version` file.
 ./miri rustc-pull
 # Update local toolchain and apply formatting.
 ./miri toolchain && ./miri fmt
@@ -266,12 +268,6 @@ needed.
 
 ### Exporting changes to the rustc repo
 
-Keep in mind that pushing is the most complicated job that josh has to do -- pulling just filters
-the rustc history, but pushing needs to construct a new rustc history that would filter to the given
-Miri history! To avoid problems, it is a good idea to always pull immediately before you push. If
-you are getting strange errors, chances are you are running into [this josh
-bug](https://github.com/josh-project/josh/issues/998). In that case, please get in touch on Zulip.
-
 We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
 assuming we are on an up-to-date master branch:
 
@@ -280,9 +276,9 @@ assuming we are on an up-to-date master branch:
 ./miri rustc-push YOUR_NAME miri
 ```
 
-This will create a new branch called 'miri' in your fork, and the output should
-include a link to create a rustc PR that will integrate those changes into the
-main repository.
+This will create a new branch called `miri` in your fork, and the output should include a link that
+creates a rustc PR to integrate those changes into the main repository. If that PR has conflicts,
+you need to pull rustc changes into Miri first, and then re-do the rustc push.
 
 If this fails due to authentication problems, it can help to make josh push via ssh instead of
 https. Add the following to your `.gitconfig`:

src/tools/miri/ci/ci.sh

@@ -27,60 +27,59 @@ export RUSTFLAGS="-D warnings"
 export CARGO_INCREMENTAL=0
 export CARGO_EXTRA_FLAGS="--locked"
 
-# Determine configuration for installed build
+# Determine configuration for installed build (used by test-cargo-miri).
 echo "Installing release version of Miri"
-./miri install
-
-echo "Checking various feature flag configurations"
-./miri check --no-default-features # make sure this can be built
-./miri check # and this, too
-# `--all-features` is used for the build below, so no extra check needed.
+time ./miri install
 
 # Prepare debug build for direct `./miri` invocations.
 # We enable all features to make sure the Stacked Borrows consistency check runs.
 echo "Building debug version of Miri"
 export CARGO_EXTRA_FLAGS="$CARGO_EXTRA_FLAGS --all-features"
-./miri build --all-targets # the build that all the `./miri test` below will use
+time ./miri build --all-targets # the build that all the `./miri test` below will use
 
 endgroup
 
-# Test
+# Run tests. Recognizes these variables:
+# - MIRI_TEST_TARGET: the target to test. Empty for host target.
+# - GC_STRESS: if non-empty, run the GC stress test for the main test suite.
+# - MIR_OPT: if non-empty, re-run test `pass` tests with mir-opt-level=4
+# - MANY_SEEDS: if set to N, run the "many-seeds" tests N times
+# - TEST_BENCH: if non-empty, check that the benchmarks all build
+# - CARGO_MIRI_ENV: if non-empty, set some env vars and config to potentially confuse cargo-miri
 function run_tests {
-  if [ -n "${MIRI_TEST_TARGET:-}" ]; then
+  if [ -n "${MIRI_TEST_TARGET-}" ]; then
     begingroup "Testing foreign architecture $MIRI_TEST_TARGET"
   else
     begingroup "Testing host architecture"
   fi
 
   ## ui test suite
-  # On the host, also stress-test the GC.
-  if [ -z "${MIRI_TEST_TARGET:-}" ]; then
-    MIRIFLAGS="${MIRIFLAGS:-} -Zmiri-provenance-gc=1" ./miri test
+  if [ -n "${GC_STRESS-}" ]; then
+    time MIRIFLAGS="${MIRIFLAGS-} -Zmiri-provenance-gc=1" ./miri test
   else
-    ./miri test
+    time ./miri test
   fi
 
-  # Host-only tests
-  if [ -z "${MIRI_TEST_TARGET:-}" ]; then
-    # Running these on all targets is unlikely to catch more problems and would
-    # cost a lot of CI time.
-
+  ## advanced tests
+  if [ -n "${MIR_OPT-}" ]; then
     # Tests with optimizations (`-O` is what cargo passes, but crank MIR optimizations up all the
     # way, too).
     # Optimizations change diagnostics (mostly backtraces), so we don't check
     # them. Also error locations change so we don't run the failing tests.
     # We explicitly enable debug-assertions here, they are disabled by -O but we have tests
     # which exist to check that we panic on debug assertion failures.
-    MIRIFLAGS="${MIRIFLAGS:-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic}
-
+    time MIRIFLAGS="${MIRIFLAGS-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic}
+  fi
+  if [ -n "${MANY_SEEDS-}" ]; then
     # Also run some many-seeds tests. 64 seeds means this takes around a minute per test.
     # (Need to invoke via explicit `bash -c` for Windows.)
-    for FILE in tests/many-seeds/*.rs; do
-      MIRI_SEEDS=64 ./miri many-seeds "$BASH" -c "./miri run '$FILE'"
+    time for FILE in tests/many-seeds/*.rs; do
+      MIRI_SEEDS=$MANY_SEEDS ./miri many-seeds "$BASH" -c "./miri run '$FILE'"
     done
-
+  fi
+  if [ -n "${TEST_BENCH-}" ]; then
     # Check that the benchmarks build and run, but without actually benchmarking.
-    HYPERFINE="'$BASH' -c" ./miri bench
+    time HYPERFINE="'$BASH' -c" ./miri bench
   fi
 
   ## test-cargo-miri
@@ -91,16 +90,18 @@ function run_tests {
     PYTHON=python
   fi
   # Some environment setup that attempts to confuse the heck out of cargo-miri.
-  if [ "$HOST_TARGET" = x86_64-unknown-linux-gnu ]; then
-    # These act up on Windows (`which miri` produces a filename that does not exist?!?),
-    # so let's do this only on Linux. Also makes sure things work without these set.
-    export RUSTC=$(which rustc) # Produces a warning unless we also set MIRI
+  if [ -n "${CARGO_MIRI_ENV-}" ]; then
+    # These act up on Windows (`which miri` produces a filename that does not exist?!?).
+    # RUSTC is the main thing to set (it changes the first argument our wrapper will see).
+    # Unless MIRI is also set, that produces a warning.
+    export RUSTC=$(which rustc)
     export MIRI=$(rustc +miri --print sysroot)/bin/miri
+    # We entirely ignore other wrappers.
+    mkdir -p .cargo
+    echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml
   fi
-  mkdir -p .cargo
-  echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml
   # Run the actual test
-  ${PYTHON} test-cargo-miri/run-test.py
+  time ${PYTHON} test-cargo-miri/run-test.py
   # Clean up
   unset RUSTC MIRI
   rm -rf .cargo
@@ -109,7 +110,7 @@ function run_tests {
 }
 
 function run_tests_minimal {
-  if [ -n "${MIRI_TEST_TARGET:-}" ]; then
+  if [ -n "${MIRI_TEST_TARGET-}" ]; then
     begingroup "Testing MINIMAL foreign architecture $MIRI_TEST_TARGET: only testing $@"
   else
     echo "run_tests_minimal requires MIRI_TEST_TARGET to be set"
@@ -126,38 +127,49 @@ function run_tests_minimal {
 
 ## Main Testing Logic ##
 
-# Host target.
-run_tests
-
-# Extra targets.
 # In particular, fully cover all tier 1 targets.
 case $HOST_TARGET in
   x86_64-unknown-linux-gnu)
+    # Host
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 1
     MIRI_TEST_TARGET=i686-unknown-linux-gnu run_tests
     MIRI_TEST_TARGET=aarch64-unknown-linux-gnu run_tests
-    MIRI_TEST_TARGET=aarch64-apple-darwin run_tests
+    MIRI_TEST_TARGET=x86_64-apple-darwin run_tests
     MIRI_TEST_TARGET=i686-pc-windows-gnu run_tests
     MIRI_TEST_TARGET=x86_64-pc-windows-gnu run_tests
+    # Extra tier 2
+    MIRI_TEST_TARGET=aarch64-apple-darwin run_tests
     MIRI_TEST_TARGET=arm-unknown-linux-gnueabi run_tests
-    # Some targets are only partially supported.
+    # Partially supported targets (tier 2)
     MIRI_TEST_TARGET=x86_64-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus
     MIRI_TEST_TARGET=i686-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus
-
     MIRI_TEST_TARGET=aarch64-linux-android run_tests_minimal hello integer vec panic/panic
     MIRI_TEST_TARGET=wasm32-wasi run_tests_minimal no_std integer strings wasm
     MIRI_TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std integer strings wasm
-    MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std # no_std embedded architecture
-    MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std # JSON target file
+    MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std
+    # Custom target JSON file
+    MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std
     ;;
-  x86_64-apple-darwin)
+  aarch64-apple-darwin)
+    # Host (tier 2)
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 1
+    MIRI_TEST_TARGET=x86_64-pc-windows-msvc CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 2
     MIRI_TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture
-    MIRI_TEST_TARGET=x86_64-pc-windows-msvc run_tests
     ;;
   i686-pc-windows-msvc)
+    # Host
+    # Only smoke-test `many-seeds`; 64 runs take 15min here!
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=1 TEST_BENCH=1 run_tests
+    # Extra tier 1
+    # We really want to ensure a Linux target works on a Windows host,
+    # and a 64bit target works on a 32bit host.
     MIRI_TEST_TARGET=x86_64-unknown-linux-gnu run_tests
     ;;
   *)
-    echo "FATAL: unknown OS"
+    echo "FATAL: unknown host target: $HOST_TARGET"
     exit 1
     ;;
 esac

src/tools/miri/miri-script/src/commands.rs

@@ -297,7 +297,7 @@ impl Command {
         };
         // Prepare the branch. Pushing works much better if we use as base exactly
         // the commit that we pulled from last time, so we use the `rust-version`
-        // file as a good approximation of that.
+        // file to find out which commit that would be.
         println!("Preparing {github_user}/rust (base: {base})...");
         if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
             .ignore_stderr()

src/tools/miri/rust-version

@@ -1 +1 @@
-5baf1e13f568b61e121953bf6a3d09faee7dd446
+23d47dba319331d4418827cfbb8c1af283497d3c

src/tools/miri/src/borrow_tracker/tree_borrows/diagnostics.rs

@@ -48,7 +48,7 @@ impl AccessCause {
 /// Complete data for an event:
 #[derive(Clone, Debug)]
 pub struct Event {
-    /// Transformation of permissions that occured because of this event.
+    /// Transformation of permissions that occurred because of this event.
     pub transition: PermTransition,
     /// Kind of the access that triggered this event.
     pub access_cause: AccessCause,
@@ -58,7 +58,7 @@ pub struct Event {
     /// `None` means that this is an implicit access to the entire allocation
     /// (used for the implicit read on protector release).
     pub access_range: Option<AllocRange>,
-    /// The transition recorded by this event only occured on a subrange of
+    /// The transition recorded by this event only occurred on a subrange of
     /// `access_range`: a single access on `access_range` triggers several events,
     /// each with their own mutually disjoint `transition_range`. No-op transitions
     /// should not be recorded as events, so the union of all `transition_range` is not

src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs

@@ -473,7 +473,7 @@ impl Tree {
         let rperms = {
             let mut perms = UniValMap::default();
             // We manually set it to `Active` on all in-bounds positions.
-            // We also ensure that it is initalized, so that no `Active` but
+            // We also ensure that it is initialized, so that no `Active` but
             // not yet initialized nodes exist. Essentially, we pretend there
             // was a write that initialized these to `Active`.
             perms.insert(root_idx, LocationState::new_init(Permission::new_active()));