From 7b9595a5ab2c6eb455ea10f3708c9fb97a430978 Mon Sep 17 00:00:00 2001 From: hkalbasi Date: Fri, 15 Dec 2023 02:10:25 +0330 Subject: [PATCH] Run rust-analyzer on rustc tests in metrics --- .github/workflows/metrics.yaml | 9 +- Cargo.lock | 1 + crates/ide-diagnostics/src/lib.rs | 4 +- crates/ide/src/lib.rs | 4 +- crates/rust-analyzer/Cargo.toml | 1 + crates/rust-analyzer/src/bin/main.rs | 1 + crates/rust-analyzer/src/cli.rs | 1 + crates/rust-analyzer/src/cli/flags.rs | 16 ++ crates/rust-analyzer/src/cli/rustc_tests.rs | 236 ++++++++++++++++++++ xtask/src/flags.rs | 3 + xtask/src/metrics.rs | 17 ++ 11 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 crates/rust-analyzer/src/cli/rustc_tests.rs diff --git a/.github/workflows/metrics.yaml b/.github/workflows/metrics.yaml index 741e559953fc8..e6a9917a0bf3d 100644 --- a/.github/workflows/metrics.yaml +++ b/.github/workflows/metrics.yaml @@ -67,7 +67,7 @@ jobs: other_metrics: strategy: matrix: - names: [self, ripgrep-13.0.0, webrender-2022, diesel-1.4.8, hyper-0.14.18] + names: [self, rustc_tests, ripgrep-13.0.0, webrender-2022, diesel-1.4.8, hyper-0.14.18] runs-on: ubuntu-latest needs: [setup_cargo, build_metrics] @@ -118,6 +118,11 @@ jobs: with: name: self-${{ github.sha }} + - name: Download rustc_tests metrics + uses: actions/download-artifact@v3 + with: + name: rustc_tests-${{ github.sha }} + - name: Download ripgrep-13.0.0 metrics uses: actions/download-artifact@v3 with: @@ -146,7 +151,7 @@ jobs: chmod 700 ~/.ssh git clone --depth 1 git@github.com:rust-analyzer/metrics.git - jq -s ".[0] * .[1] * .[2] * .[3] * .[4] * .[5]" build.json self.json ripgrep-13.0.0.json webrender-2022.json diesel-1.4.8.json hyper-0.14.18.json -c >> metrics/metrics.json + jq -s ".[0] * .[1] * .[2] * .[3] * .[4] * .[5] * .[6]" build.json self.json rustc_tests.json ripgrep-13.0.0.json webrender-2022.json diesel-1.4.8.json hyper-0.14.18.json -c >> metrics/metrics.json cd metrics git add . git -c user.name=Bot -c user.email=dummy@example.com commit --message 📈 diff --git a/Cargo.lock b/Cargo.lock index f94b855ca7d18..227d1db0ec72d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1545,6 +1545,7 @@ dependencies = [ "triomphe", "vfs", "vfs-notify", + "walkdir", "winapi", "xflags", "xshell", diff --git a/crates/ide-diagnostics/src/lib.rs b/crates/ide-diagnostics/src/lib.rs index 6541bf605794a..579386c72ef4d 100644 --- a/crates/ide-diagnostics/src/lib.rs +++ b/crates/ide-diagnostics/src/lib.rs @@ -94,7 +94,7 @@ use syntax::{ }; // FIXME: Make this an enum -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DiagnosticCode { RustcHardError(&'static str), RustcLint(&'static str), @@ -198,7 +198,7 @@ impl Diagnostic { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub enum Severity { Error, Warning, diff --git a/crates/ide/src/lib.rs b/crates/ide/src/lib.rs index e3548f3f0cbf4..a19952e4cae97 100644 --- a/crates/ide/src/lib.rs +++ b/crates/ide/src/lib.rs @@ -133,7 +133,9 @@ pub use ide_db::{ symbol_index::Query, RootDatabase, SymbolKind, }; -pub use ide_diagnostics::{Diagnostic, DiagnosticsConfig, ExprFillDefaultMode, Severity}; +pub use ide_diagnostics::{ + Diagnostic, DiagnosticCode, DiagnosticsConfig, ExprFillDefaultMode, Severity, +}; pub use ide_ssr::SsrError; pub use syntax::{TextRange, TextSize}; pub use text_edit::{Indel, TextEdit}; diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 408c1fb6f39b7..39ac338aa1a92 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -42,6 +42,7 @@ tracing-tree.workspace = true triomphe.workspace = true nohash-hasher.workspace = true always-assert = "0.1.2" +walkdir = "2.3.2" cfg.workspace = true flycheck.workspace = true diff --git a/crates/rust-analyzer/src/bin/main.rs b/crates/rust-analyzer/src/bin/main.rs index 29bd02f92da70..8472e49de9838 100644 --- a/crates/rust-analyzer/src/bin/main.rs +++ b/crates/rust-analyzer/src/bin/main.rs @@ -87,6 +87,7 @@ fn main() -> anyhow::Result<()> { flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?, flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?, flags::RustAnalyzerCmd::RunTests(cmd) => cmd.run()?, + flags::RustAnalyzerCmd::RustcTests(cmd) => cmd.run()?, } Ok(()) } diff --git a/crates/rust-analyzer/src/cli.rs b/crates/rust-analyzer/src/cli.rs index 64646b33ad480..de00c4192b46a 100644 --- a/crates/rust-analyzer/src/cli.rs +++ b/crates/rust-analyzer/src/cli.rs @@ -10,6 +10,7 @@ mod ssr; mod lsif; mod scip; mod run_tests; +mod rustc_tests; mod progress_report; diff --git a/crates/rust-analyzer/src/cli/flags.rs b/crates/rust-analyzer/src/cli/flags.rs index fe5022f8606d1..5633c0c488aa7 100644 --- a/crates/rust-analyzer/src/cli/flags.rs +++ b/crates/rust-analyzer/src/cli/flags.rs @@ -98,6 +98,15 @@ xflags::xflags! { required path: PathBuf } + /// Run unit tests of the project using mir interpreter + cmd rustc-tests { + /// Directory with Cargo.toml. + required rustc_repo: PathBuf + + /// Only run tests with filter as substring + optional --filter path: String + } + cmd diagnostics { /// Directory with Cargo.toml. required path: PathBuf @@ -159,6 +168,7 @@ pub enum RustAnalyzerCmd { Highlight(Highlight), AnalysisStats(AnalysisStats), RunTests(RunTests), + RustcTests(RustcTests), Diagnostics(Diagnostics), Ssr(Ssr), Search(Search), @@ -211,6 +221,12 @@ pub struct RunTests { pub path: PathBuf, } +#[derive(Debug)] +pub struct RustcTests { + pub rustc_repo: PathBuf, + pub filter: Option, +} + #[derive(Debug)] pub struct Diagnostics { pub path: PathBuf, diff --git a/crates/rust-analyzer/src/cli/rustc_tests.rs b/crates/rust-analyzer/src/cli/rustc_tests.rs new file mode 100644 index 0000000000000..c89b88ac0f9e6 --- /dev/null +++ b/crates/rust-analyzer/src/cli/rustc_tests.rs @@ -0,0 +1,236 @@ +//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter. + +use std::{ + cell::RefCell, collections::HashMap, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf, +}; + +use hir::Crate; +use ide::{AnalysisHost, Change, DiagnosticCode, DiagnosticsConfig}; +use profile::StopWatch; +use project_model::{CargoConfig, ProjectWorkspace, RustLibSource, Sysroot}; + +use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}; +use triomphe::Arc; +use vfs::{AbsPathBuf, FileId}; +use walkdir::WalkDir; + +use crate::cli::{flags, report_metric, Result}; + +struct Tester { + host: AnalysisHost, + root_file: FileId, + pass_count: u64, + ignore_count: u64, + fail_count: u64, + stopwatch: StopWatch, +} + +fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode { + thread_local! { + static LEAK_STORE: RefCell> = RefCell::new(HashMap::new()); + } + LEAK_STORE.with_borrow_mut(|s| match s.get(code) { + Some(c) => *c, + None => { + let v = DiagnosticCode::RustcHardError(format!("E{code}").leak()); + s.insert(code.to_owned(), v); + v + } + }) +} + +fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> HashMap { + let text = read_to_string(p).unwrap(); + let mut result = HashMap::new(); + { + let mut text = &*text; + while let Some(p) = text.find("error[E") { + text = &text[p + 7..]; + let code = string_to_diagnostic_code_leaky(&text[..4]); + *result.entry(code).or_insert(0) += 1; + } + } + result +} + +impl Tester { + fn new() -> Result { + let tmp_file = AbsPathBuf::assert("/tmp/ra-rustc-test.rs".into()); + std::fs::write(&tmp_file, "")?; + let mut cargo_config = CargoConfig::default(); + cargo_config.sysroot = Some(RustLibSource::Discover); + let workspace = ProjectWorkspace::DetachedFiles { + files: vec![tmp_file.clone()], + sysroot: Ok( + Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env).unwrap() + ), + rustc_cfg: vec![], + }; + let load_cargo_config = LoadCargoConfig { + load_out_dirs_from_check: false, + with_proc_macro_server: ProcMacroServerChoice::Sysroot, + prefill_caches: false, + }; + let (host, _vfs, _proc_macro) = + load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?; + let db = host.raw_database(); + let krates = Crate::all(db); + let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap(); + let root_file = root_crate.root_file(db); + Ok(Self { + host, + root_file, + pass_count: 0, + ignore_count: 0, + fail_count: 0, + stopwatch: StopWatch::start(), + }) + } + + fn test(&mut self, p: PathBuf) { + if p.parent().unwrap().file_name().unwrap() == "auxiliary" { + // These are not tests + return; + } + if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) { + println!("{p:?} IGNORE"); + self.ignore_count += 1; + return; + } + let stderr_path = p.with_extension("stderr"); + let expected = if stderr_path.exists() { + detect_errors_from_rustc_stderr_file(stderr_path) + } else { + HashMap::new() + }; + let text = read_to_string(&p).unwrap(); + let mut change = Change::new(); + // Ignore unstable tests, since they move too fast and we do not intend to support all of them. + let mut ignore_test = text.contains("#![feature"); + // Ignore test with extern crates, as this infra don't support them yet. + ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:"); + // Ignore test with extern modules similarly. + ignore_test |= text.contains("mod "); + // These should work, but they don't, and I don't know why, so ignore them. + ignore_test |= text.contains("extern crate proc_macro"); + let should_have_no_error = text.contains("// check-pass") + || text.contains("// build-pass") + || text.contains("// run-pass"); + change.change_file(self.root_file, Some(Arc::from(text))); + self.host.apply_change(change); + let diagnostic_config = DiagnosticsConfig::test_sample(); + let diags = self + .host + .analysis() + .diagnostics(&diagnostic_config, ide::AssistResolveStrategy::None, self.root_file) + .unwrap(); + let mut actual = HashMap::new(); + for diag in diags { + if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) { + continue; + } + if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) { + continue; + } + *actual.entry(diag.code).or_insert(0) += 1; + } + // Ignore tests with diagnostics that we don't emit. + ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k)); + if ignore_test { + println!("{p:?} IGNORE"); + self.ignore_count += 1; + } else if actual == expected { + println!("{p:?} PASS"); + self.pass_count += 1; + } else { + println!("{p:?} FAIL"); + println!("actual (r-a) = {:?}", actual); + println!("expected (rustc) = {:?}", expected); + self.fail_count += 1; + } + } + + fn report(&mut self) { + println!( + "Pass count = {}, Fail count = {}, Ignore count = {}", + self.pass_count, self.fail_count, self.ignore_count + ); + println!("Testing time and memory = {}", self.stopwatch.elapsed()); + report_metric("rustc failed tests", self.fail_count, "#"); + report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms"); + } +} + +/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them. +const IGNORED_TESTS: &[&str] = &[ + "trait-with-missing-associated-type-restriction.rs", // #15646 + "trait-with-missing-associated-type-restriction-fixable.rs", // #15646 + "resolve-self-in-impl.rs", + "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs + "issue-26056.rs", + "float-field.rs", + "invalid_operator_trait.rs", + "type-alias-impl-trait-assoc-dyn.rs", + "deeply-nested_closures.rs", // exponential time + "hang-on-deeply-nested-dyn.rs", // exponential time + "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0 + "issue-16098.rs", // Huge recursion limit for macros? + "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0 +]; + +const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[ + DiagnosticCode::RustcHardError("E0023"), + DiagnosticCode::RustcHardError("E0046"), + DiagnosticCode::RustcHardError("E0063"), + DiagnosticCode::RustcHardError("E0107"), + DiagnosticCode::RustcHardError("E0117"), + DiagnosticCode::RustcHardError("E0133"), + DiagnosticCode::RustcHardError("E0210"), + DiagnosticCode::RustcHardError("E0268"), + DiagnosticCode::RustcHardError("E0308"), + DiagnosticCode::RustcHardError("E0384"), + DiagnosticCode::RustcHardError("E0407"), + DiagnosticCode::RustcHardError("E0432"), + DiagnosticCode::RustcHardError("E0451"), + DiagnosticCode::RustcHardError("E0507"), + DiagnosticCode::RustcHardError("E0583"), + DiagnosticCode::RustcHardError("E0559"), + DiagnosticCode::RustcHardError("E0616"), + DiagnosticCode::RustcHardError("E0618"), + DiagnosticCode::RustcHardError("E0624"), + DiagnosticCode::RustcHardError("E0774"), + DiagnosticCode::RustcHardError("E0767"), + DiagnosticCode::RustcHardError("E0777"), +]; + +impl flags::RustcTests { + pub fn run(self) -> Result<()> { + let mut tester = Tester::new()?; + let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui")); + for i in walk_dir { + let i = i?; + let p = i.into_path(); + if let Some(f) = &self.filter { + if !p.as_os_str().to_string_lossy().contains(f) { + continue; + } + } + if p.extension().map_or(true, |x| x != "rs") { + continue; + } + if let Err(e) = std::panic::catch_unwind({ + let tester = AssertUnwindSafe(&mut tester); + let p = p.clone(); + move || { + let tester = tester; + tester.0.test(p); + } + }) { + println!("panic detected at test {:?}", p); + std::panic::resume_unwind(e); + } + } + tester.report(); + Ok(()) + } +} diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index e52cbfca3e6fb..092ab8c593ce8 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -110,6 +110,7 @@ pub struct PublishReleaseNotes { #[derive(Debug)] pub enum MeasurementType { Build, + RustcTests, AnalyzeSelf, AnalyzeRipgrep, AnalyzeWebRender, @@ -122,6 +123,7 @@ impl FromStr for MeasurementType { fn from_str(s: &str) -> Result { match s { "build" => Ok(Self::Build), + "rustc_tests" => Ok(Self::RustcTests), "self" => Ok(Self::AnalyzeSelf), "ripgrep-13.0.0" => Ok(Self::AnalyzeRipgrep), "webrender-2022" => Ok(Self::AnalyzeWebRender), @@ -135,6 +137,7 @@ impl AsRef for MeasurementType { fn as_ref(&self) -> &str { match self { Self::Build => "build", + Self::RustcTests => "rustc_tests", Self::AnalyzeSelf => "self", Self::AnalyzeRipgrep => "ripgrep-13.0.0", Self::AnalyzeWebRender => "webrender-2022", diff --git a/xtask/src/metrics.rs b/xtask/src/metrics.rs index 59d41d8e4b842..3d28ecdb0eb2b 100644 --- a/xtask/src/metrics.rs +++ b/xtask/src/metrics.rs @@ -36,6 +36,9 @@ impl flags::Metrics { MeasurementType::Build => { metrics.measure_build(sh)?; } + MeasurementType::RustcTests => { + metrics.measure_rustc_tests(sh)?; + } MeasurementType::AnalyzeSelf => { metrics.measure_analysis_stats_self(sh)?; } @@ -50,6 +53,7 @@ impl flags::Metrics { } None => { metrics.measure_build(sh)?; + metrics.measure_rustc_tests(sh)?; metrics.measure_analysis_stats_self(sh)?; metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeRipgrep.as_ref())?; metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeWebRender.as_ref())?; @@ -78,6 +82,19 @@ impl Metrics { self.report("build", time.as_millis() as u64, "ms".into()); Ok(()) } + + fn measure_rustc_tests(&mut self, sh: &Shell) -> anyhow::Result<()> { + eprintln!("\nMeasuring rustc tests"); + + cmd!(sh, "git clone https://github.com/rust-lang/rust").run()?; + + let output = cmd!(sh, "./target/release/rust-analyzer rustc-tests ./rust").read()?; + for (metric, value, unit) in parse_metrics(&output) { + self.report(metric, value, unit.into()); + } + Ok(()) + } + fn measure_analysis_stats_self(&mut self, sh: &Shell) -> anyhow::Result<()> { self.measure_analysis_stats_path(sh, "self", ".") }