From 29870051575bb9ddf45a1cfba3c8e0bd06f52f40 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 4 Jul 2024 09:11:28 +0200 Subject: [PATCH] Simple Red-Knot benchmark --- Cargo.lock | 3 + Cargo.toml | 1 + crates/red_knot/src/program/check.rs | 8 +- crates/ruff_benchmark/Cargo.toml | 7 ++ crates/ruff_benchmark/benches/red_knot.rs | 143 ++++++++++++++++++++++ 5 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 crates/ruff_benchmark/benches/red_knot.rs diff --git a/Cargo.lock b/Cargo.lock index 82908741fd6234..464b8cde5d968a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2053,6 +2053,9 @@ dependencies = [ "criterion", "mimalloc", "once_cell", + "red_knot", + "red_knot_module_resolver", + "ruff_db", "ruff_linter", "ruff_python_ast", "ruff_python_formatter", diff --git a/Cargo.toml b/Cargo.toml index a563af269b580d..bfc8d351dca776 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ ruff_source_file = { path = "crates/ruff_source_file" } ruff_text_size = { path = "crates/ruff_text_size" } ruff_workspace = { path = "crates/ruff_workspace" } +red_knot = { path = "crates/red_knot" } red_knot_module_resolver = { path = "crates/red_knot_module_resolver" } red_knot_python_semantic = { path = "crates/red_knot_python_semantic" } diff --git a/crates/red_knot/src/program/check.rs b/crates/red_knot/src/program/check.rs index 22633ad9a3ebd3..8fe0d58f5fe4bb 100644 --- a/crates/red_knot/src/program/check.rs +++ b/crates/red_knot/src/program/check.rs @@ -11,7 +11,7 @@ impl Program { self.with_db(|db| { let mut result = Vec::new(); for open_file in db.workspace.open_files() { - result.extend_from_slice(&db.check_file(open_file)); + result.extend_from_slice(&db.check_file_impl(open_file)); } result @@ -19,7 +19,11 @@ impl Program { } #[tracing::instrument(level = "debug", skip(self))] - fn check_file(&self, file: VfsFile) -> Diagnostics { + pub fn check_file(&self, file: VfsFile) -> Result { + self.with_db(|db| db.check_file_impl(file)) + } + + fn check_file_impl(&self, file: VfsFile) -> Diagnostics { let mut diagnostics = Vec::new(); diagnostics.extend_from_slice(lint_syntax(self, file)); diagnostics.extend_from_slice(lint_semantic(self, file)); diff --git a/crates/ruff_benchmark/Cargo.toml b/crates/ruff_benchmark/Cargo.toml index c95caf0d13bfdb..a2fe36f3188736 100644 --- a/crates/ruff_benchmark/Cargo.toml +++ b/crates/ruff_benchmark/Cargo.toml @@ -31,6 +31,10 @@ harness = false name = "formatter" harness = false +[[bench]] +name = "red_knot" +harness = false + [dependencies] once_cell = { workspace = true } serde = { workspace = true } @@ -41,11 +45,14 @@ criterion = { workspace = true, default-features = false } codspeed-criterion-compat = { workspace = true, default-features = false, optional = true } [dev-dependencies] +ruff_db = { workspace = true } ruff_linter = { workspace = true } ruff_python_ast = { workspace = true } ruff_python_formatter = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_trivia = { workspace = true } +red_knot = { workspace = true } +red_knot_module_resolver = { workspace = true } [lints] workspace = true diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs new file mode 100644 index 00000000000000..f64a48d5e75e8f --- /dev/null +++ b/crates/ruff_benchmark/benches/red_knot.rs @@ -0,0 +1,143 @@ +#![allow(clippy::disallowed_names)] + +use criterion::BenchmarkId; +use red_knot::program::Program; +use red_knot::Workspace; +use red_knot_module_resolver::{set_module_resolution_settings, ModuleResolutionSettings}; +use ruff_benchmark::criterion::{ + criterion_group, criterion_main, BatchSize, Criterion, Throughput, +}; +use ruff_db::file_system::{FileSystemPath, MemoryFileSystem}; +use ruff_db::parsed::parsed_module; +use ruff_db::vfs::{system_path_to_file, VfsFile}; +use ruff_db::Upcast; + +static FOO_CODE: &str = r#" +import typing + +from bar import Bar + +class Foo(Bar): + def foo() -> str: + return "foo" + + @typing.override + def bar() -> str: + return "foo_bar" +"#; + +static BAR_CODE: &str = r#" +class Bar: + def bar() -> str: + return "bar" + + def random(arg: int) -> int: + if arg == 1: + return 48472783 + if arg < 10: + return 20 + return 36673 +"#; + +static TYPING_CODE: &str = r#" +def override(): ... +"#; + +struct Case { + program: Program, + foo: VfsFile, + bar: VfsFile, + typing: VfsFile, +} + +fn setup_case() -> Case { + let fs = MemoryFileSystem::new(); + let foo_path = FileSystemPath::new("/src/foo.py"); + let bar_path = FileSystemPath::new("/src/bar.py"); + let typing_path = FileSystemPath::new("/src/typing.pyi"); + fs.write_files([ + (foo_path, FOO_CODE), + (bar_path, BAR_CODE), + (typing_path, TYPING_CODE), + ]) + .unwrap(); + + let workspace_root = FileSystemPath::new("/src"); + let workspace = Workspace::new(workspace_root.to_path_buf()); + + let mut program = Program::new(workspace, fs); + let foo = system_path_to_file(&program, foo_path).unwrap(); + + set_module_resolution_settings( + &mut program, + ModuleResolutionSettings { + extra_paths: vec![], + workspace_root: workspace_root.to_path_buf(), + site_packages: None, + custom_typeshed: None, + }, + ); + + program.workspace_mut().open_file(foo); + + let bar = system_path_to_file(&program, bar_path).unwrap(); + let typing = system_path_to_file(&program, typing_path).unwrap(); + + Case { + program, + foo, + bar, + typing, + } +} + +fn benchmark_warm(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("red_knot/check_file"); + group.throughput(Throughput::Bytes(FOO_CODE.len() as u64)); + + group.bench_function(BenchmarkId::from_parameter("warm"), |b| { + b.iter_batched( + || { + let case = setup_case(); + // Pre-parse the module to only measure the semantic time. + parsed_module(case.program.upcast(), case.foo); + parsed_module(case.program.upcast(), case.bar); + parsed_module(case.program.upcast(), case.typing); + case + }, + |case| { + let Case { program, foo, .. } = case; + let result = program.check_file(foo).unwrap(); + + assert_eq!(result.as_slice(), [] as [String; 0]); + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn benchmark_cold(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("red_knot/check_file"); + group.throughput(Throughput::Bytes(FOO_CODE.len() as u64)); + + group.bench_function(BenchmarkId::from_parameter("cold"), |b| { + b.iter_batched( + setup_case, + |case| { + let Case { program, foo, .. } = case; + let result = program.check_file(foo).unwrap(); + + assert_eq!(result.as_slice(), [] as [String; 0]); + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +criterion_group!(cold, benchmark_warm); +criterion_group!(warm, benchmark_cold); +criterion_main!(warm, cold);