Maintain synchronicity between the lexer and the parser

crates/red_knot/src/parse.rs

@@ -32,17 +32,19 @@ impl Parsed {
         let result = ruff_python_parser::parse(text, Mode::Module);
 
         let (module, errors) = match result {
-            Ok(ast::Mod::Module(module)) => (module, vec![]),
-            Ok(ast::Mod::Expression(expression)) => (
-                ast::ModModule {
-                    range: expression.range(),
-                    body: vec![ast::Stmt::Expr(ast::StmtExpr {
+            Ok(parsed) => match parsed.into_syntax() {
+                ast::Mod::Module(module) => (module, vec![]),
+                ast::Mod::Expression(expression) => (
+                    ast::ModModule {
                         range: expression.range(),
-                        value: expression.body,
-                    })],
-                },
-                vec![],
-            ),
+                        body: vec![ast::Stmt::Expr(ast::StmtExpr {
+                            range: expression.range(),
+                            value: expression.body,
+                        })],
+                    },
+                    vec![],
+                ),
+            },
             Err(errors) => (
                 ast::ModModule {
                     range: TextRange::default(),

crates/ruff_benchmark/Cargo.toml

@@ -44,7 +44,6 @@ codspeed-criterion-compat = { workspace = true, default-features = false, option
 ruff_linter = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_formatter = { workspace = true }
-ruff_python_index = { workspace = true }
 ruff_python_parser = { workspace = true }
 
 [lints]

crates/ruff_benchmark/benches/formatter.rs

@@ -5,9 +5,7 @@ use ruff_benchmark::criterion::{
 };
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
-use ruff_python_index::CommentRangesBuilder;
-use ruff_python_parser::lexer::lex;
-use ruff_python_parser::{allocate_tokens_vec, parse_tokens, Mode};
+use ruff_python_parser::{parse, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -52,28 +50,15 @@ fn benchmark_formatter(criterion: &mut Criterion) {
             BenchmarkId::from_parameter(case.name()),
             &case,
             |b, case| {
-                let mut tokens = allocate_tokens_vec(case.code());
-                let mut comment_ranges = CommentRangesBuilder::default();
-
-                for result in lex(case.code(), Mode::Module) {
-                    let (token, range) = result.expect("Input to be a valid python program.");
-
-                    comment_ranges.visit_token(&token, range);
-                    tokens.push(Ok((token, range)));
-                }
-
-                let comment_ranges = comment_ranges.finish();
-
-                // Parse the AST.
-                let module = parse_tokens(tokens, case.code(), Mode::Module)
-                    .expect("Input to be a valid python program");
+                // Parse the source.
+                let parsed =
+                    parse(case.code(), Mode::Module).expect("Input should be a valid Python code");
 
                 b.iter(|| {
                     let options = PyFormatOptions::from_extension(Path::new(case.name()))
                         .with_preview(PreviewMode::Enabled);
-                    let formatted =
-                        format_module_ast(&module, &comment_ranges, case.code(), options)
-                            .expect("Formatting to succeed");
+                    let formatted = format_module_ast(&parsed, case.code(), options)
+                        .expect("Formatting to succeed");
 
                     formatted.print().expect("Printing to succeed")
                 });

crates/ruff_benchmark/benches/lexer.rs

@@ -2,7 +2,7 @@ use ruff_benchmark::criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
-use ruff_python_parser::{lexer, Mode};
+use ruff_python_parser::{lexer, Mode, TokenKind};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -47,9 +47,15 @@ fn benchmark_lexer(criterion: &mut Criterion<WallTime>) {
             &case,
             |b, case| {
                 b.iter(|| {
-                    let result =
-                        lexer::lex(case.code(), Mode::Module).find(std::result::Result::is_err);
-                    assert_eq!(result, None, "Input to be a valid Python program");
+                    let mut lexer = lexer::lex(case.code(), Mode::Module);
+                    loop {
+                        let token = lexer.next_token();
+                        match token {
+                            TokenKind::EndOfFile => break,
+                            TokenKind::Unknown => panic!("Input to be a valid Python source code"),
+                            _ => {}
+                        }
+                    }
                 });
             },
         );

crates/ruff_benchmark/benches/linter.rs

@@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings};
 use ruff_linter::source_kind::SourceKind;
 use ruff_linter::{registry::Rule, RuleSelector};
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{parse_program_tokens, tokenize, Mode};
+use ruff_python_parser::parse_module;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -54,15 +54,13 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
             BenchmarkId::from_parameter(case.name()),
             &case,
             |b, case| {
-                // Tokenize the source.
-                let tokens = tokenize(case.code(), Mode::Module);
-
                 // Parse the source.
-                let ast = parse_program_tokens(tokens.clone(), case.code(), false).unwrap();
+                let parsed =
+                    parse_module(case.code()).expect("Input should be a valid Python code");
 
                 b.iter_batched(
-                    || (ast.clone(), tokens.clone()),
-                    |(ast, tokens)| {
+                    || parsed.clone(),
+                    |parsed| {
                         let path = case.path();
                         let result = lint_only(
                             &path,
@@ -71,7 +69,7 @@ fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
                             flags::Noqa::Enabled,
                             &SourceKind::Python(case.code().to_string()),
                             PySourceType::from(path.as_path()),
-                            ParseSource::Precomputed { tokens, ast },
+                            ParseSource::Precomputed(parsed),
                         );
 
                         // Assert that file contains no parse errors

crates/ruff_benchmark/benches/parser.rs

@@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;
-use ruff_python_parser::parse_suite;
+use ruff_python_parser::parse_module;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -60,7 +60,9 @@ fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
             &case,
             |b, case| {
                 b.iter(|| {
-                    let parsed = parse_suite(case.code()).unwrap();
+                    let parsed = parse_module(case.code())
+                        .expect("Input should be a valid Python code")
+                        .into_suite();
 
                     let mut visitor = CountVisitor { count: 0 };
                     visitor.visit_body(&parsed);

crates/ruff_dev/Cargo.toml

@@ -22,6 +22,7 @@ ruff_python_formatter = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_stdlib = { workspace = true }
 ruff_python_trivia = { workspace = true }
+ruff_text_size = { workspace = true }
 ruff_workspace = { workspace = true, features = ["schemars"] }
 
 anyhow = { workspace = true }

crates/ruff_dev/src/print_ast.rs

@@ -24,7 +24,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
             args.file.display()
         )
     })?;
-    let python_ast = parse(source_kind.source_code(), source_type.as_mode())?;
+    let python_ast = parse(source_kind.source_code(), source_type.as_mode())?.into_syntax();
     println!("{python_ast:#?}");
     Ok(())
 }

crates/ruff_dev/src/print_tokens.rs

@@ -7,7 +7,8 @@ use anyhow::Result;
 
 use ruff_linter::source_kind::SourceKind;
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{lexer, AsMode};
+use ruff_python_parser::parse_unchecked_source;
+use ruff_text_size::Ranged;
 
 #[derive(clap::Args)]
 pub(crate) struct Args {
@@ -24,11 +25,13 @@ pub(crate) fn main(args: &Args) -> Result<()> {
             args.file.display()
         )
     })?;
-    for (tok, range) in lexer::lex(source_kind.source_code(), source_type.as_mode()).flatten() {
+    let parsed = parse_unchecked_source(source_kind.source_code(), source_type);
+    for token in parsed.tokens() {
         println!(
-            "{start:#?} {tok:#?} {end:#?}",
-            start = range.start(),
-            end = range.end()
+            "{start:#?} {kind:#?} {end:#?}",
+            start = token.start(),
+            end = token.end(),
+            kind = token.kind(),
         );
     }
     Ok(())

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -1160,7 +1160,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                     }
                 }
                 if checker.enabled(Rule::PrintfStringFormatting) {
-                    pyupgrade::rules::printf_string_formatting(checker, expr, right);
+                    pyupgrade::rules::printf_string_formatting(checker, bin_op, format_string);
                 }
                 if checker.enabled(Rule::BadStringFormatCharacter) {
                     pylint::rules::bad_string_format_character::percent(

crates/ruff_linter/src/checkers/ast/analyze/statement.rs

@@ -765,7 +765,7 @@ pub(crate) fn statement(stmt: &Stmt, checker: &mut Checker) {
                 pyupgrade::rules::deprecated_c_element_tree(checker, stmt);
             }
             if checker.enabled(Rule::DeprecatedImport) {
-                pyupgrade::rules::deprecated_import(checker, stmt, names, module, level);
+                pyupgrade::rules::deprecated_import(checker, import_from);
             }
             if checker.enabled(Rule::UnnecessaryBuiltinImport) {
                 if let Some(module) = module {

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -32,8 +32,10 @@ use itertools::Itertools;
 use log::debug;
 use ruff_python_ast::{
     self as ast, AnyParameterRef, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext,
-    FStringElement, Keyword, MatchCase, Parameter, Parameters, Pattern, Stmt, Suite, UnaryOp,
+    FStringElement, Keyword, MatchCase, ModModule, Parameter, Parameters, Pattern, Stmt, Suite,
+    UnaryOp,
 };
+use ruff_python_parser::Parsed;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use ruff_diagnostics::{Diagnostic, IsolationLevel};
@@ -174,6 +176,8 @@ impl ExpectedDocstringKind {
 }
 
 pub(crate) struct Checker<'a> {
+    /// The parsed [`Parsed`].
+    parsed: &'a Parsed<ModModule>,
     /// The [`Path`] to the file under analysis.
     path: &'a Path,
     /// The [`Path`] to the package containing the current file.
@@ -223,6 +227,7 @@ pub(crate) struct Checker<'a> {
 impl<'a> Checker<'a> {
     #[allow(clippy::too_many_arguments)]
     pub(crate) fn new(
+        parsed: &'a Parsed<ModModule>,
         settings: &'a LinterSettings,
         noqa_line_for: &'a NoqaMapping,
         noqa: flags::Noqa,
@@ -232,12 +237,12 @@ impl<'a> Checker<'a> {
         locator: &'a Locator,
         stylist: &'a Stylist,
         indexer: &'a Indexer,
-        importer: Importer<'a>,
         source_type: PySourceType,
         cell_offsets: Option<&'a CellOffsets>,
         notebook_index: Option<&'a NotebookIndex>,
     ) -> Checker<'a> {
         Checker {
+            parsed,
             settings,
             noqa_line_for,
             noqa,
@@ -248,7 +253,7 @@ impl<'a> Checker<'a> {
             locator,
             stylist,
             indexer,
-            importer,
+            importer: Importer::new(parsed, locator, stylist),
             semantic: SemanticModel::new(&settings.typing_modules, path, module),
             visit: deferred::Visit::default(),
             analyze: deferred::Analyze::default(),
@@ -318,6 +323,11 @@ impl<'a> Checker<'a> {
         }
     }
 
+    /// The [`Parsed`] output for the current file, which contains the tokens, AST, and more.
+    pub(crate) const fn parsed(&self) -> &'a Parsed<ModModule> {
+        self.parsed
+    }
+
     /// The [`Locator`] for the current file, which enables extraction of source code from byte
     /// offsets.
     pub(crate) const fn locator(&self) -> &'a Locator<'a> {
@@ -2326,7 +2336,7 @@ impl<'a> Checker<'a> {
 
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn check_ast(
-    python_ast: &Suite,
+    parsed: &Parsed<ModModule>,
     locator: &Locator,
     stylist: &Stylist,
     indexer: &Indexer,
@@ -2356,10 +2366,11 @@ pub(crate) fn check_ast(
         } else {
             ModuleSource::File(path)
         },
-        python_ast,
+        python_ast: parsed.suite(),
     };
 
     let mut checker = Checker::new(
+        parsed,
         settings,
         noqa_line_for,
         noqa,
@@ -2369,16 +2380,15 @@ pub(crate) fn check_ast(
         locator,
         stylist,
         indexer,
-        Importer::new(python_ast, locator, stylist),
         source_type,
         cell_offsets,
         notebook_index,
     );
     checker.bind_builtins();
 
     // Iterate over the AST.
-    checker.visit_module(python_ast);
-    checker.visit_body(python_ast);
+    checker.visit_module(parsed.suite());
+    checker.visit_body(parsed.suite());
 
     // Visit any deferred syntax nodes. Take care to visit in order, such that we avoid adding
     // new deferred nodes after visiting nodes of that kind. For example, visiting a deferred