Fix CI

* Run `cargo shear --fix`
* Use transformed source code
* Fix fuzzer to use the new parser API

crates/ruff_benchmark/Cargo.toml

@@ -44,7 +44,6 @@ codspeed-criterion-compat = { workspace = true, default-features = false, option
 ruff_linter = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_formatter = { workspace = true }
-ruff_python_index = { workspace = true }
 ruff_python_parser = { workspace = true }
 
 [lints]

crates/ruff_linter/src/linter.rs

@@ -538,7 +538,7 @@ pub fn lint_fix<'a>(
     loop {
         // Parse once.
         let program =
-            ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type);
+            ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type);
 
         // Map row and column locations to byte slices (lazily).
         let locator = Locator::new(transformed.source_code());

crates/ruff_python_formatter/Cargo.toml

@@ -20,7 +20,6 @@ ruff_macros = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_python_ast = { workspace = true }
-ruff_python_index = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_text_size = { workspace = true }
 

crates/ruff_python_trivia_integration_tests/Cargo.toml

@@ -12,7 +12,6 @@ license.workspace = true
 [dependencies]
 
 [dev-dependencies]
-ruff_python_index = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true }

crates/ruff_wasm/Cargo.toml

@@ -28,7 +28,6 @@ ruff_python_index = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
-ruff_python_trivia = { workspace = true }
 ruff_workspace = { workspace = true }
 
 console_error_panic_hook = { workspace = true, optional = true }

fuzz/Cargo.toml

@@ -24,6 +24,7 @@ ruff_python_index = { path = "../crates/ruff_python_index" }
 ruff_python_parser = { path = "../crates/ruff_python_parser" }
 ruff_source_file = { path = "../crates/ruff_source_file" }
 ruff_python_formatter = { path = "../crates/ruff_python_formatter"}
+ruff_text_size = { path = "../crates/ruff_text_size" }
 
 arbitrary = { version = "1.3.0", features = ["derive"] }
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer", default-features = false }

fuzz/fuzz_targets/ruff_parse_simple.rs

@@ -5,8 +5,9 @@
 
 use libfuzzer_sys::{fuzz_target, Corpus};
 use ruff_python_codegen::{Generator, Stylist};
-use ruff_python_parser::{lexer, parse_suite, Mode, ParseError};
+use ruff_python_parser::{parse_module, ParseError};
 use ruff_source_file::Locator;
+use ruff_text_size::Ranged;
 
 fn do_fuzz(case: &[u8]) -> Corpus {
     let Ok(code) = std::str::from_utf8(case) else {
@@ -15,8 +16,8 @@ fn do_fuzz(case: &[u8]) -> Corpus {
 
     // just round-trip it once to trigger both parse and unparse
     let locator = Locator::new(code);
-    let python_ast = match parse_suite(code) {
-        Ok(stmts) => stmts,
+    let program = match parse_module(code) {
+        Ok(program) => program,
         Err(ParseError { location, .. }) => {
             let offset = location.start().to_usize();
             assert!(
@@ -28,38 +29,24 @@ fn do_fuzz(case: &[u8]) -> Corpus {
         }
     };
 
-    let tokens: Vec<_> = lexer::lex(code, Mode::Module).collect();
-
-    for maybe_token in tokens.iter() {
-        match maybe_token.as_ref() {
-            Ok((_, range)) => {
-                let start = range.start().to_usize();
-                let end = range.end().to_usize();
-                assert!(
-                    code.is_char_boundary(start),
-                    "Invalid start position {} (not at char boundary)",
-                    start
-                );
-                assert!(
-                    code.is_char_boundary(end),
-                    "Invalid end position {} (not at char boundary)",
-                    end
-                );
-            }
-            Err(err) => {
-                let offset = err.location().start().to_usize();
-                assert!(
-                    code.is_char_boundary(offset),
-                    "Invalid error location {} (not at char boundary)",
-                    offset
-                );
-            }
-        }
+    for token in program.tokens() {
+        let start = token.start().to_usize();
+        let end = token.end().to_usize();
+        assert!(
+            code.is_char_boundary(start),
+            "Invalid start position {} (not at char boundary)",
+            start
+        );
+        assert!(
+            code.is_char_boundary(end),
+            "Invalid end position {} (not at char boundary)",
+            end
+        );
     }
 
-    let stylist = Stylist::from_tokens(&tokens, &locator);
+    let stylist = Stylist::from_tokens(program.tokens(), &locator);
     let mut generator: Generator = (&stylist).into();
-    generator.unparse_suite(&python_ast);
+    generator.unparse_suite(program.suite());
 
     Corpus::Keep
 }