Update token-based rules to use Tokens

crates/ruff_linter/src/checkers/tokens.rs

@@ -8,7 +8,9 @@ use ruff_python_codegen::Stylist;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_index::Indexer;
+use ruff_python_parser::Tokens;
 use ruff_source_file::Locator;
+use ruff_text_size::Ranged;
 
 use crate::directives::TodoComment;
 use crate::linter::TokenSource;
@@ -22,7 +24,7 @@ use crate::settings::LinterSettings;
 
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn check_tokens(
-    tokens: &TokenSource,
+    tokens: &Tokens,
     path: &Path,
     locator: &Locator,
     indexer: &Indexer,
@@ -42,7 +44,7 @@ pub(crate) fn check_tokens(
         Rule::BlankLinesBeforeNestedDefinition,
     ]) {
         BlankLinesChecker::new(locator, stylist, settings, source_type, cell_offsets)
-            .check_lines(tokens.kinds(), &mut diagnostics);
+            .check_lines(tokens, &mut diagnostics);
     }
 
     if settings.rules.enabled(Rule::BlanketTypeIgnore) {
@@ -86,8 +88,13 @@ pub(crate) fn check_tokens(
         Rule::InvalidCharacterNul,
         Rule::InvalidCharacterZeroWidthSpace,
     ]) {
-        for (token, range) in tokens.kinds() {
-            pylint::rules::invalid_string_characters(&mut diagnostics, token, range, locator);
+        for token in tokens.up_to_first_unknown() {
+            pylint::rules::invalid_string_characters(
+                &mut diagnostics,
+                token.kind(),
+                token.range(),
+                locator,
+            );
         }
     }
 
@@ -98,7 +105,7 @@ pub(crate) fn check_tokens(
     ]) {
         pycodestyle::rules::compound_statements(
             &mut diagnostics,
-            tokens.kinds(),
+            tokens,
             locator,
             indexer,
             source_type,
@@ -112,7 +119,7 @@ pub(crate) fn check_tokens(
     ]) {
         flake8_implicit_str_concat::rules::implicit(
             &mut diagnostics,
-            tokens.kinds(),
+            tokens,
             settings,
             locator,
             indexer,
@@ -124,11 +131,11 @@ pub(crate) fn check_tokens(
         Rule::TrailingCommaOnBareTuple,
         Rule::ProhibitedTrailingComma,
     ]) {
-        flake8_commas::rules::trailing_commas(&mut diagnostics, tokens.kinds(), locator, indexer);
+        flake8_commas::rules::trailing_commas(&mut diagnostics, tokens, locator, indexer);
     }
 
     if settings.rules.enabled(Rule::ExtraneousParentheses) {
-        pyupgrade::rules::extraneous_parentheses(&mut diagnostics, tokens.kinds(), locator);
+        pyupgrade::rules::extraneous_parentheses(&mut diagnostics, tokens, locator);
     }
 
     if source_type.is_stub() && settings.rules.enabled(Rule::TypeCommentInStub) {
@@ -172,7 +179,7 @@ pub(crate) fn check_tokens(
     }
 
     if settings.rules.enabled(Rule::TooManyNewlinesAtEndOfFile) {
-        pycodestyle::rules::too_many_newlines_at_end_of_file(&mut diagnostics, tokens.kinds());
+        pycodestyle::rules::too_many_newlines_at_end_of_file(&mut diagnostics, tokens);
     }
 
     diagnostics.retain(|diagnostic| settings.rules.enabled(diagnostic.kind.rule()));

crates/ruff_linter/src/linter.rs

@@ -103,7 +103,7 @@ pub fn check_path(
         .any(|rule_code| rule_code.lint_source().is_tokens())
     {
         diagnostics.extend(check_tokens(
-            &program,
+            program.tokens(),
             path,
             locator,
             indexer,

crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs

@@ -2,7 +2,7 @@ use ruff_diagnostics::{AlwaysFixableViolation, Violation};
 use ruff_diagnostics::{Diagnostic, Edit, Fix};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, TokenKindIter};
+use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::Locator;
 use ruff_text_size::{Ranged, TextRange};
 
@@ -27,31 +27,31 @@ enum TokenType {
 
 /// Simplified token specialized for the task.
 #[derive(Copy, Clone)]
-struct Token {
+struct RuleToken {
     ty: TokenType,
     range: TextRange,
 }
 
-impl Ranged for Token {
+impl Ranged for RuleToken {
     fn range(&self) -> TextRange {
         self.range
     }
 }
 
-impl Token {
+impl RuleToken {
     fn new(ty: TokenType, range: TextRange) -> Self {
         Self { ty, range }
     }
 
-    fn irrelevant() -> Token {
-        Token {
+    fn irrelevant() -> RuleToken {
+        RuleToken {
             ty: TokenType::Irrelevant,
             range: TextRange::default(),
         }
     }
 }
 
-impl From<(TokenKind, TextRange)> for Token {
+impl From<(TokenKind, TextRange)> for RuleToken {
     fn from((tok, range): (TokenKind, TextRange)) -> Self {
         let ty = match tok {
             TokenKind::Name => TokenType::Named,
@@ -226,13 +226,13 @@ impl AlwaysFixableViolation for ProhibitedTrailingComma {
 /// COM812, COM818, COM819
 pub(crate) fn trailing_commas(
     diagnostics: &mut Vec<Diagnostic>,
-    tokens: TokenKindIter,
+    tokens: &Tokens,
     locator: &Locator,
     indexer: &Indexer,
 ) {
     let mut fstrings = 0u32;
-    let tokens = tokens.filter_map(|(token, tok_range)| {
-        match token {
+    let rule_tokens = tokens.up_to_first_unknown().iter().filter_map(|token| {
+        match token.kind() {
             // Completely ignore comments -- they just interfere with the logic.
             TokenKind::Comment => None,
             // F-strings are handled as `String` token type with the complete range
@@ -247,28 +247,28 @@ pub(crate) fn trailing_commas(
                 if fstrings == 0 {
                     indexer
                         .fstring_ranges()
-                        .outermost(tok_range.start())
-                        .map(|range| Token::new(TokenType::String, range))
+                        .outermost(token.start())
+                        .map(|range| RuleToken::new(TokenType::String, range))
                 } else {
                     None
                 }
             }
             _ => {
                 if fstrings == 0 {
-                    Some(Token::from((token, tok_range)))
+                    Some(RuleToken::from(token.as_tuple()))
                 } else {
                     None
                 }
             }
         }
     });
 
-    let mut prev = Token::irrelevant();
-    let mut prev_prev = Token::irrelevant();
+    let mut prev = RuleToken::irrelevant();
+    let mut prev_prev = RuleToken::irrelevant();
 
     let mut stack = vec![Context::new(ContextType::No)];
 
-    for token in tokens {
+    for token in rule_tokens {
         if prev.ty == TokenType::NonLogicalNewline && token.ty == TokenType::NonLogicalNewline {
             // Collapse consecutive newlines to the first one -- trailing commas are
             // added before the first newline.
@@ -301,9 +301,9 @@ pub(crate) fn trailing_commas(
 }
 
 fn check_token(
-    token: Token,
-    prev: Token,
-    prev_prev: Token,
+    token: RuleToken,
+    prev: RuleToken,
+    prev_prev: RuleToken,
     context: Context,
     locator: &Locator,
 ) -> Option<Diagnostic> {
@@ -387,9 +387,9 @@ fn check_token(
 }
 
 fn update_context(
-    token: Token,
-    prev: Token,
-    prev_prev: Token,
+    token: RuleToken,
+    prev: RuleToken,
+    prev_prev: RuleToken,
     stack: &mut Vec<Context>,
 ) -> Context {
     let new_context = match token.ty {

crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/implicit.rs

@@ -4,9 +4,9 @@ use ruff_diagnostics::{Diagnostic, Edit, Fix, FixAvailability, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::str::{leading_quote, trailing_quote};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, TokenKindIter};
+use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::Locator;
-use ruff_text_size::TextRange;
+use ruff_text_size::{Ranged, TextRange};
 
 use crate::settings::LinterSettings;
 
@@ -92,37 +92,39 @@ impl Violation for MultiLineImplicitStringConcatenation {
 /// ISC001, ISC002
 pub(crate) fn implicit(
     diagnostics: &mut Vec<Diagnostic>,
-    tokens: TokenKindIter,
+    tokens: &Tokens,
     settings: &LinterSettings,
     locator: &Locator,
     indexer: &Indexer,
 ) {
-    for ((a_tok, a_range), (b_tok, b_range)) in tokens
-        .filter(|(token, _)| {
-            *token != TokenKind::Comment
+    for (a_token, b_token) in tokens
+        .up_to_first_unknown()
+        .iter()
+        .filter(|token| {
+            token.kind() != TokenKind::Comment
                 && (settings.flake8_implicit_str_concat.allow_multiline
-                    || *token != TokenKind::NonLogicalNewline)
+                    || token.kind() != TokenKind::NonLogicalNewline)
         })
         .tuple_windows()
     {
-        let (a_range, b_range) = match (a_tok, b_tok) {
-            (TokenKind::String, TokenKind::String) => (a_range, b_range),
+        let (a_range, b_range) = match (a_token.kind(), b_token.kind()) {
+            (TokenKind::String, TokenKind::String) => (a_token.range(), b_token.range()),
             (TokenKind::String, TokenKind::FStringStart) => {
-                match indexer.fstring_ranges().innermost(b_range.start()) {
-                    Some(b_range) => (a_range, b_range),
+                match indexer.fstring_ranges().innermost(a_token.start()) {
+                    Some(b_range) => (a_token.range(), b_range),
                     None => continue,
                 }
             }
             (TokenKind::FStringEnd, TokenKind::String) => {
-                match indexer.fstring_ranges().innermost(a_range.start()) {
-                    Some(a_range) => (a_range, b_range),
+                match indexer.fstring_ranges().innermost(a_token.start()) {
+                    Some(a_range) => (a_range, b_token.range()),
                     None => continue,
                 }
             }
             (TokenKind::FStringEnd, TokenKind::FStringStart) => {
                 match (
-                    indexer.fstring_ranges().innermost(a_range.start()),
-                    indexer.fstring_ranges().innermost(b_range.start()),
+                    indexer.fstring_ranges().innermost(a_token.start()),
+                    indexer.fstring_ranges().innermost(b_token.start()),
                 ) {
                     (Some(a_range), Some(b_range)) => (a_range, b_range),
                     _ => continue,