Allow trailing whitespace without newline at EOF

native/libcst/src/tokenizer/core/mod.rs

@@ -533,9 +533,13 @@ impl<'t> TokState<'t> {
             }
         }
 
-        // Lines with only whitespace and/or comments and/or a line continuation character shouldn't
-        // affect the indentation and are not passed to the parser as NEWLINE tokens.
-        self.blank_line = matches!(self.text_pos.peek(), Some('#') | Some('\n') | Some('\\'));
+        // Lines with only whitespace and/or comments and/or a line continuation
+        // character shouldn't affect the indentation and are not passed to the parser
+        // as NEWLINE tokens.
+        self.blank_line = matches!(
+            self.text_pos.peek(),
+            Some('#') | Some('\n') | Some('\\') | None
+        );
 
         if self.blank_line || !self.paren_stack.is_empty() {
             return Ok(());

native/libcst/src/tokenizer/tests.rs

@@ -727,3 +727,20 @@ fn test_add_dedents_for_dangling_indent_with_comment() {
         ])
     );
 }
+
+#[test]
+fn test_inconsistent_indentation_at_eof() {
+    assert_eq!(
+        tokenize_all("if 1:\n  pass\n ", &default_config()),
+        Ok(vec![
+            (TokType::Name, "if"),
+            (TokType::Number, "1"),
+            (TokType::Op, ":"),
+            (TokType::Newline, "\n"),
+            (TokType::Indent, ""),
+            (TokType::Name, "pass"),
+            (TokType::Newline, "\n"),
+            (TokType::Dedent, ""),
+        ])
+    )
+}