From 3bb68a862e842afc610fef1b8115f573649b0274 Mon Sep 17 00:00:00 2001 From: Zsolt Dollenstein Date: Mon, 17 Jan 2022 16:37:52 +0000 Subject: [PATCH] Stop indentation checking at EOF --- native/libcst/src/tokenizer/core/mod.rs | 10 +++++++--- native/libcst/src/tokenizer/tests.rs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/native/libcst/src/tokenizer/core/mod.rs b/native/libcst/src/tokenizer/core/mod.rs index 3ff97e7db..0dd60944e 100644 --- a/native/libcst/src/tokenizer/core/mod.rs +++ b/native/libcst/src/tokenizer/core/mod.rs @@ -533,9 +533,13 @@ impl<'t> TokState<'t> { } } - // Lines with only whitespace and/or comments and/or a line continuation character shouldn't - // affect the indentation and are not passed to the parser as NEWLINE tokens. - self.blank_line = matches!(self.text_pos.peek(), Some('#') | Some('\n') | Some('\\')); + // Lines with only whitespace and/or comments and/or a line continuation + // character shouldn't affect the indentation and are not passed to the parser + // as NEWLINE tokens. + self.blank_line = matches!( + self.text_pos.peek(), + Some('#') | Some('\n') | Some('\\') | None + ); if self.blank_line || !self.paren_stack.is_empty() { return Ok(()); diff --git a/native/libcst/src/tokenizer/tests.rs b/native/libcst/src/tokenizer/tests.rs index db437857d..2be070d13 100644 --- a/native/libcst/src/tokenizer/tests.rs +++ b/native/libcst/src/tokenizer/tests.rs @@ -727,3 +727,20 @@ fn test_add_dedents_for_dangling_indent_with_comment() { ]) ); } + +#[test] +fn test_inconsistent_indentation_at_eof() { + assert_eq!( + tokenize_all("if 1:\n pass\n ", &default_config()), + Ok(vec![ + (TokType::Name, "if"), + (TokType::Number, "1"), + (TokType::Op, ":"), + (TokType::Newline, "\n"), + (TokType::Indent, ""), + (TokType::Name, "pass"), + (TokType::Newline, "\n"), + (TokType::Dedent, ""), + ]) + ) +}