Fix crash on escaped backslashes in rf-string (#921)

Instagram · May 17, 2023 · ea19578 · ea19578
1 parent 889ce56
commit ea19578
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/native/libcst/src/tokenizer/core/mod.rs b/native/libcst/src/tokenizer/core/mod.rs
@@ -973,8 +973,8 @@ impl<'t> TokState<'t> {
                 }
                 (Some('\\'), _) if is_raw_string => {
                     self.text_pos.next();
-                    if let Some('"' | '\'') = self.text_pos.peek() {
-                        // these aren't end of string markers, skip them
+                    // skip escaped end-of-string marker or backslash
+                    if let Some('"' | '\'' | '\\') = self.text_pos.peek() {
                         self.text_pos.next();
                     }
                 }

diff --git a/native/libcst/src/tokenizer/tests.rs b/native/libcst/src/tokenizer/tests.rs
@@ -529,6 +529,10 @@ fn test_string_prefix() {
         tokenize_all(r#"r"\"""#, &default_config()),
         Ok(vec![(TokType::String, r#"r"\"""#)]),
     );
+    assert_eq!(
+        tokenize_all(r#"r'\\'"#, &default_config()),
+        Ok(vec![(TokType::String, r#"r'\\'"#)]),
+    );
     let config = TokConfig {
         split_fstring: true,
         ..default_config()
@@ -549,6 +553,14 @@ fn test_string_prefix() {
             (TokType::FStringEnd, "\""),
         ]),
     );
+    assert_eq!(
+        tokenize_all(r#"rf'\\'"#, &config),
+        Ok(vec![
+            (TokType::FStringStart, "rf'"),
+            (TokType::FStringString, r#"\\"#),
+            (TokType::FStringEnd, "'"),
+        ]),
+    );
 }
 
 #[test]