From cd4ffc1945a3a1ca89776e9abdcd60b1896f356c Mon Sep 17 00:00:00 2001 From: Julian Wollersberger <24991778+Julian-Wollersberger@users.noreply.github.com> Date: Sun, 24 May 2020 13:12:16 +0200 Subject: [PATCH 1/2] Update to rustc_lexer version 660. Change `unescape_*()` to `unescape_literal()`. --- Cargo.lock | 4 ++-- crates/ra_syntax/Cargo.toml | 2 +- crates/ra_syntax/src/ast/tokens.rs | 5 +++-- crates/ra_syntax/src/validation.rs | 16 ++++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 007f05b4d03e..a511e0d28049 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,9 +1398,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_lexer" -version = "656.0.0" +version = "660.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cbba98ec46e96a4663197dfa8c0378752de2006e314e5400c0ca74929d6692f" +checksum = "30760dbcc7667c9e0da561e980e24867ca7f4526ce060a3d7e6d9dcfeaae88d1" dependencies = [ "unicode-xid", ] diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index c07ff488e4f0..a9a5cc7bc6ed 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -13,7 +13,7 @@ doctest = false [dependencies] itertools = "0.9.0" rowan = "0.10.0" -rustc_lexer = { version = "656.0.0", package = "rustc-ap-rustc_lexer" } +rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" } rustc-hash = "1.1.0" arrayvec = "0.5.1" once_cell = "1.3.1" diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index 74906d8a62ef..3cd6d99c3889 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -6,6 +6,7 @@ use crate::{ ast::{AstToken, Comment, RawString, String, Whitespace}, TextRange, TextSize, }; +use rustc_lexer::unescape::{unescape_literal, Mode}; impl Comment { pub fn kind(&self) -> CommentKind { @@ -147,7 +148,7 @@ impl HasStringValue for String { let mut buf = std::string::String::with_capacity(text.len()); let mut has_error = false; - rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char { + unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char { Ok(c) => buf.push(c), Err(_) => has_error = true, }); @@ -498,7 +499,7 @@ impl HasFormatSpecifier for String { let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); let mut res = Vec::with_capacity(text.len()); - rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| { + unescape_literal(text, Mode::Str, &mut |range, unescaped_char| { res.push(( TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()) + offset, diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs index d68cf0a82bf4..436ab033dc8b 100644 --- a/crates/ra_syntax/src/validation.rs +++ b/crates/ra_syntax/src/validation.rs @@ -2,15 +2,15 @@ mod block; -use std::convert::TryFrom; - -use rustc_lexer::unescape; - use crate::{ ast, match_ast, AstNode, SyntaxError, SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF}, SyntaxNode, SyntaxToken, TextSize, T, }; +use rustc_lexer::unescape::{ + self, unescape_byte, unescape_byte_literal, unescape_char, unescape_literal, Mode, +}; +use std::convert::TryFrom; fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { use unescape::EscapeError as EE; @@ -121,18 +121,18 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { match token.kind() { BYTE => { - if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape::unescape_byte) { + if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { push_err(2, e); } } CHAR => { - if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape::unescape_char) { + if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { push_err(1, e); } } BYTE_STRING => { if let Some(without_quotes) = unquote(text, 2, '"') { - unescape::unescape_byte_str(without_quotes, &mut |range, char| { + unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| { if let Err(err) = char { push_err(2, (range.start, err)); } @@ -141,7 +141,7 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { } STRING => { if let Some(without_quotes) = unquote(text, 1, '"') { - unescape::unescape_str(without_quotes, &mut |range, char| { + unescape_literal(without_quotes, Mode::Str, &mut |range, char| { if let Err(err) = char { push_err(1, (range.start, err)); } From ff9d553fe3af092e4ef5467f07cef62af27b868b Mon Sep 17 00:00:00 2001 From: Julian Wollersberger <24991778+Julian-Wollersberger@users.noreply.github.com> Date: Sun, 24 May 2020 13:17:16 +0200 Subject: [PATCH 2/2] Some FIXMEs were outdated: * Done at line 243: "Add validation of `crate` keyword not appearing in the middle of the symbol path" * Already happened: "Remove validation of unterminated literals (it is already implemented in `tokenize()`)" * Happens in `unescape()`: "Add validation of character literal containing only a single char" * Missing: "raw string literals and raw byte string literals" --- crates/ra_syntax/src/validation.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs index 436ab033dc8b..fdec48fb0b88 100644 --- a/crates/ra_syntax/src/validation.rs +++ b/crates/ra_syntax/src/validation.rs @@ -81,10 +81,8 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { pub(crate) fn validate(root: &SyntaxNode) -> Vec { // FIXME: - // * Add validation of character literal containing only a single char - // * Add validation of `crate` keyword not appearing in the middle of the symbol path + // * Add unescape validation of raw string literals and raw byte string literals // * Add validation of doc comments are being attached to nodes - // * Remove validation of unterminated literals (it is already implemented in `tokenize()`) let mut errors = Vec::new(); for node in root.descendants() {