Fix lexer issue where select/plural/other/underscores cannot be in id…

…entifier names. (#119190)
Maatteogekko · Jan 25, 2023 · 02a9c15 · 02a9c15
1 parent a45727d
commit 02a9c15
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 9 deletions.
diff --git a/packages/flutter_tools/lib/src/localizations/message_parser.dart b/packages/flutter_tools/lib/src/localizations/message_parser.dart
@@ -158,20 +158,14 @@ RegExp normalString = RegExp(r'[^{}]+');
 RegExp brace = RegExp(r'{|}');
 
 RegExp whitespace = RegExp(r'\s+');
-RegExp pluralKeyword = RegExp(r'plural');
-RegExp selectKeyword = RegExp(r'select');
-RegExp otherKeyword = RegExp(r'other');
 RegExp numeric = RegExp(r'[0-9]+');
-RegExp alphanumeric = RegExp(r'[a-zA-Z0-9]+');
+RegExp alphanumeric = RegExp(r'[a-zA-Z0-9|_]+');
 RegExp comma = RegExp(r',');
 RegExp equalSign = RegExp(r'=');
 
 // List of token matchers ordered by precedence
 Map<ST, RegExp> matchers = <ST, RegExp>{
   ST.empty: whitespace,
-  ST.plural: pluralKeyword,
-  ST.select: selectKeyword,
-  ST.other: otherKeyword,
   ST.number: numeric,
   ST.comma: comma,
   ST.equalSign: equalSign,
@@ -303,12 +297,25 @@ class Parser {
           // Do not add whitespace as a token.
           startIndex = match.end;
           continue;
-        } else if (<ST>[ST.plural, ST.select].contains(matchedType) && tokens.last.type == ST.openBrace) {
-          // Treat "plural" or "select" as identifier if it comes right after an open brace.
+        } else if (<ST>[ST.identifier].contains(matchedType) && tokens.last.type == ST.openBrace) {
+          // Treat any token as identifier if it comes right after an open brace, whether it's a keyword or not.
           tokens.add(Node(ST.identifier, startIndex, value: match.group(0)));
           startIndex = match.end;
           continue;
         } else {
+          // Handle keywords separately. Otherwise, lexer will assume parts of identifiers may be keywords.
+          final String tokenStr = match.group(0)!;
+          switch(tokenStr) {
+            case 'plural':
+              matchedType = ST.plural;
+              break;
+            case 'select':
+              matchedType = ST.select;
+              break;
+            case 'other':
+              matchedType = ST.other;
+              break;
+          }
           tokens.add(Node(matchedType!, startIndex, value: match.group(0)));
           startIndex = match.end;
           continue;

diff --git a/packages/flutter_tools/test/general.shard/message_parser_test.dart b/packages/flutter_tools/test/general.shard/message_parser_test.dart
@@ -226,6 +226,22 @@ void main() {
     expect(tokens[5].type, equals(ST.identifier));
   });
 
+  testWithoutContext('lexer identifier names can contain underscores', () {
+    final List<Node> tokens = Parser('keywords', 'app_en.arb', '{ test_placeholder } { test_select, select, singular{test} other{hmm} }').lexIntoTokens();
+    expect(tokens[1].value, equals('test_placeholder'));
+    expect(tokens[1].type, equals(ST.identifier));
+    expect(tokens[5].value, equals('test_select'));
+    expect(tokens[5].type, equals(ST.identifier));
+  });
+
+   testWithoutContext('lexer identifier names can contain the strings select or plural', () {
+    final List<Node> tokens = Parser('keywords', 'app_en.arb', '{ selectTest } { pluralTest, select, singular{test} other{hmm} }').lexIntoTokens();
+    expect(tokens[1].value, equals('selectTest'));
+    expect(tokens[1].type, equals(ST.identifier));
+    expect(tokens[5].value, equals('pluralTest'));
+    expect(tokens[5].type, equals(ST.identifier));
+  });
+
   testWithoutContext('lexer: lexically correct but syntactically incorrect', () {
     final List<Node> tokens = Parser(
       'syntax',