Merge pull request #20449 from alrz/features/leading-digit-separator

Allow digit separator after base specifier
Merging on behalf of @alrz. Thanks for the contribution!

src/Compilers/CSharp/Portable/CSharpResources.resx

@@ -5111,4 +5111,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ
   <data name="ERR_DynamicLocalFunctionTypeParameter" xml:space="preserve">
     <value>Cannot pass argument with dynamic type to generic local function '{0}' with inferred type arguments.</value>
   </data>
+  <data name="IDS_FeatureLeadingDigitSeparator" xml:space="preserve">
+    <value>leading digit separator</value>
+  </data>
 </root>

src/Compilers/CSharp/Portable/Errors/MessageID.cs

@@ -132,6 +132,8 @@ internal enum MessageID
         IDS_FeatureGenericPatternMatching = MessageBase + 12720,
         IDS_FeatureAsyncMain = MessageBase + 12721,
         IDS_LangVersions = MessageBase +  12722,
+
+        IDS_FeatureLeadingDigitSeparator = MessageBase + 12723,
     }
 
     // Message IDs may refer to strings that need to be localized.
@@ -188,6 +190,10 @@ internal static LanguageVersion RequiredVersion(this MessageID feature)
             // Checks are in the LanguageParser unless otherwise noted.
             switch (feature)
             {
+                // C# 7.2 features.
+                case MessageID.IDS_FeatureLeadingDigitSeparator:
+                    return LanguageVersion.CSharp7_2;
+
                 // C# 7.1 features.
                 case MessageID.IDS_FeatureAsyncMain:
                 case MessageID.IDS_FeatureDefaultLiteral:

src/Compilers/CSharp/Portable/Parser/Lexer.cs

@@ -932,27 +932,34 @@ private bool ScanInteger()
             return start < TextWindow.Position; 
         }
 
-        // Allows underscores in integers, except at beginning and end
+        // Allows underscores in integers, except at beginning for decimal and end
         private void ScanNumericLiteralSingleInteger(ref bool underscoreInWrongPlace, ref bool usedUnderscore, bool isHex, bool isBinary)
         {
+            bool firstCharWasUnderscore = false;
             if (TextWindow.PeekChar() == '_')
             {
-                underscoreInWrongPlace = true;
+                if (isHex || isBinary)
+                {
+                    firstCharWasUnderscore = true;
+                }
+                else
+                {
+                    underscoreInWrongPlace = true;
+                }
             }
 
-            char ch;
-            var lastCharWasUnderscore = false;
+            bool lastCharWasUnderscore = false;
             while (true)
             {
-                ch = TextWindow.PeekChar();
+                char ch = TextWindow.PeekChar();
                 if (ch == '_')
                 {
                     usedUnderscore = true;
                     lastCharWasUnderscore = true;
                 }
-                else if ((isHex && !SyntaxFacts.IsHexDigit(ch))
-                        || (isBinary && !SyntaxFacts.IsBinaryDigit(ch))
-                        || (!isHex && !isBinary && !SyntaxFacts.IsDecDigit(ch)))
+                else if (!(isHex ? SyntaxFacts.IsHexDigit(ch) :
+                           isBinary ? SyntaxFacts.IsBinaryDigit(ch) :
+                           SyntaxFacts.IsDecDigit(ch)))
                 {
                     break;
                 }
@@ -964,7 +971,13 @@ private void ScanNumericLiteralSingleInteger(ref bool underscoreInWrongPlace, re
                 TextWindow.AdvanceChar();
             }
 
-            if (lastCharWasUnderscore)
+            if (firstCharWasUnderscore)
+            {
+                CheckFeatureAvailability(MessageID.IDS_FeatureLeadingDigitSeparator);
+                // No need for cascading feature error
+                usedUnderscore = false;
+            }
+            else if (lastCharWasUnderscore)
             {
                 underscoreInWrongPlace = true;
             }

src/Compilers/CSharp/Test/Syntax/LexicalAndXml/LexicalTests.cs

@@ -16,6 +16,8 @@ public class LexicalTests
     {
         private readonly CSharpParseOptions _options;
         private readonly CSharpParseOptions _options6;
+        private readonly CSharpParseOptions _options7;
+        private readonly CSharpParseOptions _options72;
         private readonly CSharpParseOptions _binaryOptions;
         private readonly CSharpParseOptions _underscoreOptions;
         private readonly CSharpParseOptions _binaryUnderscoreOptions;
@@ -24,8 +26,10 @@ public LexicalTests()
         {
             _options = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp3);
             _options6 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp6);
-            _binaryOptions = _options.WithLanguageVersion(LanguageVersion.CSharp7);
-            _underscoreOptions = _options.WithLanguageVersion(LanguageVersion.CSharp7);
+            _options7 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp7);
+            _options72 = new CSharpParseOptions(languageVersion: LanguageVersion.CSharp7_2);
+            _binaryOptions = _options7;
+            _underscoreOptions = _options7;
             _binaryUnderscoreOptions = _binaryOptions;
         }
 
@@ -2614,6 +2618,89 @@ public void TestNumericWithUnderscoresWithoutFeatureFlag()
             Assert.Equal(text, token.Text);
         }
 
+        [Fact]
+        [Trait("Feature", "Literals")]
+        public void TestNumericWithLeadingUnderscores()
+        {
+            var text = "0x_A";
+            var token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            Assert.Equal(0, token.Errors().Length);
+            Assert.Equal(0xA, token.Value);
+            Assert.Equal(text, token.Text);
+
+            text = "0b_1";
+            token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            Assert.Equal(0, token.Errors().Length);
+            Assert.Equal(1, token.Value);
+            Assert.Equal(text, token.Text);
+
+            text = "0x__A_1L";
+            token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            Assert.Equal(0, token.Errors().Length);
+            Assert.Equal(0xA1L, token.Value);
+            Assert.Equal(text, token.Text);
+
+            text = "0b__1_1L";
+            token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            Assert.Equal(0, token.Errors().Length);
+            Assert.Equal(0b11L, token.Value);
+            Assert.Equal(text, token.Text);
+        }
+
+        [Fact]
+        [Trait("Feature", "Literals")]
+        public void TestNumericWithLeadingUnderscoresWithoutFeatureFlag()
+        {
+            var text = "0x_A";
+            var token = LexToken(text, _options7);
+
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            var errors = token.Errors();
+            Assert.Equal(1, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion7, errors[0].Code);
+            Assert.Equal(text, token.Text);
+
+            text = "0b_1";
+            token = LexToken(text, _options7);
+
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            errors = token.Errors();
+            Assert.Equal(1, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion7, errors[0].Code);
+            Assert.Equal(text, token.Text);
+
+            text = "0x_1";
+            token = LexToken(text, _options6);
+
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            errors = token.Errors();
+            Assert.Equal(1, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion6, errors[0].Code);
+            Assert.Equal(text, token.Text);
+
+            text = "0x_123_456_789_ABC_DEF_123";
+            token = LexToken(text, _options6);
+
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            errors = token.Errors();
+            Assert.Equal(2, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_FeatureNotAvailableInVersion6, errors[0].Code);
+            Assert.Equal((int)ErrorCode.ERR_IntOverflow, errors[1].Code);
+            Assert.Equal(text, token.Text);
+        }
+
         [Fact]
         [Trait("Feature", "Literals")]
         public void TestNumericWithBadUnderscores()
@@ -2695,7 +2782,7 @@ public void TestNumericWithBadUnderscores()
             Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
             Assert.Equal(text, token.Text);
 
-            text = "0x_A";
+            text = "0xA_";
             token = LexToken(text, _underscoreOptions);
 
             Assert.NotNull(token);
@@ -2706,8 +2793,8 @@ public void TestNumericWithBadUnderscores()
             Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
             Assert.Equal(text, token.Text);
 
-            text = "0xA_";
-            token = LexToken(text, _underscoreOptions);
+            text = "0b1_";
+            token = LexToken(text, _binaryUnderscoreOptions);
 
             Assert.NotNull(token);
             Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
@@ -2717,8 +2804,8 @@ public void TestNumericWithBadUnderscores()
             Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
             Assert.Equal(text, token.Text);
 
-            text = "0b_1";
-            token = LexToken(text, _binaryUnderscoreOptions);
+            text = "0x_";
+            token = LexToken(text, _options72);
 
             Assert.NotNull(token);
             Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
@@ -2728,8 +2815,8 @@ public void TestNumericWithBadUnderscores()
             Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
             Assert.Equal(text, token.Text);
 
-            text = "0b1_";
-            token = LexToken(text, _binaryUnderscoreOptions);
+            text = "1E+_2";
+            token = LexToken(text, _options72);
 
             Assert.NotNull(token);
             Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
@@ -2738,6 +2825,29 @@ public void TestNumericWithBadUnderscores()
             Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
             Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
             Assert.Equal(text, token.Text);
+
+            text = "1E-_2";
+            token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            errors = token.Errors();
+            Assert.Equal(1, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
+            Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
+            Assert.Equal(text, token.Text);
+
+            text = "1E_";
+            token = LexToken(text, _options72);
+
+            Assert.NotNull(token);
+            Assert.Equal(SyntaxKind.NumericLiteralToken, token.Kind());
+            errors = token.Errors();
+            Assert.Equal(2, errors.Length);
+            Assert.Equal((int)ErrorCode.ERR_InvalidNumber, errors[0].Code);
+            Assert.Equal((int)ErrorCode.ERR_FloatOverflow, errors[1].Code);
+            Assert.Equal("error CS1013: Invalid number", errors[0].ToString(EnsureEnglishUICulture.PreferredOrNull));
+            Assert.Equal(text, token.Text);
         }
 
         [Fact]

src/Compilers/VisualBasic/Portable/Errors/Errors.vb

@@ -2009,5 +2009,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic
         FEATURE_BinaryLiterals
         FEATURE_Tuples
         FEATURE_IOperation
+        FEATURE_LeadingDigitSeparator
     End Enum
 End Namespace

src/Compilers/VisualBasic/Portable/Parser/ParserFeature.vb

@@ -34,6 +34,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
         Tuples
         IOperation
         InferredTupleNames
+        LeadingDigitSeparator
     End Enum
 
     Friend Module FeatureExtensions
@@ -90,6 +91,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
                 Case Feature.InferredTupleNames
                     Return LanguageVersion.VisualBasic15_3
 
+                Case Feature.LeadingDigitSeparator
+                    Return LanguageVersion.VisualBasic15_5
+
                 Case Else
                     Throw ExceptionUtilities.UnexpectedValue(feature)
             End Select
@@ -153,6 +157,8 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
                     Return ERRID.FEATURE_Tuples
                 Case Feature.IOperation
                     Return ERRID.FEATURE_IOperation
+                Case Feature.LeadingDigitSeparator
+                    Return ERRID.FEATURE_LeadingDigitSeparator
                 Case Else
                     Throw ExceptionUtilities.UnexpectedValue(feature)
             End Select

src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb

@@ -1673,6 +1673,7 @@ FullWidthRepeat:
             Dim IntegerLiteralStart As Integer
             Dim UnderscoreInWrongPlace As Boolean
             Dim UnderscoreUsed As Boolean = False
+            Dim LeadingUnderscoreUsed = False
 
             Dim Base As LiteralBase = LiteralBase.Decimal
             Dim literalKind As NumericLiteralKind = NumericLiteralKind.Integral
@@ -1695,7 +1696,10 @@ FullWidthRepeat:
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Hexadecimal
 
-                        UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
+                        If CanGet(Here) AndAlso Peek(Here) = "_"c Then
+                            LeadingUnderscoreUsed = True
+                        End If
+
                         While CanGet(Here)
                             ch = Peek(Here)
                             If Not IsHexDigit(ch) AndAlso ch <> "_"c Then
@@ -1713,7 +1717,10 @@ FullWidthRepeat:
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Binary
 
-                        UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
+                        If CanGet(Here) AndAlso Peek(Here) = "_"c Then
+                            LeadingUnderscoreUsed = True
+                        End If
+
                         While CanGet(Here)
                             ch = Peek(Here)
                             If Not IsBinaryDigit(ch) AndAlso ch <> "_"c Then
@@ -1731,7 +1738,10 @@ FullWidthRepeat:
                         IntegerLiteralStart = Here
                         Base = LiteralBase.Octal
 
-                        UnderscoreInWrongPlace = (CanGet(Here) AndAlso Peek(Here) = "_"c)
+                        If CanGet(Here) AndAlso Peek(Here) = "_"c Then
+                            LeadingUnderscoreUsed = True
+                        End If
+
                         While CanGet(Here)
                             ch = Peek(Here)
                             If Not IsOctalDigit(ch) AndAlso ch <> "_"c Then
@@ -2085,13 +2095,16 @@ FullWidthRepeat2:
 
             If Overflows Then
                 result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Overflow)), SyntaxToken)
-            ElseIf UnderscoreInWrongPlace Then
-                result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Syntax)), SyntaxToken)
             End If
 
-            If UnderscoreUsed Then
+            If UnderscoreInWrongPlace Then
+                result = DirectCast(result.AddError(ErrorFactory.ErrorInfo(ERRID.ERR_Syntax)), SyntaxToken)
+            ElseIf LeadingUnderscoreUsed Then
+                result = CheckFeatureAvailability(result, Feature.LeadingDigitSeparator)
+            ElseIf UnderscoreUsed Then
                 result = CheckFeatureAvailability(result, Feature.DigitSeparators)
             End If
+
             If Base = LiteralBase.Binary Then
                 result = CheckFeatureAvailability(result, Feature.BinaryLiterals)
             End If