Implement concatenation operator for "u8" literals. (#62044)

dotnet/csharplang#6221

src/Compilers/CSharp/Portable/Binder/Binder_Operators.cs

@@ -8,6 +8,7 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.Linq;
 using Microsoft.CodeAnalysis.CSharp.Symbols;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Microsoft.CodeAnalysis.PooledObjects;
@@ -796,7 +797,7 @@ private void ReportAssignmentOperatorError(AssignmentExpressionSyntax node, Bind
             }
         }
 
-        private static void ReportBinaryOperatorError(ExpressionSyntax node, BindingDiagnosticBag diagnostics, SyntaxToken operatorToken, BoundExpression left, BoundExpression right, LookupResultKind resultKind)
+        private void ReportBinaryOperatorError(ExpressionSyntax node, BindingDiagnosticBag diagnostics, SyntaxToken operatorToken, BoundExpression left, BoundExpression right, LookupResultKind resultKind)
         {
             bool isEquality = operatorToken.Kind() == SyntaxKind.EqualsEqualsToken || operatorToken.Kind() == SyntaxKind.ExclamationEqualsToken;
             switch (left.Kind, right.Kind)
@@ -825,11 +826,33 @@ private static void ReportBinaryOperatorError(ExpressionSyntax node, BindingDiag
                     return;
             }
 
-            ErrorCode errorCode = resultKind == LookupResultKind.Ambiguous ?
-                ErrorCode.ERR_AmbigBinaryOps : // Operator '{0}' is ambiguous on operands of type '{1}' and '{2}'
-                ErrorCode.ERR_BadBinaryOps;    // Operator '{0}' cannot be applied to operands of type '{1}' and '{2}'
+#nullable enable
+            ErrorCode errorCode;
+
+            switch (resultKind)
+            {
+                case LookupResultKind.Ambiguous:
+                    errorCode = ErrorCode.ERR_AmbigBinaryOps; // Operator '{0}' is ambiguous on operands of type '{1}' and '{2}'
+                    break;
+
+                case LookupResultKind.OverloadResolutionFailure when operatorToken.Kind() is SyntaxKind.PlusToken && isReadOnlySpanOfByte(left.Type) && isReadOnlySpanOfByte(right.Type):
+                    errorCode = ErrorCode.ERR_BadBinaryReadOnlySpanConcatenation; // Operator '{0}' cannot be applied to operands of type '{1}' and '{2}' that are not UTF8 byte representations
+                    break;
+
+                default:
+                    errorCode = ErrorCode.ERR_BadBinaryOps;    // Operator '{0}' cannot be applied to operands of type '{1}' and '{2}'
+                    break;
+            }
 
             Error(diagnostics, errorCode, node, operatorToken.Text, left.Display, right.Display);
+
+            bool isReadOnlySpanOfByte(TypeSymbol? type)
+            {
+                return type is NamedTypeSymbol namedType && Compilation.IsReadOnlySpanType(namedType) &&
+                    namedType.TypeArgumentsWithAnnotationsNoUseSiteDiagnostics.Single().Type.SpecialType is SpecialType.System_Byte;
+
+            }
+#nullable disable
         }
 
         private BoundExpression BindConditionalLogicalOperator(BinaryExpressionSyntax node, BindingDiagnosticBag diagnostics)

src/Compilers/CSharp/Portable/Binder/Semantics/Operators/BinaryOperatorOverloadResolution.cs

@@ -730,6 +730,13 @@ private void GetAllBuiltInOperators(BinaryOperatorKind kind, bool isChecked, Bou
 
                 // We similarly limit pointer operator candidates considered.
                 GetPointerOperators(kind, left, right, operators);
+
+                if (kind.Operator() is BinaryOperatorKind.Addition &&
+                    isUTF8ByteRepresentation(left) &&
+                    isUTF8ByteRepresentation(right))
+                {
+                    this.Compilation.builtInOperators.GetUTF8ConcatenationBuiltInOperator(left.Type, operators);
+                }
             }
 
             CandidateOperators(isChecked, operators, left, right, results, ref useSiteInfo);
@@ -743,6 +750,11 @@ static bool useOnlyReferenceEquality(Conversions conversions, BoundExpression le
                     ((object)left.Type == null || (!left.Type.IsDelegateType() && left.Type.SpecialType != SpecialType.System_String && left.Type.SpecialType != SpecialType.System_Delegate)) &&
                     ((object)right.Type == null || (!right.Type.IsDelegateType() && right.Type.SpecialType != SpecialType.System_String && right.Type.SpecialType != SpecialType.System_Delegate));
             }
+
+            static bool isUTF8ByteRepresentation(BoundExpression value)
+            {
+                return value is BoundUTF8String or BoundBinaryOperator { OperatorKind: BinaryOperatorKind.UTF8Addition };
+            }
         }
 
         private void GetReferenceEquality(BinaryOperatorKind kind, ArrayBuilder<BinaryOperatorSignature> operators)

src/Compilers/CSharp/Portable/Binder/Semantics/Operators/OperatorKind.cs

@@ -51,6 +51,7 @@ internal enum UnaryOperatorKind
         _NullableNull = 0x00000027, // reserved for binary op
         UserDefined = 0x00000028,
         Dynamic = 0x00000029,
+        _UTF8 = 0x0000002A, // reserved for binary op
 
         OpMask = 0x0000FF00,
         PostfixIncrement = 0x00001000,
@@ -324,6 +325,7 @@ internal enum BinaryOperatorKind
         NullableNull = UnaryOperatorKind._NullableNull,
         UserDefined = UnaryOperatorKind.UserDefined,
         Dynamic = UnaryOperatorKind.Dynamic,
+        UTF8 = UnaryOperatorKind._UTF8,
 
         OpMask = 0x0000FF00,
         Multiplication = 0x00001000,
@@ -453,6 +455,7 @@ internal enum BinaryOperatorKind
         ObjectAndStringConcatenation = ObjectAndString | Addition,
         DelegateCombination = Delegate | Addition,
         DynamicAddition = Dynamic | Addition,
+        UTF8Addition = UTF8 | Addition,
 
         IntSubtraction = Int | Subtraction,
         UIntSubtraction = UInt | Subtraction,

src/Compilers/CSharp/Portable/CSharpResources.resx

@@ -7133,4 +7133,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ
   <data name="ERR_BadAbstractEqualityOperatorSignature" xml:space="preserve">
     <value>One of the parameters of an equality, or inequality operator declared in interface '{0}' must be a type parameter on '{0}' constrained to '{0}'</value>
   </data>
+  <data name="ERR_BadBinaryReadOnlySpanConcatenation" xml:space="preserve">
+    <value>Operator '{0}' cannot be applied to operands of type '{1}' and '{2}' that are not UTF8 byte representations</value>
+  </data>
 </root>

src/Compilers/CSharp/Portable/Compilation/BuiltInOperators.cs

@@ -7,6 +7,8 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using Microsoft.CodeAnalysis.CSharp.Symbols;
 using Microsoft.CodeAnalysis.PooledObjects;
@@ -25,6 +27,7 @@ internal class BuiltInOperators
         //actual lazily-constructed caches of built-in operators.
         private ImmutableArray<UnaryOperatorSignature>[] _builtInUnaryOperators;
         private ImmutableArray<BinaryOperatorSignature>[][] _builtInOperators;
+        private StrongBox<BinaryOperatorSignature> _builtInUTF8Concatenation;
 
         internal BuiltInOperators(CSharpCompilation compilation)
         {
@@ -684,6 +687,21 @@ internal void GetSimpleBuiltInOperators(BinaryOperatorKind kind, ArrayBuilder<Bi
             }
         }
 
+        internal void GetUTF8ConcatenationBuiltInOperator(TypeSymbol readonlySpanOfByte, ArrayBuilder<BinaryOperatorSignature> operators)
+        {
+            Debug.Assert(_compilation.IsReadOnlySpanType(readonlySpanOfByte));
+            Debug.Assert(((NamedTypeSymbol)readonlySpanOfByte).TypeArgumentsWithAnnotationsNoUseSiteDiagnostics.Single().Type.SpecialType is SpecialType.System_Byte);
+
+            if (_builtInUTF8Concatenation is null)
+            {
+                Interlocked.CompareExchange(ref _builtInUTF8Concatenation,
+                    new StrongBox<BinaryOperatorSignature>(
+                        new BinaryOperatorSignature(BinaryOperatorKind.UTF8Addition, readonlySpanOfByte, readonlySpanOfByte, readonlySpanOfByte)), null);
+            }
+
+            operators.Add(_builtInUTF8Concatenation.Value);
+        }
+
         internal BinaryOperatorSignature GetSignature(BinaryOperatorKind kind)
         {
             var left = LeftType(kind);

src/Compilers/CSharp/Portable/Errors/ErrorCode.cs

@@ -2091,6 +2091,7 @@ internal enum ErrorCode
         ERR_ImplicitImplementationOfInaccessibleInterfaceMember = 9044,
         ERR_ScriptsAndSubmissionsCannotHaveRequiredMembers = 9045,
         ERR_BadAbstractEqualityOperatorSignature = 9046,
+        ERR_BadBinaryReadOnlySpanConcatenation = 9047,
 
         #endregion
 

src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_BinaryOperator.cs

@@ -117,6 +117,12 @@ public BoundExpression VisitBinaryOperator(BoundBinaryOperator node, BoundUnaryO
                 return LowerPartsToString(data, parts, node.Syntax, node.Type);
             }
 
+            if (node.OperatorKind is BinaryOperatorKind.UTF8Addition)
+            {
+                Debug.Assert(applyParentUnaryOperator is null);
+                return VisitUTF8Addition(node);
+            }
+
             // In machine-generated code we frequently end up with binary operator trees that are deep on the left,
             // such as a + b + c + d ...
             // To avoid blowing the call stack, we make an explicit stack of the binary operators to the left, 
@@ -127,7 +133,7 @@ public BoundExpression VisitBinaryOperator(BoundBinaryOperator node, BoundUnaryO
             for (BoundBinaryOperator? current = node; current != null && current.ConstantValue == null; current = current.Left as BoundBinaryOperator)
             {
                 // The regular visit mechanism will handle this.
-                if (current.InterpolatedStringHandlerData is not null)
+                if (current.InterpolatedStringHandlerData is not null || current.OperatorKind is BinaryOperatorKind.UTF8Addition)
                 {
                     Debug.Assert(stack.Count >= 1);
                     break;
@@ -513,6 +519,9 @@ private BoundExpression MakeBinaryOperator(
 
                         goto default;
 
+                    case BinaryOperatorKind.UTF8Addition:
+                        throw ExceptionUtilities.UnexpectedValue(operatorKind);
+
                     default:
                         break;
                 }

src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Linq;
@@ -75,16 +76,25 @@ public override BoundNode VisitConversion(BoundConversion node)
 
         public override BoundNode VisitUTF8String(BoundUTF8String node)
         {
-            Debug.Assert(node.Type.OriginalDefinition.Equals(_compilation.GetWellKnownType(WellKnownType.System_ReadOnlySpan_T), TypeCompareKind.AllIgnoreOptions));
+            return MakeUTF8Span(node, GetUTF8ByteRepresentation(node));
+        }
+
+        private BoundExpression MakeUTF8Span(BoundExpression node, IReadOnlyList<byte>? bytes)
+        {
+            Debug.Assert(node.Type is not null);
+            Debug.Assert(_compilation.IsReadOnlySpanType(node.Type));
             var byteType = ((NamedTypeSymbol)node.Type).TypeArgumentsWithAnnotationsNoUseSiteDiagnostics.Single().Type;
             Debug.Assert(byteType.SpecialType == SpecialType.System_Byte);
 
             var save_Syntax = _factory.Syntax;
             _factory.Syntax = node.Syntax;
 
-            int length;
-            BoundExpression utf8Bytes = createUTF8ByteRepresentation(node.Syntax, node.Value, ArrayTypeSymbol.CreateSZArray(_compilation.Assembly, TypeWithAnnotations.Create(byteType)), out length);
-            BoundNode result;
+            int length = 0;
+            BoundExpression result;
+            var byteArray = ArrayTypeSymbol.CreateSZArray(_compilation.Assembly, TypeWithAnnotations.Create(byteType));
+            BoundExpression utf8Bytes = bytes is null ?
+                                            BadExpression(node.Syntax, byteArray, ImmutableArray<BoundExpression>.Empty) :
+                                            MakeUnderlyingArrayForUTF8Span(node.Syntax, byteArray, bytes, out length);
 
             if (!TryGetWellKnownTypeMember<MethodSymbol>(node.Syntax, WellKnownMember.System_ReadOnlySpan_T__ctor_Array_Start_Length, out MethodSymbol ctor))
             {
@@ -98,48 +108,97 @@ public override BoundNode VisitUTF8String(BoundUTF8String node)
             _factory.Syntax = save_Syntax;
 
             return result;
+        }
+
+        private byte[]? GetUTF8ByteRepresentation(BoundUTF8String node)
+        {
+            var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
 
-            BoundExpression createUTF8ByteRepresentation(SyntaxNode syntax, string value, ArrayTypeSymbol byteArray, out int length)
+            try
             {
-                Debug.Assert(byteArray.IsSZArray);
-                Debug.Assert(byteArray.ElementType.SpecialType == SpecialType.System_Byte);
+                return utf8.GetBytes(node.Value);
+            }
+            catch (Exception ex)
+            {
+                _diagnostics.Add(
+                    ErrorCode.ERR_CannotBeConvertedToUTF8,
+                    node.Syntax.Location,
+                    ex.Message);
 
-                var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
-                byte[] bytes;
+                return null;
+            }
+        }
 
-                try
-                {
-                    bytes = utf8.GetBytes(value);
-                }
-                catch (Exception ex)
-                {
-                    _diagnostics.Add(
-                        ErrorCode.ERR_CannotBeConvertedToUTF8,
-                        syntax.Location,
-                        ex.Message);
+        private BoundArrayCreation MakeUnderlyingArrayForUTF8Span(SyntaxNode syntax, ArrayTypeSymbol byteArray, IReadOnlyList<byte> bytes, out int length)
+        {
+            Debug.Assert(byteArray.IsSZArray);
+            Debug.Assert(byteArray.ElementType.SpecialType == SpecialType.System_Byte);
 
-                    length = 0;
-                    return BadExpression(syntax, byteArray, ImmutableArray<BoundExpression>.Empty);
-                }
+            var builder = ArrayBuilder<BoundExpression>.GetInstance(bytes.Count + 1);
+            foreach (byte b in bytes)
+            {
+                builder.Add(_factory.Literal(b));
+            }
+
+            length = builder.Count;
 
-                var builder = ArrayBuilder<BoundExpression>.GetInstance(bytes.Length);
-                foreach (byte b in bytes)
+            // Zero terminate memory
+            builder.Add(_factory.Literal((byte)0));
+
+            var utf8Bytes = new BoundArrayCreation(
+                                    syntax,
+                                    ImmutableArray.Create<BoundExpression>(_factory.Literal(builder.Count)),
+                                    new BoundArrayInitialization(syntax, isInferred: false, builder.ToImmutableAndFree()),
+                                    byteArray);
+            return utf8Bytes;
+        }
+
+        private BoundExpression VisitUTF8Addition(BoundBinaryOperator node)
+        {
+            Debug.Assert(node.OperatorKind is BinaryOperatorKind.UTF8Addition);
+
+            var bytesBuilder = ArrayBuilder<byte>.GetInstance();
+            bool haveRepresentationError = false;
+            var stack = ArrayBuilder<BoundExpression>.GetInstance();
+
+            stack.Add(node);
+
+            while (stack.Count != 0)
+            {
+                var current = stack.Pop();
+
+                switch (current)
                 {
-                    builder.Add(_factory.Literal(b));
-                }
+                    case BoundUTF8String literal:
+                        byte[]? bytes = GetUTF8ByteRepresentation(literal);
 
-                length = builder.Count;
+                        if (bytes is null)
+                        {
+                            haveRepresentationError = true;
+                        }
+                        else if (!haveRepresentationError)
+                        {
+                            bytesBuilder.AddRange(bytes);
+                        }
+                        break;
 
-                // Zero terminate memory
-                builder.Add(_factory.Literal((byte)0));
+                    case BoundBinaryOperator utf8Addition:
+                        Debug.Assert(utf8Addition.OperatorKind is BinaryOperatorKind.UTF8Addition);
+                        stack.Push(utf8Addition.Right);
+                        stack.Push(utf8Addition.Left);
+                        break;
 
-                var utf8Bytes = new BoundArrayCreation(
-                                        syntax,
-                                        ImmutableArray.Create<BoundExpression>(_factory.Literal(builder.Count)),
-                                        new BoundArrayInitialization(syntax, isInferred: false, builder.ToImmutableAndFree()),
-                                        byteArray);
-                return utf8Bytes;
+                    default:
+                        throw ExceptionUtilities.UnexpectedValue(current);
+                }
             }
+
+            stack.Free();
+
+            BoundExpression result = MakeUTF8Span(node, haveRepresentationError ? null : bytesBuilder);
+
+            bytesBuilder.Free();
+            return result;
         }
 
         private static bool IsFloatingPointExpressionOfUnknownPrecision(BoundExpression rewrittenNode)