apacheGH-25163: [C#] Support half-float arrays. (apache#34618)

### Rationale for this change .NET 5 introduced the [`System.Half`](https://devblogs.microsoft.com/dotnet/introducing-the-half-type/) type, which represents 16-bit floats. This PR adds support for them in Apache Arrow. ### What changes are included in this PR? I multi-targeted the `Apache.Arrow` project to .NET 6 (because .NET 5 is unsupported) and added a `HalfFloatArray` type with a very similar implementation as the other floating-point array types. I also updated the README. ### Are these changes tested? Yes. I also refactored the array tests to reduce duplication among the various numeric types. ### Are there any user-facing changes? Yes. * Closes: apache#25163 Lead-authored-by: Theodore Tsirpanis <[email protected]> Co-authored-by: Eric Erhardt <[email protected]> Signed-off-by: Eric Erhardt <[email protected]>
Bit-Quill · May 15, 2023 · c662f09 · c662f09
1 parent 75cd740
commit c662f09
Show file tree

Hide file tree

Showing 12 changed files with 159 additions and 59 deletions.
diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props
@@ -34,7 +34,7 @@
 
   <PropertyGroup>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
-    <LangVersion>8.0</LangVersion>
+    <LangVersion>latest</LangVersion>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>$(CSharpDir)ApacheArrow.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>

diff --git a/csharp/README.md b/csharp/README.md
@@ -79,7 +79,7 @@ for currently available features.
 
 - Int8, Int16, Int32, Int64
 - UInt8, UInt16, UInt32, UInt64
-- Float, Double
+- Float, Double, Half-float (.NET 6+)
 - Binary (variable-length)
 - String (utf-8)
 - Null
@@ -126,12 +126,10 @@ for currently available features.
     - Dictionary Encoding
 - Types
     - Tensor
-    - Table
 - Arrays
     - Union
         - Dense
         - Sparse
-    - Half-Float
 - Array Operations
 	- Equality / Comparison
 	- Casting

diff --git a/csharp/src/Apache.Arrow/Apache.Arrow.csproj b/csharp/src/Apache.Arrow/Apache.Arrow.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard1.3;netstandard2.0;netcoreapp3.1</TargetFrameworks>
+    <TargetFrameworks>netstandard1.3;netstandard2.0;netcoreapp3.1;net6.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);UNSAFE_BYTEBUFFER;BYTEBUFFER_NO_BOUNDS_CHECK;ENABLE_SPAN_T</DefineConstants>
 
@@ -41,4 +41,7 @@
     <Compile Remove="Extensions\StreamExtensions.netstandard.cs" />
     <Compile Remove="Extensions\TupleExtensions.netstandard.cs" />
   </ItemGroup>
+  <ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net5.0'))">
+    <Compile Remove="Arrays\HalfFloatArray.cs" />
+  </ItemGroup>
 </Project>
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs b/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
@@ -42,6 +42,12 @@ internal static IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>
                     return new UInt64Array.Builder();
                 case ArrowTypeId.Int64:
                     return new Int64Array.Builder();
+                case ArrowTypeId.HalfFloat:
+#if NET5_0_OR_GREATER
+                    return new HalfFloatArray.Builder();
+#else
+                    throw new NotSupportedException("Half-float arrays are not supported by this target framework.");
+#endif
                 case ArrowTypeId.Float:
                     return new FloatArray.Builder();
                 case ArrowTypeId.Double:
@@ -70,7 +76,6 @@ internal static IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>
                 case ArrowTypeId.Union:
                 case ArrowTypeId.Dictionary:
                 case ArrowTypeId.FixedSizedBinary:
-                case ArrowTypeId.HalfFloat:
                 case ArrowTypeId.Interval:
                 case ArrowTypeId.Map:
                 default:

diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs b/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
@@ -76,6 +76,11 @@ public static IArrowArray BuildArray(ArrayData data)
                 case ArrowTypeId.Dictionary:
                     return new DictionaryArray(data);
                 case ArrowTypeId.HalfFloat:
+#if NET5_0_OR_GREATER
+                    return new HalfFloatArray(data);
+#else
+                    throw new NotSupportedException("Half-float arrays are not supported by this target framework.");
+#endif
                 case ArrowTypeId.Interval:
                 case ArrowTypeId.Map:
                 default:

diff --git a/csharp/src/Apache.Arrow/Arrays/HalfFloatArray.cs b/csharp/src/Apache.Arrow/Arrays/HalfFloatArray.cs
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Apache.Arrow.Types;
+using System;
+
+namespace Apache.Arrow
+{
+    public class HalfFloatArray : PrimitiveArray<Half>
+    {
+        public class Builder : PrimitiveArrayBuilder<Half, HalfFloatArray, Builder>
+        {
+            protected override HalfFloatArray Build(
+                ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
+                int length, int nullCount, int offset) =>
+                new HalfFloatArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
+        }
+
+        public HalfFloatArray(
+            ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
+            int length, int nullCount, int offset)
+            : this(new ArrayData(HalfFloatType.Default, length, nullCount, offset,
+                new[] { nullBitmapBuffer, valueBuffer }))
+        { }
+
+        public HalfFloatArray(ArrayData data)
+            : base(data)
+        {
+            data.EnsureDataType(ArrowTypeId.HalfFloat);
+        }
+
+        public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
+    }
+}
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -38,6 +38,9 @@ internal class ArrowRecordBatchFlatBufferBuilder :
             IArrowArrayVisitor<UInt16Array>,
             IArrowArrayVisitor<UInt32Array>,
             IArrowArrayVisitor<UInt64Array>,
+#if NET5_0_OR_GREATER
+            IArrowArrayVisitor<HalfFloatArray>,
+#endif
             IArrowArrayVisitor<FloatArray>,
             IArrowArrayVisitor<DoubleArray>,
             IArrowArrayVisitor<BooleanArray>,
@@ -87,6 +90,9 @@ public ArrowRecordBatchFlatBufferBuilder()
             public void Visit(UInt16Array array) => CreateBuffers(array);
             public void Visit(UInt32Array array) => CreateBuffers(array);
             public void Visit(UInt64Array array) => CreateBuffers(array);
+#if NET5_0_OR_GREATER
+            public void Visit(HalfFloatArray array) => CreateBuffers(array);
+#endif
             public void Visit(FloatArray array) => CreateBuffers(array);
             public void Visit(DoubleArray array) => CreateBuffers(array);
             public void Visit(TimestampArray array) => CreateBuffers(array);

diff --git a/csharp/src/Apache.Arrow/RecordBatch.Builder.cs b/csharp/src/Apache.Arrow/RecordBatch.Builder.cs
@@ -42,6 +42,9 @@ internal ArrayBuilder(MemoryAllocator allocator)
             public UInt16Array UInt16(Action<UInt16Array.Builder> action) => Build<UInt16Array, UInt16Array.Builder>(new UInt16Array.Builder(), action);
             public UInt32Array UInt32(Action<UInt32Array.Builder> action) => Build<UInt32Array, UInt32Array.Builder>(new UInt32Array.Builder(), action);
             public UInt64Array UInt64(Action<UInt64Array.Builder> action) => Build<UInt64Array, UInt64Array.Builder>(new UInt64Array.Builder(), action);
+#if NET5_0_OR_GREATER
+            public HalfFloatArray HalfFloat(Action<HalfFloatArray.Builder> action) => Build<HalfFloatArray, HalfFloatArray.Builder>(new HalfFloatArray.Builder(), action);
+#endif
             public FloatArray Float(Action<FloatArray.Builder> action) => Build<FloatArray, FloatArray.Builder>(new FloatArray.Builder(), action);
             public DoubleArray Double(Action<DoubleArray.Builder> action) => Build<DoubleArray, DoubleArray.Builder>(new DoubleArray.Builder(), action);
             public Decimal128Array Decimal128(Decimal128Type type, Action<Decimal128Array.Builder> action) =>

diff --git a/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs b/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
@@ -17,6 +17,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 using Xunit;
 
 namespace Apache.Arrow.Tests
@@ -28,36 +29,49 @@ public class ArrayBuilderTests
         [Fact]
         public void PrimitiveArrayBuildersProduceExpectedArray()
         {
-            TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<Time32Array, Time32Array.Builder>(x => x.Append(10).Append(20).Append(30));
-            TestArrayBuilder<Time64Array, Time64Array.Builder>(x => x.Append(10).Append(20).Append(30));
+            Test<sbyte, Int8Array, Int8Array.Builder>();
+            Test<short, Int16Array, Int16Array.Builder>();
+            Test<int, Int32Array, Int32Array.Builder>();
+            Test<long, Int64Array, Int64Array.Builder>();
+            Test<byte, UInt8Array, UInt8Array.Builder>();
+            Test<ushort, UInt16Array, UInt16Array.Builder>();
+            Test<uint, UInt32Array, UInt32Array.Builder>();
+            Test<ulong, UInt64Array, UInt64Array.Builder>();
+            Test<Half, HalfFloatArray, HalfFloatArray.Builder>();
+            Test<float, FloatArray, FloatArray.Builder>();
+            Test<double, DoubleArray, DoubleArray.Builder>();
+            Test<int, Time32Array, Time32Array.Builder>();
+            Test<long, Time64Array, Time64Array.Builder>();
+
+            static void Test<T, TArray, TBuilder>()
+                where T : struct, INumber<T>
+                where TArray : PrimitiveArray<T>
+                where TBuilder : PrimitiveArrayBuilder<T, TArray, TBuilder>, new() =>
+                TestArrayBuilder<TArray, TBuilder>(x => x.Append(T.CreateChecked(10)).Append(T.CreateChecked(20)).Append(T.CreateChecked(30)));
         }
 
         [Fact]
         public void PrimitiveArrayBuildersProduceExpectedArrayWithNulls()
         {
-            TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09);
-            TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09);
-            TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<Time32Array, Time32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
-            TestArrayBuilder<Time64Array, Time64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
+            Test<sbyte, Int8Array, Int8Array.Builder>();
+            Test<short, Int16Array, Int16Array.Builder>();
+            Test<int, Int32Array, Int32Array.Builder>();
+            Test<long, Int64Array, Int64Array.Builder>();
+            Test<byte, UInt8Array, UInt8Array.Builder>();
+            Test<ushort, UInt16Array, UInt16Array.Builder>();
+            Test<uint, UInt32Array, UInt32Array.Builder>();
+            Test<ulong, UInt64Array, UInt64Array.Builder>();
+            Test<Half, HalfFloatArray, HalfFloatArray.Builder>();
+            Test<float, FloatArray, FloatArray.Builder>();
+            Test<double, DoubleArray, DoubleArray.Builder>();
+            Test<int, Time32Array, Time32Array.Builder>();
+            Test<long, Time64Array, Time64Array.Builder>();
+
+            static void Test<T, TArray, TBuilder>()
+                where T : struct, INumber<T>
+                where TArray : PrimitiveArray<T>
+                where TBuilder : PrimitiveArrayBuilder<T, TArray, TBuilder>, new() =>
+                TestArrayBuilder<TArray, TBuilder>(x => x.Append(T.CreateChecked(123)).AppendNull().AppendNull().Append(T.CreateChecked(127)), 4, 2, 0x09);
         }
 
         [Fact]
@@ -138,7 +152,7 @@ List<string> ConvertStringArrayToList(StringArray array)
         [Fact]
         public void ListArrayBuilderValidityBuffer()
         {
-            ListArray listArray = new ListArray.Builder(Int64Type.Default).Append().AppendNull().Build();   
+            ListArray listArray = new ListArray.Builder(Int64Type.Default).Append().AppendNull().Build();
             Assert.False(listArray.IsValid(2));
         }