apacheGH-43643: [Java] LargeListViewVector IPC Integration (apache#43681

) ### Rationale for this change Newly introduced `LargeListViewVector` requires the IPC integration for C Data integration tests while mainly supporting IPC format to include this type. ### What changes are included in this PR? Includes the `JsonFileWriter` and `JsonFileReader` along with the corresponding test cases. ### Are these changes tested? Yes, using existing tests but adding new configurations. ### Are there any user-facing changes? No * GitHub Issue: apache#43643 Authored-by: Vibhatha Abeykoon <[email protected]> Signed-off-by: David Li <[email protected]>
vibhatha · Aug 14, 2024 · 712cfe6 · 712cfe6
1 parent ce251a6
commit 712cfe6
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 4 deletions.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -73,6 +73,7 @@
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.LargeListView;
 import org.apache.arrow.vector.types.pojo.ArrowType.ListView;
 import org.apache.arrow.vector.types.pojo.ArrowType.Union;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -729,7 +730,8 @@ private List<ArrowBuf> readIntoBuffer(
     } else if (bufferType.equals(OFFSET) || bufferType.equals(SIZE)) {
       if (type == MinorType.LARGELIST
           || type == MinorType.LARGEVARCHAR
-          || type == MinorType.LARGEVARBINARY) {
+          || type == MinorType.LARGEVARBINARY
+          || type == MinorType.LARGELISTVIEW) {
         reader = helper.INT8;
       } else {
         reader = helper.INT4;
@@ -890,7 +892,10 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws IOEx
         BufferType bufferType = vectorTypes.get(v);
         nextFieldIs(bufferType.getName());
         int innerBufferValueCount = valueCount;
-        if (bufferType.equals(OFFSET) && !(type instanceof Union) && !(type instanceof ListView)) {
+        if (bufferType.equals(OFFSET)
+            && !(type instanceof Union)
+            && !(type instanceof ListView)
+            && !(type instanceof LargeListView)) {
           /* offset buffer has 1 additional value capacity except for dense unions and ListView */
           innerBufferValueCount = valueCount + 1;
         }

diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -73,6 +73,7 @@
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.BaseLargeRepeatedValueViewVector;
 import org.apache.arrow.vector.complex.BaseRepeatedValueViewVector;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
@@ -232,7 +233,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
         final int bufferValueCount =
             (bufferType.equals(OFFSET)
                     && vector.getMinorType() != MinorType.DENSEUNION
-                    && vector.getMinorType() != MinorType.LISTVIEW)
+                    && vector.getMinorType() != MinorType.LISTVIEW
+                    && vector.getMinorType() != MinorType.LARGELISTVIEW)
                 ? valueCount + 1
                 : valueCount;
         for (int i = 0; i < bufferValueCount; i++) {
@@ -274,6 +276,7 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
           } else if (bufferType.equals(OFFSET)
               && vector.getValueCount() == 0
               && (vector.getMinorType() == MinorType.LARGELIST
+                  || vector.getMinorType() == MinorType.LARGELISTVIEW
                   || vector.getMinorType() == MinorType.LARGEVARBINARY
                   || vector.getMinorType() == MinorType.LARGEVARCHAR)) {
             // Empty vectors may not have allocated an offsets buffer
@@ -427,6 +430,10 @@ private void writeValueToGenerator(
           generator.writeNumber(
               buffer.getInt((long) index * BaseRepeatedValueViewVector.OFFSET_WIDTH));
           break;
+        case LARGELISTVIEW:
+          generator.writeNumber(
+              buffer.getInt((long) index * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+          break;
         case LARGELIST:
         case LARGEVARBINARY:
         case LARGEVARCHAR:
@@ -582,7 +589,12 @@ private void writeValueToGenerator(
           throw new UnsupportedOperationException("minor type: " + vector.getMinorType());
       }
     } else if (bufferType.equals(SIZE)) {
-      generator.writeNumber(buffer.getInt((long) index * BaseRepeatedValueViewVector.SIZE_WIDTH));
+      if (vector.getMinorType() == MinorType.LISTVIEW) {
+        generator.writeNumber(buffer.getInt((long) index * BaseRepeatedValueViewVector.SIZE_WIDTH));
+      } else {
+        generator.writeNumber(
+            buffer.getInt((long) index * BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+      }
     }
   }
 

diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -437,10 +437,18 @@ public void testRoundtripEmptyVector() throws Exception {
                 "list",
                 FieldType.nullable(ArrowType.List.INSTANCE),
                 Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
+            new Field(
+                "listview",
+                FieldType.nullable(ArrowType.ListView.INSTANCE),
+                Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
             new Field(
                 "largelist",
                 FieldType.nullable(ArrowType.LargeList.INSTANCE),
                 Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
+            new Field(
+                "largelistview",
+                FieldType.nullable(ArrowType.LargeListView.INSTANCE),
+                Collections.singletonList(Field.nullable("items", new ArrowType.Int(32, true)))),
             new Field(
                 "map",
                 FieldType.nullable(new ArrowType.Map(/*keyssorted*/ false)),