From 4d7c92c6a75261439021f828692e346cac9f345a Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 16 Oct 2017 18:59:46 -0700 Subject: [PATCH] Point the thrift id into NON_EXISTED_THRIFT_ID if the id resolver failed --- .../thrift/HiveThriftFieldIdResolver.java | 2 + .../hive/thrift/ThriftHiveRecordCursor.java | 46 +++++++++---------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java index 67a3088afb30..077a409fd2a2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java @@ -72,6 +72,7 @@ public HiveThriftFieldIdResolver(JsonNode root) this.root = root; } + @Override public short getThriftId(int hiveIndex) { if (root == null) { @@ -97,6 +98,7 @@ public short getThriftId(int hiveIndex) } } + @Override public ThriftFieldIdResolver getNestedResolver(int hiveIndex) { if (root == null) { diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java index 122ced137231..20927497fdc8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java @@ -81,6 +81,7 @@ class ThriftHiveRecordCursor implements RecordCursor { private static final Logger log = Logger.get(ThriftHiveRecordCursor.class); + private static final short NON_EXISTED_THRIFT_ID = (short) -1; private final RecordReader recordReader; private final K key; private final V value; @@ -165,7 +166,12 @@ public ThriftHiveRecordCursor( types[i] = typeManager.getType(column.getTypeSignature()); hiveTypes[i] = column.getHiveType(); hiveIndexs[i] = column.getHiveColumnIndex(); - thriftIds[i] = thriftFieldIdResolver.getThriftId(hiveIndexs[i]); + try { + thriftIds[i] = thriftFieldIdResolver.getThriftId(hiveIndexs[i]); + } + catch (PrestoException e) { + thriftIds[i] = NON_EXISTED_THRIFT_ID; + } } } @@ -546,6 +552,10 @@ private static Block getBlockObject(Type type, ThriftFieldIdResolver resolver, O private static Block serializeObject(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) { + if (object == null) { + requireNonNull(builder, "parent builder is null").appendNull(); + return null; + } if (!isStructuralType(type)) { serializePrimitive(type, resolver, builder, object, hiveStorageTimeZone); return null; @@ -564,12 +574,7 @@ else if (isRowType(type)) { private static Block serializeList(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) { - List list = (List) object; - if (list == null) { - requireNonNull(builder, "parent builder is null").appendNull(); - return null; - } - + List list = (List) requireNonNull(object, "object is null"); List typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); @@ -598,12 +603,7 @@ private static Block serializeList(Type type, ThriftFieldIdResolver resolver, Bl private static Block serializeMap(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) { - Map map = (Map) object; - if (map == null) { - requireNonNull(builder, "parent builder is null").appendNull(); - return null; - } - + Map map = (Map) requireNonNull(object, "object is null"); List typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); Type keyType = typeParameters.get(0); @@ -638,13 +638,8 @@ private static Block serializeMap(Type type, ThriftFieldIdResolver resolver, Blo private static Block serializeStruct(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) { - if (object == null) { - requireNonNull(builder, "parent builder is null").appendNull(); - return null; - } - + ThriftGenericRow structData = (ThriftGenericRow) requireNonNull(object, "object is null"); List typeParameters = type.getTypeParameters(); - ThriftGenericRow structData = (ThriftGenericRow) object; BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); @@ -655,7 +650,12 @@ private static Block serializeStruct(Type type, ThriftFieldIdResolver resolver, for (int i = 0; i < typeParameters.size(); i++) { Object fieldValue = structData.getFieldValueForThriftId(resolver.getThriftId(i)); - serializeObject(typeParameters.get(i), resolver.getNestedResolver(i), currentBuilder, fieldValue, hiveStorageTimeZone); + if (fieldValue == null) { + currentBuilder.appendNull(); + } + else { + serializeObject(typeParameters.get(i), resolver.getNestedResolver(i), currentBuilder, fieldValue, hiveStorageTimeZone); + } } if (builder != null) { @@ -671,11 +671,7 @@ private static Block serializeStruct(Type type, ThriftFieldIdResolver resolver, private static void serializePrimitive(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) { requireNonNull(builder, "parent builder is null"); - - if (object == null) { - builder.appendNull(); - return; - } + requireNonNull(object, "object is null"); if (BOOLEAN.equals(type)) { BOOLEAN.writeBoolean(builder, (Boolean) object);