From cc95c27cd3965776d96df5fbb7937ab159715a35 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 25 Sep 2015 15:25:29 -0700 Subject: [PATCH 001/151] Ignore if the partitions have no corresponfing hdfs valid path. --- .../java/com/facebook/presto/hive/util/HiveFileIterator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java index 160f1dc0e964..077711b17982 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java @@ -33,7 +33,6 @@ import java.util.Properties; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; import static java.util.Objects.requireNonNull; public class HiveFileIterator @@ -94,7 +93,8 @@ protected LocatedFileStatus computeNext() return endOfData(); } catch (FileNotFoundException e) { - throw new PrestoException(HIVE_FILE_NOT_FOUND, "Partition location does not exist: " + path); + // We are okay if the path does not exist. + return endOfData(); } catch (IOException e) { throw new PrestoException(HIVE_FILESYSTEM_ERROR, e); From bce8187ec60304f2f30262ab710b571885b3bdb9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 10:40:07 -0700 Subject: [PATCH 002/151] Append nulls for missing values in Parquet. Parquet only calls converts for which it found the values. The missing values are not reported. The BlockBuilder must be appended with nulls for the missing values based on fieldIndex of the currently read value by Parquet. --- .../hive/parquet/ParquetHiveRecordCursor.java | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java index 8e9cf0290ff8..c7d8c0c7901f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetHiveRecordCursor.java @@ -446,7 +446,7 @@ public PrestoReadSupport(boolean useParquetColumnNames, List c converters.add(new ParquetPrimitiveColumnConverter(i)); } else { - converters.add(new ParquetColumnConverter(createGroupConverter(types[i], parquetType.getName(), parquetType), i)); + converters.add(new ParquetColumnConverter(createGroupConverter(types[i], parquetType.getName(), parquetType, i), i)); } } } @@ -674,25 +674,25 @@ private abstract static class GroupedConverter public abstract Block getBlock(); } - private static BlockConverter createConverter(Type prestoType, String columnName, parquet.schema.Type parquetType) + private static BlockConverter createConverter(Type prestoType, String columnName, parquet.schema.Type parquetType, int fieldIndex) { if (parquetType.isPrimitive()) { - return new ParquetPrimitiveConverter(prestoType); + return new ParquetPrimitiveConverter(prestoType, fieldIndex); } - return createGroupConverter(prestoType, columnName, parquetType); + return createGroupConverter(prestoType, columnName, parquetType, fieldIndex); } - private static GroupedConverter createGroupConverter(Type prestoType, String columnName, parquet.schema.Type parquetType) + private static GroupedConverter createGroupConverter(Type prestoType, String columnName, parquet.schema.Type parquetType, int fieldIndex) { GroupType groupType = parquetType.asGroupType(); switch (prestoType.getTypeSignature().getBase()) { case ARRAY: - return new ParquetListConverter(prestoType, columnName, groupType); + return new ParquetListConverter(prestoType, columnName, groupType, fieldIndex); case MAP: - return new ParquetMapConverter(prestoType, columnName, groupType); + return new ParquetMapConverter(prestoType, columnName, groupType, fieldIndex); case ROW: - return new ParquetStructConverter(prestoType, columnName, groupType); + return new ParquetStructConverter(prestoType, columnName, groupType, fieldIndex); default: throw new IllegalArgumentException("Column " + columnName + " type " + parquetType.getOriginalType() + " not supported"); } @@ -705,13 +705,14 @@ private static class ParquetStructConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type rowType; + private final int fieldIndex; private final List converters; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetStructConverter(Type prestoType, String columnName, GroupType entryType) + public ParquetStructConverter(Type prestoType, String columnName, GroupType entryType, int fieldIndex) { checkArgument(ROW.equals(prestoType.getTypeSignature().getBase())); List prestoTypeParameters = prestoType.getTypeParameters(); @@ -719,11 +720,12 @@ public ParquetStructConverter(Type prestoType, String columnName, GroupType entr checkArgument(prestoTypeParameters.size() == fieldTypes.size()); this.rowType = prestoType; + this.fieldIndex = fieldIndex; ImmutableList.Builder converters = ImmutableList.builder(); for (int i = 0; i < prestoTypeParameters.size(); i++) { parquet.schema.Type fieldType = fieldTypes.get(i); - converters.add(createConverter(prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType)); + converters.add(createConverter(prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType, i)); } this.converters = converters.build(); } @@ -750,6 +752,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } for (BlockConverter converter : converters) { @@ -763,6 +768,10 @@ public void end() for (BlockConverter converter : converters) { converter.afterValue(); } + while (currentEntryBuilder.getPositionCount() < converters.size()) { + currentEntryBuilder.appendNull(); + } + if (builder == null) { nullBuilder.closeEntry(); } @@ -791,13 +800,14 @@ private static class ParquetListConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type arrayType; + private final int fieldIndex; private final BlockConverter elementConverter; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetListConverter(Type prestoType, String columnName, GroupType listType) + public ParquetListConverter(Type prestoType, String columnName, GroupType listType, int fieldIndex) { checkArgument(listType.getFieldCount() == 1, "Expected LIST column '%s' to only have one field, but has %s fields", @@ -806,6 +816,7 @@ public ParquetListConverter(Type prestoType, String columnName, GroupType listTy checkArgument(ARRAY.equals(prestoType.getTypeSignature().getBase())); this.arrayType = prestoType; + this.fieldIndex = fieldIndex; // The Parquet specification requires that the element value of a // LIST type be wrapped in an inner repeated group, like so: @@ -821,7 +832,7 @@ public ParquetListConverter(Type prestoType, String columnName, GroupType listTy // documentation at http://git.io/vOpNz. parquet.schema.Type elementType = listType.getType(0); if (isElementType(elementType, listType.getName())) { - elementConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".element", elementType); + elementConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".element", elementType, 0); } else { elementConverter = new ParquetListEntryConverter(prestoType.getTypeParameters().get(0), columnName, elementType.asGroupType()); @@ -875,6 +886,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } elementConverter.beforeValue(currentEntryBuilder); @@ -926,7 +940,7 @@ public ParquetListEntryConverter(Type prestoType, String columnName, GroupType e columnName, elementType.getFieldCount()); - elementConverter = createConverter(prestoType, columnName + ".element", elementType.getType(0)); + elementConverter = createConverter(prestoType, columnName + ".element", elementType.getType(0), 0); } @Override @@ -969,13 +983,14 @@ private static class ParquetMapConverter private static final int NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD = 32768; private final Type mapType; + private final int fieldIndex; private final ParquetMapEntryConverter entryConverter; private BlockBuilder builder; private BlockBuilder nullBuilder; // used internally when builder is set to null private BlockBuilder currentEntryBuilder; - public ParquetMapConverter(Type type, String columnName, GroupType mapType) + public ParquetMapConverter(Type type, String columnName, GroupType mapType, int fieldIndex) { checkArgument(mapType.getFieldCount() == 1, "Expected MAP column '%s' to only have one field, but has %s fields", @@ -983,6 +998,7 @@ public ParquetMapConverter(Type type, String columnName, GroupType mapType) mapType.getFieldCount()); this.mapType = type; + this.fieldIndex = fieldIndex; parquet.schema.Type entryType = mapType.getFields().get(0); @@ -1014,6 +1030,9 @@ public void start() currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } currentEntryBuilder = builder.beginBlockEntry(); } entryConverter.beforeValue(currentEntryBuilder); @@ -1084,8 +1103,8 @@ public ParquetMapEntryConverter(Type prestoType, String columnName, GroupType en columnName, entryGroupType.getType(0)); - keyConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".key", entryGroupType.getFields().get(0)); - valueConverter = createConverter(prestoType.getTypeParameters().get(1), columnName + ".value", entryGroupType.getFields().get(1)); + keyConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".key", entryGroupType.getFields().get(0), 0); + valueConverter = createConverter(prestoType.getTypeParameters().get(1), columnName + ".value", entryGroupType.getFields().get(1), 1); } @Override @@ -1131,12 +1150,14 @@ private static class ParquetPrimitiveConverter implements BlockConverter { private final Type type; + private final int fieldIndex; private BlockBuilder builder; private boolean wroteValue; - public ParquetPrimitiveConverter(Type type) + public ParquetPrimitiveConverter(Type type, int fieldIndex) { this.type = type; + this.fieldIndex = fieldIndex; } @Override @@ -1149,11 +1170,13 @@ public void beforeValue(BlockBuilder builder) @Override public void afterValue() { - if (wroteValue) { - return; - } + } - builder.appendNull(); + private void addMissingValues() + { + while (builder.getPositionCount() < fieldIndex) { + builder.appendNull(); + } } @Override @@ -1187,6 +1210,7 @@ public void addValueFromDictionary(int dictionaryId) @Override public void addBoolean(boolean value) { + addMissingValues(); BOOLEAN.writeBoolean(builder, value); wroteValue = true; } @@ -1194,6 +1218,7 @@ public void addBoolean(boolean value) @Override public void addDouble(double value) { + addMissingValues(); DOUBLE.writeDouble(builder, value); wroteValue = true; } @@ -1201,6 +1226,7 @@ public void addDouble(double value) @Override public void addLong(long value) { + addMissingValues(); BIGINT.writeLong(builder, value); wroteValue = true; } @@ -1208,6 +1234,7 @@ public void addLong(long value) @Override public void addBinary(Binary value) { + addMissingValues(); if (type == TIMESTAMP) { builder.writeLong(ParquetTimestampUtils.getTimestampMillis(value)).closeEntry(); } @@ -1220,6 +1247,7 @@ public void addBinary(Binary value) @Override public void addFloat(float value) { + addMissingValues(); DOUBLE.writeDouble(builder, value); wroteValue = true; } @@ -1227,6 +1255,7 @@ public void addFloat(float value) @Override public void addInt(int value) { + addMissingValues(); BIGINT.writeLong(builder, value); wroteValue = true; } From 3bbcbb77181ef0d9a543dcbcceb988fcfad0de39 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 11:27:09 -0700 Subject: [PATCH 003/151] [maven-release-plugin] prepare for tw-0.1 release. --- pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 35 files changed, 35 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 1ce3b8933afa..2e906d46ab60 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT pom presto-root diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index bf0ea89449f7..becc2aad6552 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index bea67680cd7e..fcc11ed9be7d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index eea12d2ea187..3dca897d8d00 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 4bd015fe78bf..5f68c004c6e3 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index e4f4e3b4f115..b4276f9a2118 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 812de79ce39b..e0fa76890e65 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 53672dfabdb5..33caecd03adb 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 460610625774..22a19de7cdf6 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 8b44291c6a43..c805d506939e 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index cac75be155a5..4c764cc832c3 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 3caded1f3f89..f775951da3c0 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 01deea780ac4..28bba6912311 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 9028adcbcaf7..0b47093c5d7a 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a0edca6dd537..40bfd35c4cd0 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f6939498365..198b7dba9b74 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 4dbffd33d13f..11364ccb4163 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index bf3b0a07e618..727e8efad01c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index a8b129bfcf72..f2a2d446e7d3 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4f2b30229f50..d4678be9d7e6 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 604afe0f5e1f..3005cbee88c0 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 6fe0e61a7972..2d9f2539d18a 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index eef17295c589..f99df2a20914 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 755c12ab097f..2e3f97bf0bf4 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-postgresql diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 83d83e4accff..72ce96885268 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 25555b4a25d2..6d344eada7e4 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 8d44339c6c78..1dc2d16c7fb8 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 33e66de413fc..30e1a7c0a1e8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index ac51dbc823c4..468278f4a955 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index ad8a095f4f39..2e9c6a0c9c13 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 40bafdb094d3..05fe47646434 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index a321b43f5771..d30e0605f888 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 44501fcdf2bf..9ec7a73cd2da 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index ea19dd90dcf6..932924f8b8ed 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 3491c458ba4d..8cade2a0ed97 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.120-SNAPSHOT + 0.119-tw-0.1-SNAPSHOT presto-verifier From ca8560fde9575f6d45dd6b2d24c5775105c5bf32 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 1 Oct 2015 11:42:10 -0700 Subject: [PATCH 004/151] [maven-release-plugin] prepare for release 0.119-tw-0.1. --- pom.xml | 8 ++++---- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 35 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2e906d46ab60..73f572f41e55 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 pom presto-root @@ -28,9 +28,9 @@ - scm:git:git://github.com/facebook/presto.git - https://github.com/facebook/presto - HEAD + scm:git:git://github.com/twitter-forks/presto.git + https://github.com/twitter-forks/presto + 0.119-tw-0.1 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index becc2aad6552..bf85b92c69ab 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index fcc11ed9be7d..dabd15b59a88 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 3dca897d8d00..7900d3a15d20 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5f68c004c6e3..1d62047b7b8c 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index b4276f9a2118..85b7806119be 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e0fa76890e65..53f55d5d5c91 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 33caecd03adb..242560f261a8 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 22a19de7cdf6..364b9e3e3825 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index c805d506939e..07d4930b6b8b 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 4c764cc832c3..1af8e55a0d46 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f775951da3c0..fa858095ac23 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 28bba6912311..99226c90a798 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 0b47093c5d7a..b0a353cd0575 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 40bfd35c4cd0..b94aaa9676fd 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 198b7dba9b74..f4a2653f06a2 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 11364ccb4163..5c1c1418f45c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 727e8efad01c..029add19febd 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index f2a2d446e7d3..5bdea99bb96a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index d4678be9d7e6..911c908772bb 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 3005cbee88c0..21d1697a581b 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 2d9f2539d18a..5730006d2752 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index f99df2a20914..f14de7f6b725 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 2e3f97bf0bf4..a43d0b35f4a0 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-postgresql diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 72ce96885268..9a8940684940 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 6d344eada7e4..506e6e6fb4d8 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 1dc2d16c7fb8..da5609eb349d 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 30e1a7c0a1e8..2c9e1c9a8c72 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 468278f4a955..9c258c8b88dc 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 2e9c6a0c9c13..b41a6498d846 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 05fe47646434..1ee4d99cfda8 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index d30e0605f888..661f3db9a240 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 9ec7a73cd2da..f76a75aaba9b 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 932924f8b8ed..6b3161861e6f 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 8cade2a0ed97..c22f6a70eb88 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.119-tw-0.1-SNAPSHOT + 0.119-tw-0.1 presto-verifier From ec109d1dce3d3d93a9335f3750c8bd3c424bb22e Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 2 Oct 2015 15:47:29 -0700 Subject: [PATCH 005/151] Compare types ignoring cases. This is not complete solution and should be reverted once #2863 is resolved. --- .../java/com/facebook/presto/metadata/FunctionRegistry.java | 2 +- .../com/facebook/presto/operator/scalar/RowFieldReference.java | 3 +-- .../com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java | 2 +- .../main/java/com/facebook/presto/type/RowParametricType.java | 2 +- .../main/java/com/facebook/presto/spi/type/TypeSignature.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java index 8c4d9a48af1d..45bff811f1a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java @@ -449,7 +449,7 @@ public FunctionInfo resolveFunction(QualifiedName name, List para RowType rowType = RowParametricType.ROW.createType(resolveTypes(typeSignature.getParameters(), typeManager), typeSignature.getLiteralParameters()); // search for exact match for (ParametricFunction function : RowParametricType.ROW.createFunctions(rowType)) { - if (!function.getSignature().getName().equals(name.toString())) { + if (!function.getSignature().getName().equalsIgnoreCase(name.toString())) { continue; } Map boundTypeParameters = function.getSignature().bindTypeParameters(resolvedTypes, false, typeManager); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java index f4a94125f49a..4291e7970fc9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowFieldReference.java @@ -28,7 +28,6 @@ import java.lang.invoke.MethodHandle; import java.util.Map; -import java.util.Optional; import static com.facebook.presto.sql.QueryUtil.mangleFieldReference; import static com.facebook.presto.type.RowType.RowField; @@ -59,7 +58,7 @@ public RowFieldReference(RowType type, String fieldName) Type returnType = null; int index = 0; for (RowField field : type.getFields()) { - if (field.getName().equals(Optional.of(fieldName))) { + if (field.getName().get().equalsIgnoreCase(fieldName)) { returnType = field.getType(); break; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index 22b3d3559ec0..6239c559d708 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -318,7 +318,7 @@ private Type tryVisitRowFieldAccessor(QualifiedNameReference node) RowType rowType = checkType(field.getType(), RowType.class, "field.getType()"); Type rowFieldType = null; for (RowField rowField : rowType.getFields()) { - if (rowField.getName().equals(Optional.of(node.getName().getSuffix()))) { + if (rowField.getName().get().equalsIgnoreCase(node.getName().getSuffix())) { rowFieldType = rowField.getType(); break; } diff --git a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java index 5a4804d99868..f175754d696a 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java +++ b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java @@ -65,7 +65,7 @@ public List createFunctions(Type type) ImmutableList.Builder builder = ImmutableList.builder(); for (RowField field : rowType.getFields()) { field.getName() - .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getName().get()))); + .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getName().get().toLowerCase()))); } return builder.build(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java index 8e5d924fe399..75ffe4906f81 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java @@ -189,7 +189,7 @@ private static Object parseLiteral(String literal) { if (literal.startsWith("'") || literal.endsWith("'")) { checkArgument(literal.startsWith("'") && literal.endsWith("'"), "Bad literal: '%s'", literal); - return literal.substring(1, literal.length() - 1); + return literal.substring(1, literal.length() - 1).toLowerCase(); } else { return Long.parseLong(literal); From c76b42e910bfbf989078171587dc6700f2757275 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 9 Oct 2015 12:43:09 -0700 Subject: [PATCH 006/151] Revert "Reduce allocation of hadoop configuration objects" This reverts commit 7365f1ad3a75b478bcdecca65676d7857b843699. This blocked us from reading from mountable clusters. --- .../com/facebook/presto/hive/HiveHdfsConfiguration.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java index 8ad5acc65c9f..dc6d0f46f3ed 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java @@ -18,25 +18,19 @@ import javax.inject.Inject; import java.net.URI; -import java.util.Map; import static java.util.Objects.requireNonNull; public class HiveHdfsConfiguration implements HdfsConfiguration { - private static final Configuration DEFAULT_CONFIGURATION = new Configuration(); - @SuppressWarnings("ThreadLocalNotStaticFinal") private final ThreadLocal hadoopConfiguration = new ThreadLocal() { @Override protected Configuration initialValue() { - Configuration config = new Configuration(false); - for (Map.Entry entry : DEFAULT_CONFIGURATION) { - config.set(entry.getKey(), entry.getValue()); - } + Configuration config = new Configuration(); updater.updateConfiguration(config); return config; } From f0d9ab0d9780de736d7c6893ccf4960a57dc8fba Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 14:25:46 -0700 Subject: [PATCH 007/151] Bump up zookeeper version to 3.4.6 --- pom.xml | 10 +++++++++- .../facebook/presto/kafka/util/EmbeddedZookeeper.java | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 301487adddb2..c845b238ec5b 100644 --- a/pom.xml +++ b/pom.xml @@ -624,7 +624,7 @@ org.apache.zookeeper zookeeper - 3.3.6 + 3.4.6 junit @@ -634,6 +634,14 @@ log4j log4j + + org.slf4j + slf4j-jdk14 + + + org.slf4j + slf4j-log4j12 + diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java index fddf5c845746..0f319cee1cbc 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java @@ -14,7 +14,7 @@ package com.facebook.presto.kafka.util; import com.google.common.io.Files; -import org.apache.zookeeper.server.NIOServerCnxn; +import org.apache.zookeeper.server.NIOServerCnxnFactory; import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; @@ -32,7 +32,7 @@ public class EmbeddedZookeeper private final int port; private final File zkDataDir; private final ZooKeeperServer zkServer; - private final NIOServerCnxn.Factory cnxnFactory; + private final NIOServerCnxnFactory cnxnFactory; private final AtomicBoolean started = new AtomicBoolean(); private final AtomicBoolean stopped = new AtomicBoolean(); @@ -53,7 +53,8 @@ public EmbeddedZookeeper(int port) FileTxnSnapLog ftxn = new FileTxnSnapLog(zkDataDir, zkDataDir); zkServer.setTxnLogFactory(ftxn); - cnxnFactory = new NIOServerCnxn.Factory(new InetSocketAddress(this.port), 0); + cnxnFactory = new NIOServerCnxnFactory(); + cnxnFactory.configure(new InetSocketAddress(this.port), 0); } public void start() From 3bbabcefe030d9794fd329a4ec927f19cb75faca Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 14:47:57 -0700 Subject: [PATCH 008/151] Read metastore info from zookeeper. --- presto-hive/pom.xml | 24 ++++ .../presto/hive/HiveClientModule.java | 4 +- .../hive/ZookeeperMetastoreMonitor.java | 125 ++++++++++++++++++ .../hive/ZookeeperServersetHiveCluster.java | 67 ++++++++++ .../ZookeeperServersetMetastoreConfig.java | 87 ++++++++++++ ...TestZookeeperServersetMetastoreConfig.java | 55 ++++++++ 6 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 94423d7afb8b..a8f459673f39 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -43,6 +43,24 @@ hive-apache + + org.apache.curator + curator-recipes + 2.8.0 + + + + org.apache.curator + curator-framework + 2.8.0 + + + + org.apache.curator + curator-client + 2.8.0 + + org.apache.thrift libthrift @@ -88,6 +106,12 @@ configuration + + com.googlecode.json-simple + json-simple + 1.1 + + com.google.guava guava diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index 6a2e659ae27c..cc540c3076ea 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -90,8 +90,8 @@ public void configure(Binder binder) newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class)); binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); - binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(StaticMetastoreConfig.class); + binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); binder.bind(TypeManager.class).toInstance(typeManager); binder.bind(PageIndexerFactory.class).toInstance(pageIndexerFactory); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java new file mode 100644 index 000000000000..002eefe5cc13 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.common.net.HostAndPort; + +import io.airlift.log.Logger; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.utils.ZKPaths; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public class ZookeeperMetastoreMonitor implements PathChildrenCacheListener +{ + public static final Logger log = Logger.get(ZookeeperMetastoreMonitor.class); + private CuratorFramework client; + private PathChildrenCache cache; + private ConcurrentMap servers; // (Node_Name->HostAndPort) + + public ZookeeperMetastoreMonitor(String zkServer, String watchPath, int maxRetries, int retrySleepTime) + throws Exception + { + client = CuratorFrameworkFactory.newClient(zkServer, new ExponentialBackoffRetry(retrySleepTime, maxRetries)); + client.start(); + + cache = new PathChildrenCache(client, watchPath, true); // true indicating cache node contents in addition to the stat + try { + cache.start(); + } + catch (Exception ex) { + throw new RuntimeException("Curator PathCache Creation failed: " + ex.getMessage()); + } + + cache.getListenable().addListener(this); + servers = new ConcurrentHashMap<>(); + } + + public void close() + { + client.close(); + + try { + cache.close(); + } + catch (IOException ex) { + // do nothing + } + } + + public List getServers() + { + return servers.values().stream().collect(Collectors.toList()); + } + + private HostAndPort deserialize(byte[] bytes) + { + String serviceEndpoint = "serviceEndpoint"; + JSONObject data = (JSONObject) JSONValue.parse(new String(bytes)); + if (data != null && data.containsKey(serviceEndpoint)) { + Map hostPortMap = (Map) data.get(serviceEndpoint); + String host = hostPortMap.get("host").toString(); + int port = Integer.parseInt(hostPortMap.get("port").toString()); + return HostAndPort.fromParts(host, port); + } + else { + log.warn("failed to deserialize child node data"); + throw new IllegalArgumentException("No host:port found"); + } + } + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception + { + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); + + switch (event.getType()) { + case CHILD_ADDED: { + HostAndPort hostPort = deserialize(event.getData().getData()); + log.info("child added: " + node + ": " + hostPort); + servers.put(node, hostPort); + break; + } + + case CHILD_UPDATED: { + HostAndPort hostPort = deserialize(event.getData().getData()); + log.info("child updated: " + node + ": " + hostPort); + servers.put(node, hostPort); + break; + } + + case CHILD_REMOVED: { + log.info("child removed: " + node); + servers.remove(node); + break; + } + + default: + log.info("connection state changed: " + node); + break; + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java new file mode 100644 index 000000000000..57b44d5bc05e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.PrestoException; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.apache.thrift.transport.TTransportException; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static java.util.Objects.requireNonNull; + +/** + * Created by smittal on 10/9/15. + */ +public class ZookeeperServersetHiveCluster + implements HiveCluster +{ + private static final Logger log = Logger.get(ZookeeperServersetHiveCluster.class); + private final HiveMetastoreClientFactory clientFactory; + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + + @Inject + public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, HiveMetastoreClientFactory clientFactory) + throws Exception + { + String zkServerHostAndPort = requireNonNull(config.getZookeeperServerHostAndPort(), "zkServerHostAndPort is null"); + String zkMetastorePath = requireNonNull(config.getZookeeperMetastorePath(), "zkMetastorePath is null"); + int zkRetries = requireNonNull(config.getZookeeperMaxRetries(), "zkMaxRetried is null"); + int zkRetrySleepTime = requireNonNull(config.getZookeeperRetrySleepTime(), "zkRetrySleepTime is null"); + this.clientFactory = requireNonNull(clientFactory, "clientFactory is null"); + this.zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServerHostAndPort, zkMetastorePath, zkRetries, zkRetrySleepTime); + } + + @Override + public HiveMetastoreClient createMetastoreClient() + { + List metastores = zkMetastoreMonitor.getServers(); + TTransportException lastException = null; + for (HostAndPort metastore : metastores) { + try { + log.info("Connecting to metastore at: " + metastore.toString()); + return clientFactory.create(metastore.getHostText(), metastore.getPort()); + } + catch (TTransportException e) { + lastException = e; + } + } + + throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore", lastException); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java new file mode 100644 index 000000000000..339131fa3b9b --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * Created by smittal on 10/9/15. + */ +public class ZookeeperServersetMetastoreConfig +{ + private String zookeeperServerHostAndPort; + private String zookeeperMetastorePath; + private int zookeeperRetrySleepTime = 500; // ms + private int zookeeperMaxRetries = 3; + + @NotNull + public String getZookeeperServerHostAndPort() + { + return zookeeperServerHostAndPort; + } + + @Config("hive.metastore.zookeeper.uri") + @ConfigDescription("Zookeeper Host and Port") + public ZookeeperServersetMetastoreConfig setZookeeperServerHostAndPort(String zookeeperServerHostAndPort) + { + this.zookeeperServerHostAndPort = zookeeperServerHostAndPort; + return this; + } + + @NotNull + public String getZookeeperMetastorePath() + { + return zookeeperMetastorePath; + } + + @Config("hive.metastore.zookeeper.path") + @ConfigDescription("Hive metastore Zookeeper path") + public ZookeeperServersetMetastoreConfig setZookeeperMetastorePath(String zkPath) + { + this.zookeeperMetastorePath = zkPath; + return this; + } + + @NotNull + public int getZookeeperRetrySleepTime() + { + return zookeeperRetrySleepTime; + } + + @Config("hive.metastore.zookeeper.retry.sleeptime") + @ConfigDescription("Zookeeper sleep time between reties") + public ZookeeperServersetMetastoreConfig setZookeeperRetrySleepTime(int zookeeperRetrySleepTime) + { + this.zookeeperRetrySleepTime = zookeeperRetrySleepTime; + return this; + } + + @Min(1) + public int getZookeeperMaxRetries() + { + return zookeeperMaxRetries; + } + + @Config("hive.metastore.zookeeper.max.retries") + @ConfigDescription("Zookeeper max reties") + public ZookeeperServersetMetastoreConfig setZookeeperMaxRetries(int zookeeperMaxRetries) + { + this.zookeeperMaxRetries = zookeeperMaxRetries; + return this; + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java new file mode 100644 index 000000000000..bb1117697296 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestZookeeperServersetMetastoreConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(ZookeeperServersetMetastoreConfig.class) + .setZookeeperMaxRetries(3) + .setZookeeperRetrySleepTime(500) + .setZookeeperMetastorePath(null) + .setZookeeperServerHostAndPort(null)); + } + + @Test + public void testExplicitPropertyMappingsSingleMetastore() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.zookeeper.uri", "localhost:2181") + .put("hive.metastore.zookeeper.path", "/zookeeper/path/") + .put("hive.metastore.zookeeper.retry.sleeptime", "200") + .put("hive.metastore.zookeeper.max.retries", "2") + .build(); + + ZookeeperServersetMetastoreConfig expected = new ZookeeperServersetMetastoreConfig() + .setZookeeperServerHostAndPort("localhost:2181") + .setZookeeperMetastorePath("/zookeeper/path/") + .setZookeeperRetrySleepTime(200) + .setZookeeperMaxRetries(2); + + assertFullMapping(properties, expected); + } +} From 202e620ec7f7743d0d0910573eef607c2eb2cd1f Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 13 Oct 2015 13:52:37 -0700 Subject: [PATCH 009/151] Add some tests and fix injection issue. --- presto-hive/pom.xml | 20 +++ .../presto/hive/HiveClientModule.java | 2 - .../presto/hive/HiveConnectorFactory.java | 8 + .../hive/MetastoreStaticClusterModule.java | 31 ++++ .../hive/MetastoreZkDiscoveryBasedModule.java | 31 ++++ .../ZookeeperServersetMetastoreConfig.java | 2 - .../hive/TestZookeeperMetastoreMonitor.java | 157 ++++++++++++++++++ .../facebook/presto/hive/util/TestUtils.java | 30 ++++ 8 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a8f459673f39..e75baa539500 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -61,6 +61,26 @@ 2.8.0 + + org.apache.curator + curator-test + 2.8.0 + test + + + + org.apache.zookeeper + zookeeper + 3.4.6 + test + + + + com.101tec + zkclient + test + + org.apache.thrift libthrift diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index cc540c3076ea..bc48cbfc7f0f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -90,8 +90,6 @@ public void configure(Binder binder) newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class)); binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); - binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); binder.bind(TypeManager.class).toInstance(typeManager); binder.bind(PageIndexerFactory.class).toInstance(pageIndexerFactory); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index 2725e938586e..922875f49317 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -93,6 +93,14 @@ public Connector create(String connectorId, Map config) new MBeanModule(), new JsonModule(), new HiveClientModule(connectorId, metastore, typeManager, pageIndexerFactory), + installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() == null, + new MetastoreStaticClusterModule()), + installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() != null, + new MetastoreZkDiscoveryBasedModule()), installModuleIf( SecurityConfig.class, security -> "none".equalsIgnoreCase(security.getSecuritySystem()), diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java new file mode 100644 index 000000000000..18bfa3d1eff1 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreStaticClusterModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(StaticMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java new file mode 100644 index 000000000000..a4d84813ab76 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreZkDiscoveryBasedModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java index 339131fa3b9b..60867659d906 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -29,7 +29,6 @@ public class ZookeeperServersetMetastoreConfig private int zookeeperRetrySleepTime = 500; // ms private int zookeeperMaxRetries = 3; - @NotNull public String getZookeeperServerHostAndPort() { return zookeeperServerHostAndPort; @@ -43,7 +42,6 @@ public ZookeeperServersetMetastoreConfig setZookeeperServerHostAndPort(String zo return this; } - @NotNull public String getZookeeperMetastorePath() { return zookeeperMetastorePath; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java new file mode 100644 index 000000000000..5383e2e64198 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; +import com.facebook.presto.hive.util.TestUtils; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; +import org.apache.curator.test.TestingServer; +import org.json.simple.JSONObject; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertTrue; + +public class TestZookeeperMetastoreMonitor +{ + private static final Logger log = Logger.get(TestZookeeperMetastoreMonitor.class); + + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + private TestingServer zkServer; + private ZkClient zkClient; + private final String zkPath = "/metastores"; + + public TestZookeeperMetastoreMonitor() + throws Exception + { + zkServer = new TestingServer(TestUtils.findUnusedPort()); + zkClient = new ZkClient(zkServer.getConnectString(), 30_000, 30_000); + + // Set the serializer + zkClient.setZkSerializer(new ZkSerializer() { + @Override + public byte[] serialize(Object o) throws ZkMarshallingError + { + try { + return o.toString().getBytes("UTF-8"); + } + catch (Exception e) { + log.warn("Exception in serializing " + e); + } + return "".getBytes(); + } + + @Override + public Object deserialize(byte[] bytes) throws ZkMarshallingError + { + return null; + } + }); + } + + @AfterClass + public void destroy() + throws IOException + { + zkMetastoreMonitor.close(); + zkClient.close(); + zkServer.close(); + + } + + @BeforeTest + public void setUp() + throws Exception + { + log.info("Cleaning up zookeeper"); + zkClient.getChildren("/").stream() + .filter(child -> !child.equals("zookeeper")) + .forEach(child -> zkClient.deleteRecursive("/" + child)); + + zkClient.unsubscribeAll(); + + zkClient.createPersistent(zkPath); + zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServer.getConnectString(), zkPath, 3, 500); + } + + @Test + public void testGetServers() throws Exception + { + List servers; + List expected; + assertTrue(zkMetastoreMonitor.getServers().isEmpty()); + + addServerToZk("nameNode1", "host1", 10001); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + addServerToZk("nameNode2", "host2", 10002); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10002)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Change value of an existing name node + addServerToZk("nameNode2", "host2", 10003); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Delete an existing name node + zkClient.delete(getPathForNameNode("nameNode1")); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers), servers.toString()); + } + + private void addServerToZk(String nameNode, String host, int port) + { + JSONObject serviceEndpoint = new JSONObject(); + serviceEndpoint.put("host", host); + serviceEndpoint.put("port", port); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("serviceEndpoint", serviceEndpoint); + + String path = getPathForNameNode(nameNode); + + if (!zkClient.exists(path)) { + zkClient.createPersistent(path, jsonObject.toJSONString()); + } + else { + zkClient.writeData(path, jsonObject.toJSONString()); + } + } + + private String getPathForNameNode(String nameNode) + { + return zkPath + "/" + nameNode; + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java new file mode 100644 index 000000000000..4315bac667c2 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.util; + +import java.io.IOException; +import java.net.ServerSocket; + +public final class TestUtils +{ + private TestUtils() {} + + public static int findUnusedPort() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } +} From 93f55bbf1ebda92e6f4bf4929ba36ef8dadcd292 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 14 Oct 2015 09:59:22 -0700 Subject: [PATCH 010/151] Remove Created by comments added by Idea. --- .../facebook/presto/hive/ZookeeperServersetHiveCluster.java | 3 --- .../presto/hive/ZookeeperServersetMetastoreConfig.java | 3 --- 2 files changed, 6 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java index 57b44d5bc05e..cbc434b74d2f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -25,9 +25,6 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static java.util.Objects.requireNonNull; -/** - * Created by smittal on 10/9/15. - */ public class ZookeeperServersetHiveCluster implements HiveCluster { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java index 60867659d906..44c6f9d19188 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java @@ -19,9 +19,6 @@ import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; -/** - * Created by smittal on 10/9/15. - */ public class ZookeeperServersetMetastoreConfig { private String zookeeperServerHostAndPort; From 608333956fd263a0e3e103b1c52221c4d33f45fd Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Thu, 22 Oct 2015 13:42:25 -0700 Subject: [PATCH 011/151] Fix some warnings after merging latest master. --- .../facebook/presto/hive/ZookeeperServersetHiveCluster.java | 1 + .../facebook/presto/hive/TestZookeeperMetastoreMonitor.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java index cbc434b74d2f..48f869ec8ed0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.HiveMetastoreClient; import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java index 5383e2e64198..9ccf999edac4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.TimeUnit; @@ -53,7 +54,7 @@ public TestZookeeperMetastoreMonitor() public byte[] serialize(Object o) throws ZkMarshallingError { try { - return o.toString().getBytes("UTF-8"); + return o.toString().getBytes(StandardCharsets.UTF_8); } catch (Exception e) { log.warn("Exception in serializing " + e); From 972d5d166bc8174c9cef50b1066f6efc9a07a02b Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 23 Oct 2015 11:39:02 -0700 Subject: [PATCH 012/151] Create ParquetHiveRecordCursor as user, so that hdfs reads happen as that user. --- .../parquet/ParquetRecordCursorProvider.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java index eb6bb880bed3..3854449498da 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java @@ -27,10 +27,12 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -88,20 +90,26 @@ public Optional createHiveRecordCursor( throw new IllegalArgumentException("Can not read Parquet column: " + unsupportedColumns); } - return Optional.of(new ParquetHiveRecordCursor( - configuration, - path, - start, - length, - schema, - partitionKeys, - columns, - useParquetColumnNames, - hiveStorageTimeZone, - typeManager, - isParquetPredicatePushdownEnabled(session), - effectivePredicate - )); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + try { + return ugi.doAs((PrivilegedExceptionAction>) () -> Optional.of(new ParquetHiveRecordCursor( + configuration, + path, + start, + length, + schema, + partitionKeys, + columns, + useParquetColumnNames, + hiveStorageTimeZone, + typeManager, + isParquetPredicatePushdownEnabled(session), + effectivePredicate + ))); + } + catch (Exception e) { + throw new RuntimeException(e); + } } private static Predicate isParquetSupportedType() From d01400182645e82f733760a2d938273c49dd5a88 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:20:43 -0700 Subject: [PATCH 013/151] Propagate user session information throughout hive metastore. --- .../facebook/presto/hive/HiveMetadata.java | 85 ++++++++++--------- .../facebook/presto/hive/HivePageSink.java | 11 ++- .../presto/hive/HivePageSinkProvider.java | 7 +- .../presto/hive/HivePartitionManager.java | 12 +-- .../presto/hive/HiveSplitManager.java | 7 +- .../facebook/presto/hive/HiveWriteUtils.java | 9 +- .../hive/metastore/CachingHiveMetastore.java | 30 +++---- .../presto/hive/metastore/HiveMetastore.java | 34 ++++---- .../presto/hive/AbstractTestHiveClient.java | 8 +- .../presto/hive/AbstractTestHiveClientS3.java | 20 ++--- .../hive/metastore/InMemoryHiveMetastore.java | 32 +++---- .../metastore/TestCachingHiveMetastore.java | 59 ++++++------- 12 files changed, 164 insertions(+), 150 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index c8e48aa7e21b..a3ad7180b9de 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -231,14 +231,14 @@ public HivePartitionManager getPartitionManager() @Override public List listSchemaNames(ConnectorSession session) { - return metastore.getAllDatabases(); + return metastore.getAllDatabases(session.getUser()); } @Override public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { requireNonNull(tableName, "tableName is null"); - if (!metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).isPresent()) { + if (!metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).isPresent()) { return null; } return new HiveTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName()); @@ -249,12 +249,12 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect { requireNonNull(tableHandle, "tableHandle is null"); SchemaTableName tableName = schemaTableName(tableHandle); - return getTableMetadata(tableName); + return getTableMetadata(session, tableName); } - private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) + private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) { - Optional table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); } @@ -294,7 +294,7 @@ public List listTables(ConnectorSession session, String schemaN { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllTables(schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllTables(session.getUser(), schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -313,7 +313,7 @@ private List listSchemas(ConnectorSession session, String schemaNameOrNu public ColumnHandle getSampleWeightColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -335,7 +335,7 @@ public boolean canCreateSampledTables(ConnectorSession session) public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -356,7 +356,7 @@ public Map> listTableColumns(ConnectorSess ImmutableMap.Builder> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { try { - columns.put(tableName, getTableMetadata(tableName).getColumns()); + columns.put(tableName, getTableMetadata(session, tableName).getColumns()); } catch (HiveViewNotSupportedException e) { // view is not supported @@ -397,17 +397,18 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe List partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); - createTable(schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); + createTable(session, schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); } - public void createTable(String schemaName, + public void createTable(ConnectorSession session, + String schemaName, String tableName, String tableOwner, List columnHandles, HiveStorageFormat hiveStorageFormat, List partitionedBy) { - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -415,10 +416,11 @@ public void createTable(String schemaName, } createDirectory(hdfsEnvironment, targetPath); - createTable(schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); + createTable(session, schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); } - private Table createTable(String schemaName, + private Table createTable(ConnectorSession session, + String schemaName, String tableName, String tableOwner, List columnHandles, @@ -484,7 +486,7 @@ else if (!partitionColumnNames.contains(name)) { ImmutableMap.of(), ImmutableMap.of())); - metastore.createTable(table); + metastore.createTable(session.getUser(), table); return table; } @@ -496,7 +498,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); - Optional
tableMetadata = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
tableMetadata = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(handle.getSchemaTableName()); } @@ -509,7 +511,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -521,7 +523,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); - Optional
tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); + Optional
tableMetadata = metastore.getTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } @@ -538,7 +540,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan } sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); + metastore.alterTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); } @Override @@ -550,14 +552,14 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle tableHand HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
source = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); - metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -570,7 +572,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } - Optional
target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
target = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -579,7 +581,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } - metastore.dropTable(handle.getSchemaName(), handle.getTableName()); + metastore.dropTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); } @Override @@ -599,7 +601,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -633,7 +635,7 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand .map(partitionUpdateCodec::fromJson) .collect(toList()); - Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); + Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); Path writePath = new Path(handle.getWritePath().get()); // rename if using a temporary directory @@ -649,11 +651,12 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand renameDirectory(hdfsEnvironment, handle.getSchemaName(), handle.getTableName(), writePath, targetPath); } - PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); Table table = createTable( + session, handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), @@ -691,7 +694,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl verifyJvmTimeZone(); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -724,6 +727,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl private static class PartitionCommitter implements Closeable { + private final ConnectorSession session; private final String schemaName; private final String tableName; private final HiveMetastore metastore; @@ -731,8 +735,9 @@ private static class PartitionCommitter private final List batch; private final List createdPartitions = new ArrayList<>(); - public PartitionCommitter(String schemaName, String tableName, HiveMetastore metastore, int batchSize) + public PartitionCommitter(ConnectorSession session, String schemaName, String tableName, HiveMetastore metastore, int batchSize) { + this.session = session; this.schemaName = schemaName; this.tableName = tableName; this.metastore = metastore; @@ -766,7 +771,7 @@ public void abort() // drop created partitions for (Partition createdPartition : getCreatedPartitions()) { try { - metastore.dropPartition(schemaName, tableName, createdPartition.getValues()); + metastore.dropPartition(session.getUser(), schemaName, tableName, createdPartition.getValues()); } catch (Exception e) { log.error(e, "Error rolling back new partition '%s' in table '%s.%s", createdPartition.getValues(), schemaName, tableName); @@ -776,7 +781,7 @@ public void abort() private void addBatch() { - metastore.addPartitions(schemaName, tableName, batch); + metastore.addPartitions(session.getUser(), schemaName, tableName, batch); createdPartitions.addAll(batch); batch.clear(); } @@ -793,11 +798,11 @@ public void commitInsert(ConnectorSession session, ConnectorInsertTableHandle in .collect(toList()); HiveStorageFormat storageFormat = handle.getHiveStorageFormat(); - PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); - Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName())); } @@ -890,7 +895,7 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle } // Otherwise, insert was directly into the target table and partitions, and all must be checked for temp files - Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { log.error("Error rolling back insert into table %s.%s. Table was dropped during insert, and data directory may contain temporary data", handle.getSchemaName(), handle.getTableName()); return; @@ -904,10 +909,10 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle // check every existing partition that is outside for the base directory if (!table.get().getPartitionKeys().isEmpty()) { - List partitionNames = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) + List partitionNames = metastore.getPartitionNames(session.getUser(), handle.getSchemaName(), handle.getTableName()) .orElse(ImmutableList.of()); for (List partitionNameBatch : Iterables.partition(partitionNames, 10)) { - metastore.getPartitionsByNames(handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() + metastore.getPartitionsByNames(session.getUser(), handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(tableDirectory)) .forEach(locationsToClean::add); @@ -1135,7 +1140,7 @@ public void createView(ConnectorSession session, SchemaTableName viewName, Strin ImmutableMap.of())); try { - metastore.createTable(table); + metastore.createTable(session.getUser(), table); } catch (TableAlreadyExistsException e) { throw new ViewAlreadyExistsException(e.getTableName()); @@ -1151,7 +1156,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) } try { - metastore.dropTable(viewName.getSchemaName(), viewName.getTableName()); + metastore.dropTable(session.getUser(), viewName.getSchemaName(), viewName.getTableName()); } catch (TableNotFoundException e) { throw new ViewNotFoundException(e.getTableName()); @@ -1163,7 +1168,7 @@ public List listViews(ConnectorSession session, String schemaNa { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllViews(schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllViews(session.getUser(), schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -1183,7 +1188,7 @@ public Map getViews(ConnectorSession s } for (SchemaTableName schemaTableName : tableNames) { - Optional
table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), schemaTableName.getSchemaName(), schemaTableName.getTableName()); if (table.isPresent() && HiveUtil.isPrestoView(table.get())) { views.put(schemaTableName, new ConnectorViewDefinition( schemaTableName, @@ -1215,7 +1220,7 @@ public OptionalLong metadataDelete(ConnectorSession session, ConnectorTableHandl //for (HivePartition hivePartition : layoutHandle.getOrComputePartitions(this, session, tableHandle)) { for (HivePartition hivePartition : getOrComputePartitions(layoutHandle, session, tableHandle)) { - metastore.dropPartitionByName(handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); + metastore.dropPartitionByName(session.getUser(), handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); } // it is too expensive to determine the exact number of deleted rows return OptionalLong.empty(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java index 4e991d1547c4..6444e05878f1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.metastore.HiveMetastore; import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageIndexer; import com.facebook.presto.spi.PageIndexerFactory; @@ -88,6 +89,8 @@ public class HivePageSink implements ConnectorPageSink { + private final ConnectorSession session; + private final String schemaName; private final String tableName; @@ -122,6 +125,7 @@ public class HivePageSink private HiveRecordWriter[] writers = new HiveRecordWriter[0]; public HivePageSink( + ConnectorSession session, String schemaName, String tableName, boolean isCreateTable, @@ -138,6 +142,7 @@ public HivePageSink( boolean immutablePartitions, JsonCodec partitionUpdateCodec) { + this.session = requireNonNull(session, "session is null"); this.schemaName = requireNonNull(schemaName, "schemaName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); @@ -210,7 +215,7 @@ public HivePageSink( conf = new JobConf(hdfsEnvironment.getConfiguration(writePath.get())); } else { - Optional
table = metastore.getTable(schemaName, tableName); + Optional
table = metastore.getTable(session.getUser(), schemaName, tableName); if (!table.isPresent()) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table %s.%s was dropped during insert", schemaName, tableName)); } @@ -287,7 +292,7 @@ private HiveRecordWriter createWriter(List partitionRow) // attempt to get the existing partition (if this is an existing partitioned table) Optional partition = Optional.empty(); if (!partitionRow.isEmpty() && table != null) { - partition = metastore.getPartition(schemaName, tableName, partitionName); + partition = metastore.getPartition(session.getUser(), schemaName, tableName, partitionName); } if (!partition.isPresent()) { @@ -303,7 +308,7 @@ private HiveRecordWriter createWriter(List partitionRow) .map(HiveType::toHiveType) .map(HiveType::getHiveTypeName) .collect(Collectors.joining(":"))); - target = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName).toString(); + target = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName).toString(); if (!partitionRow.isEmpty()) { // verify the target directory for the partition does not already exist diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java index 7cb88075d5f6..313fe5f5592d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java @@ -64,19 +64,20 @@ public HivePageSinkProvider( public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorOutputTableHandle tableHandle) { HiveWritableTableHandle handle = checkType(tableHandle, HiveOutputTableHandle.class, "tableHandle"); - return createPageSink(handle, true); + return createPageSink(session, handle, true); } @Override public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorInsertTableHandle tableHandle) { HiveInsertTableHandle handle = checkType(tableHandle, HiveInsertTableHandle.class, "tableHandle"); - return createPageSink(handle, false); + return createPageSink(session, handle, false); } - private ConnectorPageSink createPageSink(HiveWritableTableHandle handle, boolean isCreateTable) + private ConnectorPageSink createPageSink(ConnectorSession session, HiveWritableTableHandle handle, boolean isCreateTable) { return new HivePageSink( + session, handle.getSchemaName(), handle.getTableName(), isCreateTable, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index 37eb15d30025..b41f6cb702fc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -100,7 +100,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(metastore, tableName); + Table table = getTable(session, metastore, tableName); Optional bucket = getHiveBucket(table, effectivePredicate.extractFixedValues()); TupleDomain compactEffectivePredicate = toCompactTupleDomain(effectivePredicate, domainCompactionThreshold); @@ -110,7 +110,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } List partitionColumns = getPartitionKeyColumnHandles(connectorId, table); - List partitionNames = getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate); + List partitionNames = getFilteredPartitionNames(session, metastore, tableName, partitionColumns, effectivePredicate); // do a final pass to filter based on fields that could not be used to filter the partitions ImmutableList.Builder partitions = ImmutableList.builder(); @@ -164,9 +164,9 @@ private Optional> parseValuesAndFilte return Optional.of(builder.build()); } - private Table getTable(HiveMetastore metastore, SchemaTableName tableName) + private Table getTable(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName) { - Optional
target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
target = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -185,7 +185,7 @@ private Table getTable(HiveMetastore metastore, SchemaTableName tableName) return table; } - private List getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) + private List getFilteredPartitionNames(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) { List filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { @@ -217,7 +217,7 @@ else if ((value instanceof Boolean) || (value instanceof Double) || (value insta } // fetch the partition names - return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) + return metastore.getPartitionNamesByParts(session.getUser(), tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 8f63692545bc..1506fc8f3c64 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -157,11 +157,11 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa // sort partitions partitions = Ordering.natural().onResultOf(HivePartition::getPartitionId).reverse().sortedCopy(partitions); - Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } - Iterable hivePartitions = getPartitionMetadata(table.get(), tableName, partitions); + Iterable hivePartitions = getPartitionMetadata(session, table.get(), tableName, partitions); HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader( connectorId, @@ -185,7 +185,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa return splitSource; } - private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) + private Iterable getPartitionMetadata(ConnectorSession session, Table table, SchemaTableName tableName, List hivePartitions) { if (hivePartitions.isEmpty()) { return ImmutableList.of(); @@ -201,6 +201,7 @@ private Iterable getPartitionMetadata(Table table, Schema Iterable> partitionNameBatches = partitionExponentially(hivePartitions, minPartitionBatchSize, maxPartitionBatchSize); Iterable> partitionBatches = transform(partitionNameBatches, partitionBatch -> { Optional> batch = metastore.getPartitionsByNames( + session.getUser(), tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId)); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 58e7aa397a98..0ebaa8158583 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.metastore.HiveMetastore; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; @@ -301,9 +302,9 @@ private static void checkWritable( } } - public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) + public static Path getTableDefaultLocation(ConnectorSession session, HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) { - String location = getDatabase(metastore, schemaName).getLocationUri(); + String location = getDatabase(session, metastore, schemaName).getLocationUri(); if (isNullOrEmpty(location)) { throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not set", schemaName)); } @@ -319,9 +320,9 @@ public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironm return new Path(databasePath, tableName); } - private static Database getDatabase(HiveMetastore metastore, String database) + private static Database getDatabase(ConnectorSession session, HiveMetastore metastore, String database) { - return metastore.getDatabase(database).orElseThrow(() -> new SchemaNotFoundException(database)); + return metastore.getDatabase(session.getUser(), database).orElseThrow(() -> new SchemaNotFoundException(database)); } public static boolean pathExists(HdfsEnvironment hdfsEnvironment, Path path) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index 75d7074a543a..ed668aabb338 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -305,7 +305,7 @@ private static Map getAll(LoadingCache cache, Iterable key } @Override - public List getAllDatabases() + public List getAllDatabases(String user) { return get(databaseNamesCache, ""); } @@ -328,7 +328,7 @@ private List loadAllDatabases() } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { return get(databaseCache, databaseName); } @@ -355,7 +355,7 @@ private Optional loadDatabase(String databaseName) } @Override - public Optional> getAllTables(String databaseName) + public Optional> getAllTables(String user, String databaseName) { return get(tableNamesCache, databaseName); } @@ -398,13 +398,13 @@ private Optional> loadAllTables(String databaseName) } @Override - public Optional
getTable(String databaseName, String tableName) + public Optional
getTable(String user, String databaseName, String tableName) { return get(tableCache, HiveTableName.table(databaseName, tableName)); } @Override - public Optional> getAllViews(String databaseName) + public Optional> getAllViews(String user, String databaseName) { return get(viewNamesCache, databaseName); } @@ -432,7 +432,7 @@ private Optional> loadAllViews(String databaseName) } @Override - public void createTable(Table table) + public void createTable(String user, Table table) { try { retry() @@ -467,7 +467,7 @@ public void createTable(Table table) } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { try { retry() @@ -506,7 +506,7 @@ protected void invalidateTable(String databaseName, String tableName) } @Override - public void alterTable(String databaseName, String tableName, Table table) + public void alterTable(String user, String databaseName, String tableName, Table table) { try { retry() @@ -570,7 +570,7 @@ private Optional
loadTable(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNames(String databaseName, String tableName) + public Optional> getPartitionNames(String user, String databaseName, String tableName) { return get(partitionNamesCache, HiveTableName.table(databaseName, tableName)); } @@ -602,7 +602,7 @@ private Optional> loadPartitionNames(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) { return get(partitionFilterCache, PartitionFilter.partitionFilter(databaseName, tableName, parts)); } @@ -632,7 +632,7 @@ private Optional> loadPartitionNamesByParts(PartitionFilter partiti } @Override - public void addPartitions(String databaseName, String tableName, List partitions) + public void addPartitions(String user, String databaseName, String tableName, List partitions) { if (partitions.isEmpty()) { return; @@ -676,7 +676,7 @@ public void addPartitions(String databaseName, String tableName, List } @Override - public void dropPartition(String databaseName, String tableName, List parts) + public void dropPartition(String user, String databaseName, String tableName, List parts) { try { retry() @@ -707,7 +707,7 @@ public void dropPartition(String databaseName, String tableName, List pa } @Override - public void dropPartitionByName(String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) { try { retry() @@ -753,7 +753,7 @@ private void invalidatePartitionCache(String databaseName, String tableName) } @Override - public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) { Iterable names = transform(partitionNames, name -> HivePartitionName.partition(databaseName, tableName, name)); @@ -769,7 +769,7 @@ public Optional> getPartitionsByNames(String databaseName } @Override - public Optional getPartition(String databaseName, String tableName, String partitionName) + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) { HivePartitionName name = HivePartitionName.partition(databaseName, tableName, partitionName); return get(partitionCache, name); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java index 49e9b71da794..066cb9576a23 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java @@ -30,43 +30,43 @@ public interface HiveMetastore { String DEFAULT_DATABASE_NAME = "default"; - void createTable(Table table); + void createTable(String user, Table table); - void dropTable(String databaseName, String tableName); + void dropTable(String user, String databaseName, String tableName); - void alterTable(String databaseName, String tableName, Table table); + void alterTable(String user, String databaseName, String tableName, Table table); @Managed void flushCache(); - List getAllDatabases(); + List getAllDatabases(String user); - Optional> getAllTables(String databaseName); + Optional> getAllTables(String user, String databaseName); - Optional> getAllViews(String databaseName); + Optional> getAllViews(String user, String databaseName); - Optional getDatabase(String databaseName); + Optional getDatabase(String user, String databaseName); /** * Adds partitions to the table in a single atomic task. The implementation * must either add all partitions and return normally, or add no partitions and * throw an exception. */ - void addPartitions(String databaseName, String tableName, List partitions); + void addPartitions(String user, String databaseName, String tableName, List partitions); - void dropPartition(String databaseName, String tableName, List parts); + void dropPartition(String user, String databaseName, String tableName, List parts); - void dropPartitionByName(String databaseName, String tableName, String partitionName); + void dropPartitionByName(String user, String databaseName, String tableName, String partitionName); - Optional> getPartitionNames(String databaseName, String tableName); + Optional> getPartitionNames(String user, String databaseName, String tableName); - Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); + Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts); - Optional getPartition(String databaseName, String tableName, String partitionName); + Optional getPartition(String user, String databaseName, String tableName, String partitionName); - Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames); + Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames); - Optional
getTable(String databaseName, String tableName); + Optional
getTable(String user, String databaseName, String tableName); Set getRoles(String user); @@ -81,7 +81,7 @@ default boolean isDatabaseOwner(String user, String databaseName) return true; } - Optional databaseMetadata = getDatabase(databaseName); + Optional databaseMetadata = getDatabase(user, databaseName); if (!databaseMetadata.isPresent()) { return false; } @@ -101,7 +101,7 @@ default boolean isDatabaseOwner(String user, String databaseName) default boolean isTableOwner(String user, String databaseName, String tableName) { // a table can only be owned by a user - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); return table.isPresent() && user.equals(table.get().getOwner()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 1cd4adf8046d..8a5692904b4c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -1564,7 +1564,7 @@ protected void doCreateEmptyTable(SchemaTableName tableName, HiveStorageFormat s assertEquals(tableMetadata.getColumns(), createTableColumns); // verify table format - Table table = getMetastoreClient(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); + Table table = getMetastoreClient(tableName.getSchemaName()).getTable(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()).get(); if (!table.getSd().getInputFormat().equals(storageFormat.getInputFormat())) { assertEquals(table.getSd().getInputFormat(), storageFormat.getInputFormat()); } @@ -1678,7 +1678,7 @@ protected Set listAllDataFiles(ConnectorInsertTableHandle tableHandle) protected Set listAllDataFiles(String schemaName, String tableName) throws IOException { - Table table = metastoreClient.getTable(schemaName, tableName).get(); + Table table = metastoreClient.getTable(SESSION.getUser(), schemaName, tableName).get(); Path path = new Path(table.getSd().getLocation()); Set existingFiles = new HashSet<>(); return listAllDataFiles(path, existingFiles); @@ -1715,7 +1715,7 @@ private void doInsertPartitioned(HiveStorageFormat storageFormat, SchemaTableNam insertData(tableHandle, CREATE_TABLE_PARTITIONED_DATA, SESSION); // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) @@ -1800,7 +1800,7 @@ private void doMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName t } // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java index 1bb81269e532..7869d77b9035 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java @@ -325,7 +325,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF // table, which fails without explicit configuration for S3. // We work around that by using a dummy location when creating the // table and update it here to the correct S3 location. - metastoreClient.updateTableLocation(database, tableName.getTableName(), outputHandle.getWritePath().get()); + metastoreClient.updateTableLocation(SESSION.getUser(), database, tableName.getTableName(), outputHandle.getWritePath().get()); // load the new table ConnectorTableHandle tableHandle = getTableHandle(tableName); @@ -352,7 +352,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF private void dropTable(SchemaTableName table) { try { - metastoreClient.dropTable(table.getSchemaName(), table.getTableName()); + metastoreClient.dropTable(SESSION.getUser(), table.getSchemaName(), table.getTableName()); } catch (RuntimeException e) { // this usually occurs because the table was not created @@ -402,9 +402,9 @@ public TestingHiveMetastore(HiveCluster hiveCluster, ExecutorService executor, H } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { - Optional database = super.getDatabase(databaseName); + Optional database = super.getDatabase(user, databaseName); if (database.isPresent()) { database.get().setLocationUri("s3://" + writableBucket + "/"); } @@ -412,18 +412,18 @@ public Optional getDatabase(String databaseName) } @Override - public void createTable(Table table) + public void createTable(String user, Table table) { // hack to work around the metastore not being configured for S3 table.getSd().setLocation("/"); - super.createTable(table); + super.createTable(user, table); } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { try { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -449,10 +449,10 @@ public void dropTable(String databaseName, String tableName) } } - public void updateTableLocation(String databaseName, String tableName, String location) + public void updateTableLocation(String user, String databaseName, String tableName, String location) { try { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java index fa8a81344c20..56bb639c7f33 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java @@ -88,13 +88,13 @@ public void createDatabase(Database database) } @Override - public List getAllDatabases() + public List getAllDatabases(String user) { return ImmutableList.copyOf(databases.keySet()); } @Override - public void createTable(Table table) + public void createTable(String user1, Table table) { SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName()); Table tableCopy = table.deepCopy(); @@ -137,7 +137,7 @@ else if (tableCopy.getSd().getLocation() != null) { } @Override - public void dropTable(String databaseName, String tableName) + public void dropTable(String user, String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); @@ -159,7 +159,7 @@ public void dropTable(String databaseName, String tableName) } @Override - public void alterTable(String databaseName, String tableName, Table newTable) + public void alterTable(String user, String databaseName, String tableName, Table newTable) { SchemaTableName oldName = new SchemaTableName(databaseName, tableName); SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName()); @@ -186,7 +186,7 @@ public void alterTable(String databaseName, String tableName, Table newTable) } @Override - public Optional> getAllTables(String databaseName) + public Optional> getAllTables(String user, String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.relations.keySet()) { @@ -198,7 +198,7 @@ public Optional> getAllTables(String databaseName) } @Override - public Optional> getAllViews(String databaseName) + public Optional> getAllViews(String user, String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.views.keySet()) { @@ -210,15 +210,15 @@ public Optional> getAllViews(String databaseName) } @Override - public Optional getDatabase(String databaseName) + public Optional getDatabase(String user, String databaseName) { return Optional.ofNullable(databases.get(databaseName)); } @Override - public void addPartitions(String databaseName, String tableName, List partitions) + public void addPartitions(String user, String databaseName, String tableName, List partitions) { - Optional
table = getTable(databaseName, tableName); + Optional
table = getTable(user, databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -233,7 +233,7 @@ public void addPartitions(String databaseName, String tableName, List } @Override - public void dropPartition(String databaseName, String tableName, List parts) + public void dropPartition(String user, String databaseName, String tableName, List parts) { for (Entry entry : partitions.entrySet()) { PartitionName partitionName = entry.getKey(); @@ -245,7 +245,7 @@ public void dropPartition(String databaseName, String tableName, List pa } @Override - public void dropPartitionByName(String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) { for (PartitionName partition : partitions.keySet()) { if (partition.matches(databaseName, tableName, partitionName)) { @@ -255,7 +255,7 @@ public void dropPartitionByName(String databaseName, String tableName, String pa } @Override - public Optional> getPartitionNames(String databaseName, String tableName) + public Optional> getPartitionNames(String user, String databaseName, String tableName) { return Optional.of(ImmutableList.copyOf(partitions.entrySet().stream() .filter(entry -> entry.getKey().matches(databaseName, tableName)) @@ -264,7 +264,7 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional getPartition(String databaseName, String tableName, String partitionName) + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) { PartitionName name = new PartitionName(databaseName, tableName, partitionName); Partition partition = partitions.get(name); @@ -275,7 +275,7 @@ public Optional getPartition(String databaseName, String tableName, S } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) { return Optional.of(partitions.entrySet().stream() .filter(entry -> partitionMatches(entry.getValue(), databaseName, tableName, parts)) @@ -303,7 +303,7 @@ private static boolean partitionMatches(Partition partition, String databaseName } @Override - public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (String name : partitionNames) { @@ -318,7 +318,7 @@ public Optional> getPartitionsByNames(String databaseName } @Override - public Optional
getTable(String databaseName, String tableName) + public Optional
getTable(String user, String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); return Optional.ofNullable(relations.get(schemaTableName)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index b0643d13a7e8..ac55e00933fa 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; +import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.BAD_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_PARTITION1; @@ -55,14 +56,14 @@ public void testGetAllDatabases() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); } @@ -71,21 +72,21 @@ public void testGetAllTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetAllTAbles() throws Exception { - assertFalse(metastore.getAllTables(BAD_DATABASE).isPresent()); + assertFalse(metastore.getAllTables(SESSION.getUser(), BAD_DATABASE).isPresent()); } @Test @@ -93,21 +94,21 @@ public void testGetTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetTable() throws Exception { - assertFalse(metastore.getTable(BAD_DATABASE, TEST_TABLE).isPresent()); + assertFalse(metastore.getTable(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).isPresent()); } @Test @@ -116,14 +117,14 @@ public void testGetPartitionNames() { ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -131,7 +132,7 @@ public void testGetPartitionNames() public void testInvalidGetPartitionNames() throws Exception { - assertEquals(metastore.getPartitionNames(BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); + assertEquals(metastore.getPartitionNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); } @Test @@ -142,14 +143,14 @@ public void testGetPartitionNamesByParts() ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -157,7 +158,7 @@ public void testInvalidGetPartitionNamesByParts() throws Exception { ImmutableList parts = ImmutableList.of(); - assertFalse(metastore.getPartitionNamesByParts(BAD_DATABASE, TEST_TABLE, parts).isPresent()); + assertFalse(metastore.getPartitionNamesByParts(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, parts).isPresent()); } @Test @@ -165,35 +166,35 @@ public void testGetPartitionsByNames() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - metastore.getTable(TEST_DATABASE, TEST_TABLE); + metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 3); metastore.flushCache(); // Fetching both should only result in one batched access - assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 4); } public void testInvalidGetPartitionsByNames() throws Exception { - assertFalse(metastore.getPartitionsByNames(BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); + assertFalse(metastore.getPartitionsByNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); } @Test @@ -203,7 +204,7 @@ public void testNoCacheExceptions() // Throw exceptions on usage mockClient.setThrowException(true); try { - metastore.getAllDatabases(); + metastore.getAllDatabases(SESSION.getUser()); } catch (RuntimeException ignored) { } @@ -211,7 +212,7 @@ public void testNoCacheExceptions() // Second try should hit the client again try { - metastore.getAllDatabases(); + metastore.getAllDatabases(SESSION.getUser()); } catch (RuntimeException ignored) { } From bae9de697343a0fbdd15e01b7fb43a0f2583a7a6 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:23:31 -0700 Subject: [PATCH 014/151] Revert 972d5d166bc8174c9cef50b1066f6efc9a07a02b --- .../parquet/ParquetRecordCursorProvider.java | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java index 3854449498da..eb6bb880bed3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetRecordCursorProvider.java @@ -27,12 +27,10 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; -import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -90,26 +88,20 @@ public Optional createHiveRecordCursor( throw new IllegalArgumentException("Can not read Parquet column: " + unsupportedColumns); } - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - try { - return ugi.doAs((PrivilegedExceptionAction>) () -> Optional.of(new ParquetHiveRecordCursor( - configuration, - path, - start, - length, - schema, - partitionKeys, - columns, - useParquetColumnNames, - hiveStorageTimeZone, - typeManager, - isParquetPredicatePushdownEnabled(session), - effectivePredicate - ))); - } - catch (Exception e) { - throw new RuntimeException(e); - } + return Optional.of(new ParquetHiveRecordCursor( + configuration, + path, + start, + length, + schema, + partitionKeys, + columns, + useParquetColumnNames, + hiveStorageTimeZone, + typeManager, + isParquetPredicatePushdownEnabled(session), + effectivePredicate + )); } private static Predicate isParquetSupportedType() From a39315881019902b237d11540219a1c94b7ecc0a Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:24:09 -0700 Subject: [PATCH 015/151] Load splits as user so that hdfs reads happen as that user. --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 82defdecc369..557544a96b6b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,8 +38,10 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -162,8 +164,9 @@ public TaskStatus process() try { CompletableFuture future; taskExecutionLock.readLock().lock(); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - future = loadSplits(); + future = ugi.doAs((PrivilegedExceptionAction>) BackgroundHiveSplitLoader.this::loadSplits); } finally { taskExecutionLock.readLock().unlock(); From 1b65e2fad9ac91da587e0193b114933461b7fcb6 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 13:26:36 -0700 Subject: [PATCH 016/151] UserBasedHiveMetastore to access metastore as user issuing the query. --- .../presto/hive/HiveClientModule.java | 6 +- .../hive/ThriftHiveMetastoreClient.java | 5 + .../metastore/UserBasedHiveMetastore.java | 608 ++++++++++++++++++ .../UserBasedHiveMetastoreStats.java | 141 ++++ 4 files changed, 757 insertions(+), 3 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index bc48cbfc7f0f..de95fec6e6fb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastore; +import com.facebook.presto.hive.metastore.UserBasedHiveMetastore; import com.facebook.presto.hive.orc.DwrfPageSourceFactory; import com.facebook.presto.hive.orc.DwrfRecordCursorProvider; import com.facebook.presto.hive.orc.OrcPageSourceFactory; @@ -81,9 +81,9 @@ public void configure(Binder binder) binder.bind(HiveMetastore.class).toInstance(metastore); } else { - binder.bind(HiveMetastore.class).to(CachingHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(HiveMetastore.class).to(UserBasedHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(HiveMetastore.class) - .as(generatedNameOf(CachingHiveMetastore.class, connectorId)); + .as(generatedNameOf(UserBasedHiveMetastore.class, connectorId)); } binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java index d97aaf3d464f..9ab9986b7bab 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java @@ -49,6 +49,11 @@ public ThriftHiveMetastoreClient(TProtocol protocol) this.client = new ThriftHiveMetastore.Client(protocol); } + public ThriftHiveMetastore.Client getClient() + { + return client; + } + @Override public void close() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java new file mode 100644 index 000000000000..251962a01cc9 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java @@ -0,0 +1,608 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.HiveViewNotSupportedException; +import com.facebook.presto.hive.TableAlreadyExistsException; +import com.facebook.presto.hive.ThriftHiveMetastoreClient; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaNotFoundException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableNotFoundException; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.HiveObjectType; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.thrift.TException; +import org.weakref.jmx.Flatten; +import org.weakref.jmx.Managed; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.Function; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; +import static com.facebook.presto.hive.HiveUtil.isPrestoView; +import static com.facebook.presto.hive.RetryDriver.retry; +import static com.facebook.presto.hive.metastore.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.HivePrivilege.parsePrivilege; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toSet; +import static org.apache.hadoop.hive.metastore.api.PrincipalType.USER; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS; + +public class UserBasedHiveMetastore + implements HiveMetastore +{ + private final UserBasedHiveMetastoreStats stats = new UserBasedHiveMetastoreStats(); + protected final HiveCluster clientProvider; + + @Inject + public UserBasedHiveMetastore(HiveCluster hiveCluster) + { + this.clientProvider = requireNonNull(hiveCluster, "hiveCluster is null"); + } + + @Managed + @Flatten + public UserBasedHiveMetastoreStats getStats() + { + return stats; + } + + protected Function getExceptionMapper() + { + return Function.identity(); + } + + @Override + public void createTable(String user, Table table) + { + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(AlreadyExistsException.class, InvalidObjectException.class, MetaException.class, NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("createTable", stats.getCreateTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.createTable(table); + } + return null; + })); + } + catch (AlreadyExistsException e) { + throw new TableAlreadyExistsException(new SchemaTableName(table.getDbName(), table.getTableName())); + } + catch (NoSuchObjectException e) { + throw new SchemaNotFoundException(table.getDbName()); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw Throwables.propagate(e); + } + } + + @Override + public void dropTable(String user, String databaseName, String tableName) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("dropTable", stats.getDropTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropTable(databaseName, tableName, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void alterTable(String user, String databaseName, String tableName, Table table) + { + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(InvalidOperationException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("alterTable", stats.getAlterTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + Optional
source = getTable(user, databaseName, tableName); + if (!source.isPresent()) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + client.getClient().set_ugi(user, ImmutableList.of()); + client.alterTable(databaseName, tableName, table); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (InvalidOperationException | MetaException e) { + throw com.google.common.base.Throwables.propagate(e); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void flushCache() + { + } + + @Override + public List getAllDatabases(String user) + { + try { + return retry() + .stopOnIllegalExceptions() + .run("getAllDatabases", stats.getGetAllDatabases().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return client.getAllDatabases(); + } + })); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getAllTables(String user, String databaseName) + { + Callable> getAllTables = stats.getGetAllTables().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return client.getAllTables(databaseName); + } + }); + + Callable getDatabase = stats.getGetDatabase().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.getDatabase(databaseName); + return null; + } + }); + + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getAllTables", () -> { + List tables = getAllTables.call(); + if (tables.isEmpty()) { + // Check to see if the database exists + getDatabase.call(); + } + return Optional.of(tables); + }); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getAllViews(String user, String databaseName) + { + try { + return retry() + .stopOn(UnknownDBException.class) + .stopOnIllegalExceptions() + .run("getAllViews", stats.getAllViews().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + String filter = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\""; + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getTableNamesByFilter(databaseName, filter)); + } + })); + } + catch (UnknownDBException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional getDatabase(String user, String databaseName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getDatabase", stats.getGetDatabase().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getDatabase(databaseName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public void addPartitions(String user, String databaseName, String tableName, List partitions) + { + if (partitions.isEmpty()) { + return; + } + try { + retry() + .exceptionMapper(getExceptionMapper()) + .stopOn(org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class, org.apache.hadoop.hive.metastore.api.InvalidObjectException.class, MetaException.class, org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, PrestoException.class) + .stopOnIllegalExceptions() + .run("addPartitions", stats.getAddPartitions().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + int partitionsAdded = client.addPartitions(partitions); + if (partitionsAdded != partitions.size()) { + throw new PrestoException(HIVE_METASTORE_ERROR, + format("Hive metastore only added %s of %s partitions", partitionsAdded, partitions.size())); + } + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException | org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + // todo partition already exists exception + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void dropPartition(String user, String databaseName, String tableName, List parts) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("dropPartition", stats.getDropPartition().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropPartition(databaseName, tableName, parts, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + { + try { + retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) + .stopOnIllegalExceptions() + .run("dropPartitionByName", stats.getDropPartitionByName().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + // It is observed that: (examples below assumes a table with one partition column `ds`) + // * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift call is a no-op. It doesn't throw any exception. + // * When a typo exists in partition column name (e.g. dxs=2015-09-01), this thrift call will delete ds=2015-09-01. + client.getClient().set_ugi(user, ImmutableList.of()); + client.dropPartitionByName(databaseName, tableName, partitionName, true); + } + return null; + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw com.google.common.base.Throwables.propagate(e); + } + } + + @Override + public Optional> getPartitionNames(String user, String databaseName, String tableName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionNames", stats.getGetPartitionNames().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionNames(databaseName, tableName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionNamesByParts", stats.getGetPartitionNamesPs().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionNamesFiltered(databaseName, tableName, parts)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + { + requireNonNull(partitionName, "partitionName is null"); + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionsByNames", stats.getGetPartitionByName().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + return Optional.of(client.getPartitionByName(databaseName, tableName, partitionName)); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + { + requireNonNull(partitionNames, "partitionNames is null"); + checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); + + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) + .stopOnIllegalExceptions() + .run("getPartitionsByNames", stats.getGetPartitionsByNames().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + List partitionColumnNames = ImmutableList.copyOf(Warehouse.makeSpecFromName(partitionNames.get(0)).keySet()); + ImmutableMap.Builder partitions = ImmutableMap.builder(); + client.getClient().set_ugi(user, ImmutableList.of()); + for (Partition partition : client.getPartitionsByNames(databaseName, tableName, partitionNames)) { + String partitionId = FileUtils.makePartName(partitionColumnNames, partition.getValues(), null); + partitions.put(partitionId, partition); + } + return Optional.of(partitions.build()); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + // assume none of the partitions in the batch are available + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional
getTable(String user, String databaseName, String tableName) + { + try { + return retry() + .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, HiveViewNotSupportedException.class) + .stopOnIllegalExceptions() + .run("getTable", stats.getGetTable().wrap(() -> { + try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { + client.getClient().set_ugi(user, ImmutableList.of()); + Table table = client.getTable(databaseName, tableName); + if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name()) && (!isPrestoView(table))) { + throw new HiveViewNotSupportedException(new SchemaTableName(databaseName, tableName)); + } + return Optional.of(table); + } + })); + } + catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Set getRoles(String user) + { + try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { + List roles = client.listRoles(user, USER); + if (roles == null) { + return ImmutableSet.of(); + } + return ImmutableSet.copyOf(roles.stream() + .map(Role::getRoleName) + .collect(toSet())); + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Set getDatabasePrivileges(String user, String databaseName) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + + if (isDatabaseOwner(user, databaseName)) { + privileges.add(OWNERSHIP); + } + privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.DATABASE, databaseName, null, null, null))); + + return privileges.build(); + } + + @Override + public Set getTablePrivileges(String user, String databaseName, String tableName) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + + if (isTableOwner(user, databaseName, tableName)) { + privileges.add(OWNERSHIP); + } + privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null))); + + return privileges.build(); + } + + private Set getPrivileges(String user, HiveObjectRef objectReference) + { + ImmutableSet.Builder privileges = ImmutableSet.builder(); + try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { + PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null); + + if (privilegeSet != null) { + Map> userPrivileges = privilegeSet.getUserPrivileges(); + if (userPrivileges != null) { + privileges.addAll(toGrants(userPrivileges.get(user))); + } + for (List rolePrivileges : privilegeSet.getRolePrivileges().values()) { + privileges.addAll(toGrants(rolePrivileges)); + } + // We do not add the group permissions as Hive does not seem to process these + } + } + catch (TException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + + return privileges.build(); + } + + private static Set toGrants(List userGrants) + { + if (userGrants == null) { + return ImmutableSet.of(); + } + + ImmutableSet.Builder privileges = ImmutableSet.builder(); + for (PrivilegeGrantInfo userGrant : userGrants) { + privileges.addAll(parsePrivilege(userGrant)); + if (userGrant.isGrantOption()) { + privileges.add(HivePrivilege.GRANT); + } + } + return privileges.build(); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java new file mode 100644 index 000000000000..7778e2ba9010 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +public class UserBasedHiveMetastoreStats +{ + private final HiveMetastoreApiStats getAllDatabases = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getDatabase = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getAllTables = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getAllViews = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionNames = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionNamesPs = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionByName = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats getPartitionsByNames = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats createTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats alterTable = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats addPartitions = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropPartition = new HiveMetastoreApiStats(); + private final HiveMetastoreApiStats dropPartitionByName = new HiveMetastoreApiStats(); + + @Managed + @Nested + public HiveMetastoreApiStats getGetAllDatabases() + { + return getAllDatabases; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetDatabase() + { + return getDatabase; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetAllTables() + { + return getAllTables; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAllViews() + { + return getAllViews; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetTable() + { + return getTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionNames() + { + return getPartitionNames; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionNamesPs() + { + return getPartitionNamesPs; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionByName() + { + return getPartitionByName; + } + + @Managed + @Nested + public HiveMetastoreApiStats getGetPartitionsByNames() + { + return getPartitionsByNames; + } + + @Managed + @Nested + public HiveMetastoreApiStats getCreateTable() + { + return createTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropTable() + { + return dropTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAlterTable() + { + return alterTable; + } + + @Managed + @Nested + public HiveMetastoreApiStats getAddPartitions() + { + return addPartitions; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropPartition() + { + return dropPartition; + } + + @Managed + @Nested + public HiveMetastoreApiStats getDropPartitionByName() + { + return dropPartitionByName; + } +} From 7cbc53a635331388b4fcfd6601b4cffc88aa8cbb Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 15:03:42 -0700 Subject: [PATCH 017/151] Import missed import. From acfd0c75cc96fe1f4e289a69866b32bef538010c Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 26 Oct 2015 15:43:54 -0700 Subject: [PATCH 018/151] Use standard charset for UTF-8 instead of string. From a5a4f8cdaec61eb5172c9b7680780c46e3aba484 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Tue, 3 Nov 2015 12:08:31 -0800 Subject: [PATCH 019/151] Pick username from unix system and disallow overriding it. --- .../main/java/com/facebook/presto/cli/ClientOptions.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java index 32bf33079643..364958cd6065 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java @@ -17,6 +17,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; +import com.sun.security.auth.module.UnixSystem; import io.airlift.airline.Option; import io.airlift.http.client.spnego.KerberosConfig; @@ -66,8 +67,9 @@ public class ClientOptions @Option(name = "--keystore-password", title = "keystore password", description = "Keystore password") public String keystorePassword; - @Option(name = "--user", title = "user", description = "Username") - public String user = System.getProperty("user.name"); + // Pick the user name for the logged in user. + // Do not let it be overridden by users. + public String user = new UnixSystem().getUsername(); @Option(name = "--source", title = "source", description = "Name of source making query") public String source = "presto-cli"; From e26fd783db6a07310faddf73470f83c3e1d6e845 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:53:27 -0800 Subject: [PATCH 020/151] Revert "Propagate user session information throughout hive metastore." This reverts commit d01400182645e82f733760a2d938273c49dd5a88. --- .../facebook/presto/hive/HiveMetadata.java | 85 +++++++++---------- .../facebook/presto/hive/HivePageSink.java | 11 +-- .../presto/hive/HivePageSinkProvider.java | 7 +- .../presto/hive/HivePartitionManager.java | 12 +-- .../presto/hive/HiveSplitManager.java | 7 +- .../facebook/presto/hive/HiveWriteUtils.java | 9 +- .../hive/metastore/CachingHiveMetastore.java | 30 +++---- .../presto/hive/metastore/HiveMetastore.java | 34 ++++---- .../presto/hive/AbstractTestHiveClient.java | 8 +- .../presto/hive/AbstractTestHiveClientS3.java | 20 ++--- .../hive/metastore/InMemoryHiveMetastore.java | 32 +++---- .../metastore/TestCachingHiveMetastore.java | 59 +++++++------ 12 files changed, 150 insertions(+), 164 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index a3ad7180b9de..c8e48aa7e21b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -231,14 +231,14 @@ public HivePartitionManager getPartitionManager() @Override public List listSchemaNames(ConnectorSession session) { - return metastore.getAllDatabases(session.getUser()); + return metastore.getAllDatabases(); } @Override public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { requireNonNull(tableName, "tableName is null"); - if (!metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).isPresent()) { + if (!metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).isPresent()) { return null; } return new HiveTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName()); @@ -249,12 +249,12 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect { requireNonNull(tableHandle, "tableHandle is null"); SchemaTableName tableName = schemaTableName(tableHandle); - return getTableMetadata(session, tableName); + return getTableMetadata(tableName); } - private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) + private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) { - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) { throw new TableNotFoundException(tableName); } @@ -294,7 +294,7 @@ public List listTables(ConnectorSession session, String schemaN { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllTables(session.getUser(), schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllTables(schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -313,7 +313,7 @@ private List listSchemas(ConnectorSession session, String schemaNameOrNu public ColumnHandle getSampleWeightColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -335,7 +335,7 @@ public boolean canCreateSampledTables(ConnectorSession session) public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -356,7 +356,7 @@ public Map> listTableColumns(ConnectorSess ImmutableMap.Builder> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { try { - columns.put(tableName, getTableMetadata(session, tableName).getColumns()); + columns.put(tableName, getTableMetadata(tableName).getColumns()); } catch (HiveViewNotSupportedException e) { // view is not supported @@ -397,18 +397,17 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe List partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); - createTable(session, schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); + createTable(schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy); } - public void createTable(ConnectorSession session, - String schemaName, + public void createTable(String schemaName, String tableName, String tableOwner, List columnHandles, HiveStorageFormat hiveStorageFormat, List partitionedBy) { - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -416,11 +415,10 @@ public void createTable(ConnectorSession session, } createDirectory(hdfsEnvironment, targetPath); - createTable(session, schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); + createTable(schemaName, tableName, tableOwner, columnHandles, hiveStorageFormat, partitionedBy, targetPath); } - private Table createTable(ConnectorSession session, - String schemaName, + private Table createTable(String schemaName, String tableName, String tableOwner, List columnHandles, @@ -486,7 +484,7 @@ else if (!partitionColumnNames.contains(name)) { ImmutableMap.of(), ImmutableMap.of())); - metastore.createTable(session.getUser(), table); + metastore.createTable(table); return table; } @@ -498,7 +496,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); - Optional
tableMetadata = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
tableMetadata = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(handle.getSchemaTableName()); } @@ -511,7 +509,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -523,7 +521,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); - Optional
tableMetadata = metastore.getTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); + Optional
tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } @@ -540,7 +538,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan } sd.setCols(columns.build()); table.setSd(sd); - metastore.alterTable(session.getUser(), hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); + metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); } @Override @@ -552,14 +550,14 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle tableHand HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
source = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); - metastore.alterTable(session.getUser(), handle.getSchemaName(), handle.getTableName(), table); + metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); } @Override @@ -572,7 +570,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } - Optional
target = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -581,7 +579,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } - metastore.dropTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + metastore.dropTable(handle.getSchemaName(), handle.getTableName()); } @Override @@ -601,7 +599,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto List columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName); // verify the target directory for the table if (pathExists(hdfsEnvironment, targetPath)) { @@ -635,7 +633,7 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand .map(partitionUpdateCodec::fromJson) .collect(toList()); - Path targetPath = getTableDefaultLocation(session, metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); + Path targetPath = getTableDefaultLocation(metastore, hdfsEnvironment, handle.getSchemaName(), handle.getTableName()); Path writePath = new Path(handle.getWritePath().get()); // rename if using a temporary directory @@ -651,12 +649,11 @@ public void commitCreateTable(ConnectorSession session, ConnectorOutputTableHand renameDirectory(hdfsEnvironment, handle.getSchemaName(), handle.getTableName(), writePath, targetPath); } - PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); Table table = createTable( - session, handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), @@ -694,7 +691,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl verifyJvmTimeZone(); SchemaTableName tableName = schemaTableName(tableHandle); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } @@ -727,7 +724,6 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl private static class PartitionCommitter implements Closeable { - private final ConnectorSession session; private final String schemaName; private final String tableName; private final HiveMetastore metastore; @@ -735,9 +731,8 @@ private static class PartitionCommitter private final List batch; private final List createdPartitions = new ArrayList<>(); - public PartitionCommitter(ConnectorSession session, String schemaName, String tableName, HiveMetastore metastore, int batchSize) + public PartitionCommitter(String schemaName, String tableName, HiveMetastore metastore, int batchSize) { - this.session = session; this.schemaName = schemaName; this.tableName = tableName; this.metastore = metastore; @@ -771,7 +766,7 @@ public void abort() // drop created partitions for (Partition createdPartition : getCreatedPartitions()) { try { - metastore.dropPartition(session.getUser(), schemaName, tableName, createdPartition.getValues()); + metastore.dropPartition(schemaName, tableName, createdPartition.getValues()); } catch (Exception e) { log.error(e, "Error rolling back new partition '%s' in table '%s.%s", createdPartition.getValues(), schemaName, tableName); @@ -781,7 +776,7 @@ public void abort() private void addBatch() { - metastore.addPartitions(session.getUser(), schemaName, tableName, batch); + metastore.addPartitions(schemaName, tableName, batch); createdPartitions.addAll(batch); batch.clear(); } @@ -798,11 +793,11 @@ public void commitInsert(ConnectorSession session, ConnectorInsertTableHandle in .collect(toList()); HiveStorageFormat storageFormat = handle.getHiveStorageFormat(); - PartitionCommitter partitionCommitter = new PartitionCommitter(session, handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); + PartitionCommitter partitionCommitter = new PartitionCommitter(handle.getSchemaName(), handle.getTableName(), metastore, PARTITION_COMMIT_BATCH_SIZE); try { partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates); - Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName())); } @@ -895,7 +890,7 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle } // Otherwise, insert was directly into the target table and partitions, and all must be checked for temp files - Optional
table = metastore.getTable(session.getUser(), handle.getSchemaName(), handle.getTableName()); + Optional
table = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!table.isPresent()) { log.error("Error rolling back insert into table %s.%s. Table was dropped during insert, and data directory may contain temporary data", handle.getSchemaName(), handle.getTableName()); return; @@ -909,10 +904,10 @@ public void rollbackInsert(ConnectorSession session, ConnectorInsertTableHandle // check every existing partition that is outside for the base directory if (!table.get().getPartitionKeys().isEmpty()) { - List partitionNames = metastore.getPartitionNames(session.getUser(), handle.getSchemaName(), handle.getTableName()) + List partitionNames = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) .orElse(ImmutableList.of()); for (List partitionNameBatch : Iterables.partition(partitionNames, 10)) { - metastore.getPartitionsByNames(session.getUser(), handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() + metastore.getPartitionsByNames(handle.getSchemaName(), handle.getTableName(), partitionNameBatch).orElse(ImmutableMap.of()).values().stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(tableDirectory)) .forEach(locationsToClean::add); @@ -1140,7 +1135,7 @@ public void createView(ConnectorSession session, SchemaTableName viewName, Strin ImmutableMap.of())); try { - metastore.createTable(session.getUser(), table); + metastore.createTable(table); } catch (TableAlreadyExistsException e) { throw new ViewAlreadyExistsException(e.getTableName()); @@ -1156,7 +1151,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) } try { - metastore.dropTable(session.getUser(), viewName.getSchemaName(), viewName.getTableName()); + metastore.dropTable(viewName.getSchemaName(), viewName.getTableName()); } catch (TableNotFoundException e) { throw new ViewNotFoundException(e.getTableName()); @@ -1168,7 +1163,7 @@ public List listViews(ConnectorSession session, String schemaNa { ImmutableList.Builder tableNames = ImmutableList.builder(); for (String schemaName : listSchemas(session, schemaNameOrNull)) { - for (String tableName : metastore.getAllViews(session.getUser(), schemaName).orElse(emptyList())) { + for (String tableName : metastore.getAllViews(schemaName).orElse(emptyList())) { tableNames.add(new SchemaTableName(schemaName, tableName)); } } @@ -1188,7 +1183,7 @@ public Map getViews(ConnectorSession s } for (SchemaTableName schemaTableName : tableNames) { - Optional
table = metastore.getTable(session.getUser(), schemaTableName.getSchemaName(), schemaTableName.getTableName()); + Optional
table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()); if (table.isPresent() && HiveUtil.isPrestoView(table.get())) { views.put(schemaTableName, new ConnectorViewDefinition( schemaTableName, @@ -1220,7 +1215,7 @@ public OptionalLong metadataDelete(ConnectorSession session, ConnectorTableHandl //for (HivePartition hivePartition : layoutHandle.getOrComputePartitions(this, session, tableHandle)) { for (HivePartition hivePartition : getOrComputePartitions(layoutHandle, session, tableHandle)) { - metastore.dropPartitionByName(session.getUser(), handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); + metastore.dropPartitionByName(handle.getSchemaName(), handle.getTableName(), hivePartition.getPartitionId()); } // it is too expensive to determine the exact number of deleted rows return OptionalLong.empty(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java index 6444e05878f1..4e991d1547c4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java @@ -15,7 +15,6 @@ import com.facebook.presto.hive.metastore.HiveMetastore; import com.facebook.presto.spi.ConnectorPageSink; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageIndexer; import com.facebook.presto.spi.PageIndexerFactory; @@ -89,8 +88,6 @@ public class HivePageSink implements ConnectorPageSink { - private final ConnectorSession session; - private final String schemaName; private final String tableName; @@ -125,7 +122,6 @@ public class HivePageSink private HiveRecordWriter[] writers = new HiveRecordWriter[0]; public HivePageSink( - ConnectorSession session, String schemaName, String tableName, boolean isCreateTable, @@ -142,7 +138,6 @@ public HivePageSink( boolean immutablePartitions, JsonCodec partitionUpdateCodec) { - this.session = requireNonNull(session, "session is null"); this.schemaName = requireNonNull(schemaName, "schemaName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); @@ -215,7 +210,7 @@ public HivePageSink( conf = new JobConf(hdfsEnvironment.getConfiguration(writePath.get())); } else { - Optional
table = metastore.getTable(session.getUser(), schemaName, tableName); + Optional
table = metastore.getTable(schemaName, tableName); if (!table.isPresent()) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table %s.%s was dropped during insert", schemaName, tableName)); } @@ -292,7 +287,7 @@ private HiveRecordWriter createWriter(List partitionRow) // attempt to get the existing partition (if this is an existing partitioned table) Optional partition = Optional.empty(); if (!partitionRow.isEmpty() && table != null) { - partition = metastore.getPartition(session.getUser(), schemaName, tableName, partitionName); + partition = metastore.getPartition(schemaName, tableName, partitionName); } if (!partition.isPresent()) { @@ -308,7 +303,7 @@ private HiveRecordWriter createWriter(List partitionRow) .map(HiveType::toHiveType) .map(HiveType::getHiveTypeName) .collect(Collectors.joining(":"))); - target = getTableDefaultLocation(session, metastore, hdfsEnvironment, schemaName, tableName).toString(); + target = getTableDefaultLocation(metastore, hdfsEnvironment, schemaName, tableName).toString(); if (!partitionRow.isEmpty()) { // verify the target directory for the partition does not already exist diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java index 313fe5f5592d..7cb88075d5f6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java @@ -64,20 +64,19 @@ public HivePageSinkProvider( public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorOutputTableHandle tableHandle) { HiveWritableTableHandle handle = checkType(tableHandle, HiveOutputTableHandle.class, "tableHandle"); - return createPageSink(session, handle, true); + return createPageSink(handle, true); } @Override public ConnectorPageSink createPageSink(ConnectorSession session, ConnectorInsertTableHandle tableHandle) { HiveInsertTableHandle handle = checkType(tableHandle, HiveInsertTableHandle.class, "tableHandle"); - return createPageSink(session, handle, false); + return createPageSink(handle, false); } - private ConnectorPageSink createPageSink(ConnectorSession session, HiveWritableTableHandle handle, boolean isCreateTable) + private ConnectorPageSink createPageSink(HiveWritableTableHandle handle, boolean isCreateTable) { return new HivePageSink( - session, handle.getSchemaName(), handle.getTableName(), isCreateTable, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index b41f6cb702fc..37eb15d30025 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -100,7 +100,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(session, metastore, tableName); + Table table = getTable(metastore, tableName); Optional bucket = getHiveBucket(table, effectivePredicate.extractFixedValues()); TupleDomain compactEffectivePredicate = toCompactTupleDomain(effectivePredicate, domainCompactionThreshold); @@ -110,7 +110,7 @@ public HivePartitionResult getPartitions(ConnectorSession session, HiveMetastore } List partitionColumns = getPartitionKeyColumnHandles(connectorId, table); - List partitionNames = getFilteredPartitionNames(session, metastore, tableName, partitionColumns, effectivePredicate); + List partitionNames = getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate); // do a final pass to filter based on fields that could not be used to filter the partitions ImmutableList.Builder partitions = ImmutableList.builder(); @@ -164,9 +164,9 @@ private Optional> parseValuesAndFilte return Optional.of(builder.build()); } - private Table getTable(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName) + private Table getTable(HiveMetastore metastore, SchemaTableName tableName) { - Optional
target = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } @@ -185,7 +185,7 @@ private Table getTable(ConnectorSession session, HiveMetastore metastore, Schema return table; } - private List getFilteredPartitionNames(ConnectorSession session, HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) + private List getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) { List filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { @@ -217,7 +217,7 @@ else if ((value instanceof Boolean) || (value instanceof Double) || (value insta } // fetch the partition names - return metastore.getPartitionNamesByParts(session.getUser(), tableName.getSchemaName(), tableName.getTableName(), filter) + return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 1506fc8f3c64..8f63692545bc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -157,11 +157,11 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa // sort partitions partitions = Ordering.natural().onResultOf(HivePartition::getPartitionId).reverse().sortedCopy(partitions); - Optional
table = metastore.getTable(session.getUser(), tableName.getSchemaName(), tableName.getTableName()); + Optional
table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!table.isPresent()) { throw new TableNotFoundException(tableName); } - Iterable hivePartitions = getPartitionMetadata(session, table.get(), tableName, partitions); + Iterable hivePartitions = getPartitionMetadata(table.get(), tableName, partitions); HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader( connectorId, @@ -185,7 +185,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa return splitSource; } - private Iterable getPartitionMetadata(ConnectorSession session, Table table, SchemaTableName tableName, List hivePartitions) + private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) { if (hivePartitions.isEmpty()) { return ImmutableList.of(); @@ -201,7 +201,6 @@ private Iterable getPartitionMetadata(ConnectorSession se Iterable> partitionNameBatches = partitionExponentially(hivePartitions, minPartitionBatchSize, maxPartitionBatchSize); Iterable> partitionBatches = transform(partitionNameBatches, partitionBatch -> { Optional> batch = metastore.getPartitionsByNames( - session.getUser(), tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId)); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 0ebaa8158583..58e7aa397a98 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -14,7 +14,6 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.metastore.HiveMetastore; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; @@ -302,9 +301,9 @@ private static void checkWritable( } } - public static Path getTableDefaultLocation(ConnectorSession session, HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) + public static Path getTableDefaultLocation(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) { - String location = getDatabase(session, metastore, schemaName).getLocationUri(); + String location = getDatabase(metastore, schemaName).getLocationUri(); if (isNullOrEmpty(location)) { throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not set", schemaName)); } @@ -320,9 +319,9 @@ public static Path getTableDefaultLocation(ConnectorSession session, HiveMetasto return new Path(databasePath, tableName); } - private static Database getDatabase(ConnectorSession session, HiveMetastore metastore, String database) + private static Database getDatabase(HiveMetastore metastore, String database) { - return metastore.getDatabase(session.getUser(), database).orElseThrow(() -> new SchemaNotFoundException(database)); + return metastore.getDatabase(database).orElseThrow(() -> new SchemaNotFoundException(database)); } public static boolean pathExists(HdfsEnvironment hdfsEnvironment, Path path) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index ed668aabb338..75d7074a543a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -305,7 +305,7 @@ private static Map getAll(LoadingCache cache, Iterable key } @Override - public List getAllDatabases(String user) + public List getAllDatabases() { return get(databaseNamesCache, ""); } @@ -328,7 +328,7 @@ private List loadAllDatabases() } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { return get(databaseCache, databaseName); } @@ -355,7 +355,7 @@ private Optional loadDatabase(String databaseName) } @Override - public Optional> getAllTables(String user, String databaseName) + public Optional> getAllTables(String databaseName) { return get(tableNamesCache, databaseName); } @@ -398,13 +398,13 @@ private Optional> loadAllTables(String databaseName) } @Override - public Optional
getTable(String user, String databaseName, String tableName) + public Optional
getTable(String databaseName, String tableName) { return get(tableCache, HiveTableName.table(databaseName, tableName)); } @Override - public Optional> getAllViews(String user, String databaseName) + public Optional> getAllViews(String databaseName) { return get(viewNamesCache, databaseName); } @@ -432,7 +432,7 @@ private Optional> loadAllViews(String databaseName) } @Override - public void createTable(String user, Table table) + public void createTable(Table table) { try { retry() @@ -467,7 +467,7 @@ public void createTable(String user, Table table) } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { try { retry() @@ -506,7 +506,7 @@ protected void invalidateTable(String databaseName, String tableName) } @Override - public void alterTable(String user, String databaseName, String tableName, Table table) + public void alterTable(String databaseName, String tableName, Table table) { try { retry() @@ -570,7 +570,7 @@ private Optional
loadTable(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) + public Optional> getPartitionNames(String databaseName, String tableName) { return get(partitionNamesCache, HiveTableName.table(databaseName, tableName)); } @@ -602,7 +602,7 @@ private Optional> loadPartitionNames(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) { return get(partitionFilterCache, PartitionFilter.partitionFilter(databaseName, tableName, parts)); } @@ -632,7 +632,7 @@ private Optional> loadPartitionNamesByParts(PartitionFilter partiti } @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) + public void addPartitions(String databaseName, String tableName, List partitions) { if (partitions.isEmpty()) { return; @@ -676,7 +676,7 @@ public void addPartitions(String user, String databaseName, String tableName, Li } @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) + public void dropPartition(String databaseName, String tableName, List parts) { try { retry() @@ -707,7 +707,7 @@ public void dropPartition(String user, String databaseName, String tableName, Li } @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String databaseName, String tableName, String partitionName) { try { retry() @@ -753,7 +753,7 @@ private void invalidatePartitionCache(String databaseName, String tableName) } @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) { Iterable names = transform(partitionNames, name -> HivePartitionName.partition(databaseName, tableName, name)); @@ -769,7 +769,7 @@ public Optional> getPartitionsByNames(String user, String } @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + public Optional getPartition(String databaseName, String tableName, String partitionName) { HivePartitionName name = HivePartitionName.partition(databaseName, tableName, partitionName); return get(partitionCache, name); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java index 066cb9576a23..49e9b71da794 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastore.java @@ -30,43 +30,43 @@ public interface HiveMetastore { String DEFAULT_DATABASE_NAME = "default"; - void createTable(String user, Table table); + void createTable(Table table); - void dropTable(String user, String databaseName, String tableName); + void dropTable(String databaseName, String tableName); - void alterTable(String user, String databaseName, String tableName, Table table); + void alterTable(String databaseName, String tableName, Table table); @Managed void flushCache(); - List getAllDatabases(String user); + List getAllDatabases(); - Optional> getAllTables(String user, String databaseName); + Optional> getAllTables(String databaseName); - Optional> getAllViews(String user, String databaseName); + Optional> getAllViews(String databaseName); - Optional getDatabase(String user, String databaseName); + Optional getDatabase(String databaseName); /** * Adds partitions to the table in a single atomic task. The implementation * must either add all partitions and return normally, or add no partitions and * throw an exception. */ - void addPartitions(String user, String databaseName, String tableName, List partitions); + void addPartitions(String databaseName, String tableName, List partitions); - void dropPartition(String user, String databaseName, String tableName, List parts); + void dropPartition(String databaseName, String tableName, List parts); - void dropPartitionByName(String user, String databaseName, String tableName, String partitionName); + void dropPartitionByName(String databaseName, String tableName, String partitionName); - Optional> getPartitionNames(String user, String databaseName, String tableName); + Optional> getPartitionNames(String databaseName, String tableName); - Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts); + Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); - Optional getPartition(String user, String databaseName, String tableName, String partitionName); + Optional getPartition(String databaseName, String tableName, String partitionName); - Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames); + Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames); - Optional
getTable(String user, String databaseName, String tableName); + Optional
getTable(String databaseName, String tableName); Set getRoles(String user); @@ -81,7 +81,7 @@ default boolean isDatabaseOwner(String user, String databaseName) return true; } - Optional databaseMetadata = getDatabase(user, databaseName); + Optional databaseMetadata = getDatabase(databaseName); if (!databaseMetadata.isPresent()) { return false; } @@ -101,7 +101,7 @@ default boolean isDatabaseOwner(String user, String databaseName) default boolean isTableOwner(String user, String databaseName, String tableName) { // a table can only be owned by a user - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); return table.isPresent() && user.equals(table.get().getOwner()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 8a5692904b4c..1cd4adf8046d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -1564,7 +1564,7 @@ protected void doCreateEmptyTable(SchemaTableName tableName, HiveStorageFormat s assertEquals(tableMetadata.getColumns(), createTableColumns); // verify table format - Table table = getMetastoreClient(tableName.getSchemaName()).getTable(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()).get(); + Table table = getMetastoreClient(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); if (!table.getSd().getInputFormat().equals(storageFormat.getInputFormat())) { assertEquals(table.getSd().getInputFormat(), storageFormat.getInputFormat()); } @@ -1678,7 +1678,7 @@ protected Set listAllDataFiles(ConnectorInsertTableHandle tableHandle) protected Set listAllDataFiles(String schemaName, String tableName) throws IOException { - Table table = metastoreClient.getTable(SESSION.getUser(), schemaName, tableName).get(); + Table table = metastoreClient.getTable(schemaName, tableName).get(); Path path = new Path(table.getSd().getLocation()); Set existingFiles = new HashSet<>(); return listAllDataFiles(path, existingFiles); @@ -1715,7 +1715,7 @@ private void doInsertPartitioned(HiveStorageFormat storageFormat, SchemaTableNam insertData(tableHandle, CREATE_TABLE_PARTITIONED_DATA, SESSION); // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) @@ -1800,7 +1800,7 @@ private void doMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName t } // verify partitions were created - List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(SESSION.getUser(), tableName.getSchemaName(), tableName.getTableName()) + List partitionNames = getMetastoreClient(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java index 7869d77b9035..1bb81269e532 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientS3.java @@ -325,7 +325,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF // table, which fails without explicit configuration for S3. // We work around that by using a dummy location when creating the // table and update it here to the correct S3 location. - metastoreClient.updateTableLocation(SESSION.getUser(), database, tableName.getTableName(), outputHandle.getWritePath().get()); + metastoreClient.updateTableLocation(database, tableName.getTableName(), outputHandle.getWritePath().get()); // load the new table ConnectorTableHandle tableHandle = getTableHandle(tableName); @@ -352,7 +352,7 @@ private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageF private void dropTable(SchemaTableName table) { try { - metastoreClient.dropTable(SESSION.getUser(), table.getSchemaName(), table.getTableName()); + metastoreClient.dropTable(table.getSchemaName(), table.getTableName()); } catch (RuntimeException e) { // this usually occurs because the table was not created @@ -402,9 +402,9 @@ public TestingHiveMetastore(HiveCluster hiveCluster, ExecutorService executor, H } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { - Optional database = super.getDatabase(user, databaseName); + Optional database = super.getDatabase(databaseName); if (database.isPresent()) { database.get().setLocationUri("s3://" + writableBucket + "/"); } @@ -412,18 +412,18 @@ public Optional getDatabase(String user, String databaseName) } @Override - public void createTable(String user, Table table) + public void createTable(Table table) { // hack to work around the metastore not being configured for S3 table.getSd().setLocation("/"); - super.createTable(user, table); + super.createTable(table); } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { try { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -449,10 +449,10 @@ public void dropTable(String user, String databaseName, String tableName) } } - public void updateTableLocation(String user, String databaseName, String tableName, String location) + public void updateTableLocation(String databaseName, String tableName, String location) { try { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java index 56bb639c7f33..fa8a81344c20 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/InMemoryHiveMetastore.java @@ -88,13 +88,13 @@ public void createDatabase(Database database) } @Override - public List getAllDatabases(String user) + public List getAllDatabases() { return ImmutableList.copyOf(databases.keySet()); } @Override - public void createTable(String user1, Table table) + public void createTable(Table table) { SchemaTableName schemaTableName = new SchemaTableName(table.getDbName(), table.getTableName()); Table tableCopy = table.deepCopy(); @@ -137,7 +137,7 @@ else if (tableCopy.getSd().getLocation() != null) { } @Override - public void dropTable(String user, String databaseName, String tableName) + public void dropTable(String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); @@ -159,7 +159,7 @@ public void dropTable(String user, String databaseName, String tableName) } @Override - public void alterTable(String user, String databaseName, String tableName, Table newTable) + public void alterTable(String databaseName, String tableName, Table newTable) { SchemaTableName oldName = new SchemaTableName(databaseName, tableName); SchemaTableName newName = new SchemaTableName(newTable.getDbName(), newTable.getTableName()); @@ -186,7 +186,7 @@ public void alterTable(String user, String databaseName, String tableName, Table } @Override - public Optional> getAllTables(String user, String databaseName) + public Optional> getAllTables(String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.relations.keySet()) { @@ -198,7 +198,7 @@ public Optional> getAllTables(String user, String databaseName) } @Override - public Optional> getAllViews(String user, String databaseName) + public Optional> getAllViews(String databaseName) { ImmutableList.Builder tables = ImmutableList.builder(); for (SchemaTableName schemaTableName : this.views.keySet()) { @@ -210,15 +210,15 @@ public Optional> getAllViews(String user, String databaseName) } @Override - public Optional getDatabase(String user, String databaseName) + public Optional getDatabase(String databaseName) { return Optional.ofNullable(databases.get(databaseName)); } @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) + public void addPartitions(String databaseName, String tableName, List partitions) { - Optional
table = getTable(user, databaseName, tableName); + Optional
table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } @@ -233,7 +233,7 @@ public void addPartitions(String user, String databaseName, String tableName, Li } @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) + public void dropPartition(String databaseName, String tableName, List parts) { for (Entry entry : partitions.entrySet()) { PartitionName partitionName = entry.getKey(); @@ -245,7 +245,7 @@ public void dropPartition(String user, String databaseName, String tableName, Li } @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) + public void dropPartitionByName(String databaseName, String tableName, String partitionName) { for (PartitionName partition : partitions.keySet()) { if (partition.matches(databaseName, tableName, partitionName)) { @@ -255,7 +255,7 @@ public void dropPartitionByName(String user, String databaseName, String tableNa } @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) + public Optional> getPartitionNames(String databaseName, String tableName) { return Optional.of(ImmutableList.copyOf(partitions.entrySet().stream() .filter(entry -> entry.getKey().matches(databaseName, tableName)) @@ -264,7 +264,7 @@ public Optional> getPartitionNames(String user, String databaseName } @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) + public Optional getPartition(String databaseName, String tableName, String partitionName) { PartitionName name = new PartitionName(databaseName, tableName, partitionName); Partition partition = partitions.get(name); @@ -275,7 +275,7 @@ public Optional getPartition(String user, String databaseName, String } @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) + public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) { return Optional.of(partitions.entrySet().stream() .filter(entry -> partitionMatches(entry.getValue(), databaseName, tableName, parts)) @@ -303,7 +303,7 @@ private static boolean partitionMatches(Partition partition, String databaseName } @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) + public Optional> getPartitionsByNames(String databaseName, String tableName, List partitionNames) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (String name : partitionNames) { @@ -318,7 +318,7 @@ public Optional> getPartitionsByNames(String user, String } @Override - public Optional
getTable(String user, String databaseName, String tableName) + public Optional
getTable(String databaseName, String tableName) { SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); return Optional.ofNullable(relations.get(schemaTableName)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index ac55e00933fa..b0643d13a7e8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -22,7 +22,6 @@ import java.util.concurrent.TimeUnit; -import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.BAD_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_DATABASE; import static com.facebook.presto.hive.metastore.MockHiveMetastoreClient.TEST_PARTITION1; @@ -56,14 +55,14 @@ public void testGetAllDatabases() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllDatabases(SESSION.getUser()), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); } @@ -72,21 +71,21 @@ public void testGetAllTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getAllTables(SESSION.getUser(), TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); + assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetAllTAbles() throws Exception { - assertFalse(metastore.getAllTables(SESSION.getUser(), BAD_DATABASE).isPresent()); + assertFalse(metastore.getAllTables(BAD_DATABASE).isPresent()); } @Test @@ -94,21 +93,21 @@ public void testGetTable() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertNotNull(metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidDbGetTable() throws Exception { - assertFalse(metastore.getTable(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).isPresent()); + assertFalse(metastore.getTable(BAD_DATABASE, TEST_TABLE).isPresent()); } @Test @@ -117,14 +116,14 @@ public void testGetPartitionNames() { ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -132,7 +131,7 @@ public void testGetPartitionNames() public void testInvalidGetPartitionNames() throws Exception { - assertEquals(metastore.getPartitionNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); + assertEquals(metastore.getPartitionNames(BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); } @Test @@ -143,14 +142,14 @@ public void testGetPartitionNamesByParts() ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNamesByParts(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -158,7 +157,7 @@ public void testInvalidGetPartitionNamesByParts() throws Exception { ImmutableList parts = ImmutableList.of(); - assertFalse(metastore.getPartitionNamesByParts(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, parts).isPresent()); + assertFalse(metastore.getPartitionNamesByParts(BAD_DATABASE, TEST_TABLE, parts).isPresent()); } @Test @@ -166,35 +165,35 @@ public void testGetPartitionsByNames() throws Exception { assertEquals(mockClient.getAccessCount(), 0); - metastore.getTable(SESSION.getUser(), TEST_DATABASE, TEST_TABLE); + metastore.getTable(TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).get().size(), 1); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 3); metastore.flushCache(); // Fetching both should only result in one batched access - assertEquals(metastore.getPartitionsByNames(SESSION.getUser(), TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); + assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).get().size(), 2); assertEquals(mockClient.getAccessCount(), 4); } public void testInvalidGetPartitionsByNames() throws Exception { - assertFalse(metastore.getPartitionsByNames(SESSION.getUser(), BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); + assertFalse(metastore.getPartitionsByNames(BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).isPresent()); } @Test @@ -204,7 +203,7 @@ public void testNoCacheExceptions() // Throw exceptions on usage mockClient.setThrowException(true); try { - metastore.getAllDatabases(SESSION.getUser()); + metastore.getAllDatabases(); } catch (RuntimeException ignored) { } @@ -212,7 +211,7 @@ public void testNoCacheExceptions() // Second try should hit the client again try { - metastore.getAllDatabases(SESSION.getUser()); + metastore.getAllDatabases(); } catch (RuntimeException ignored) { } From 8c99eebea149686c123cd1a160d2e8bea9a00ca9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:54:04 -0800 Subject: [PATCH 021/151] Revert "Load splits as user so that hdfs reads happen as that user." This reverts commit a39315881019902b237d11540219a1c94b7ecc0a. --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 557544a96b6b..82defdecc369 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,10 +38,8 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -164,9 +162,8 @@ public TaskStatus process() try { CompletableFuture future; taskExecutionLock.readLock().lock(); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - future = ugi.doAs((PrivilegedExceptionAction>) BackgroundHiveSplitLoader.this::loadSplits); + future = loadSplits(); } finally { taskExecutionLock.readLock().unlock(); From 884902f5dfd41f6be39454bc1c90afee9d4a3f3b Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 10:54:41 -0800 Subject: [PATCH 022/151] Revert "UserBasedHiveMetastore to access metastore as user issuing the query." This reverts commit 1b65e2fad9ac91da587e0193b114933461b7fcb6. --- .../presto/hive/HiveClientModule.java | 6 +- .../hive/ThriftHiveMetastoreClient.java | 5 - .../metastore/UserBasedHiveMetastore.java | 608 ------------------ .../UserBasedHiveMetastoreStats.java | 141 ---- 4 files changed, 3 insertions(+), 757 deletions(-) delete mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java delete mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index de95fec6e6fb..bc48cbfc7f0f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastore; -import com.facebook.presto.hive.metastore.UserBasedHiveMetastore; import com.facebook.presto.hive.orc.DwrfPageSourceFactory; import com.facebook.presto.hive.orc.DwrfRecordCursorProvider; import com.facebook.presto.hive.orc.OrcPageSourceFactory; @@ -81,9 +81,9 @@ public void configure(Binder binder) binder.bind(HiveMetastore.class).toInstance(metastore); } else { - binder.bind(HiveMetastore.class).to(UserBasedHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(HiveMetastore.class).to(CachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(HiveMetastore.class) - .as(generatedNameOf(UserBasedHiveMetastore.class, connectorId)); + .as(generatedNameOf(CachingHiveMetastore.class, connectorId)); } binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java index 9ab9986b7bab..d97aaf3d464f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ThriftHiveMetastoreClient.java @@ -49,11 +49,6 @@ public ThriftHiveMetastoreClient(TProtocol protocol) this.client = new ThriftHiveMetastore.Client(protocol); } - public ThriftHiveMetastore.Client getClient() - { - return client; - } - @Override public void close() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java deleted file mode 100644 index 251962a01cc9..000000000000 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastore.java +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive.metastore; - -import com.facebook.presto.hive.HiveCluster; -import com.facebook.presto.hive.HiveViewNotSupportedException; -import com.facebook.presto.hive.TableAlreadyExistsException; -import com.facebook.presto.hive.ThriftHiveMetastoreClient; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.SchemaNotFoundException; -import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.TableNotFoundException; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.HiveObjectType; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; -import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.UnknownDBException; -import org.apache.thrift.TException; -import org.weakref.jmx.Flatten; -import org.weakref.jmx.Managed; - -import javax.inject.Inject; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.function.Function; - -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; -import static com.facebook.presto.hive.HiveUtil.isPrestoView; -import static com.facebook.presto.hive.RetryDriver.retry; -import static com.facebook.presto.hive.metastore.HivePrivilege.OWNERSHIP; -import static com.facebook.presto.hive.metastore.HivePrivilege.parsePrivilege; -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toSet; -import static org.apache.hadoop.hive.metastore.api.PrincipalType.USER; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS; - -public class UserBasedHiveMetastore - implements HiveMetastore -{ - private final UserBasedHiveMetastoreStats stats = new UserBasedHiveMetastoreStats(); - protected final HiveCluster clientProvider; - - @Inject - public UserBasedHiveMetastore(HiveCluster hiveCluster) - { - this.clientProvider = requireNonNull(hiveCluster, "hiveCluster is null"); - } - - @Managed - @Flatten - public UserBasedHiveMetastoreStats getStats() - { - return stats; - } - - protected Function getExceptionMapper() - { - return Function.identity(); - } - - @Override - public void createTable(String user, Table table) - { - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(AlreadyExistsException.class, InvalidObjectException.class, MetaException.class, NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("createTable", stats.getCreateTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.createTable(table); - } - return null; - })); - } - catch (AlreadyExistsException e) { - throw new TableAlreadyExistsException(new SchemaTableName(table.getDbName(), table.getTableName())); - } - catch (NoSuchObjectException e) { - throw new SchemaNotFoundException(table.getDbName()); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw Throwables.propagate(e); - } - } - - @Override - public void dropTable(String user, String databaseName, String tableName) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("dropTable", stats.getDropTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropTable(databaseName, tableName, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void alterTable(String user, String databaseName, String tableName, Table table) - { - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(InvalidOperationException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("alterTable", stats.getAlterTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - Optional
source = getTable(user, databaseName, tableName); - if (!source.isPresent()) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - client.getClient().set_ugi(user, ImmutableList.of()); - client.alterTable(databaseName, tableName, table); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (InvalidOperationException | MetaException e) { - throw com.google.common.base.Throwables.propagate(e); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void flushCache() - { - } - - @Override - public List getAllDatabases(String user) - { - try { - return retry() - .stopOnIllegalExceptions() - .run("getAllDatabases", stats.getGetAllDatabases().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return client.getAllDatabases(); - } - })); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getAllTables(String user, String databaseName) - { - Callable> getAllTables = stats.getGetAllTables().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return client.getAllTables(databaseName); - } - }); - - Callable getDatabase = stats.getGetDatabase().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.getDatabase(databaseName); - return null; - } - }); - - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getAllTables", () -> { - List tables = getAllTables.call(); - if (tables.isEmpty()) { - // Check to see if the database exists - getDatabase.call(); - } - return Optional.of(tables); - }); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getAllViews(String user, String databaseName) - { - try { - return retry() - .stopOn(UnknownDBException.class) - .stopOnIllegalExceptions() - .run("getAllViews", stats.getAllViews().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - String filter = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\""; - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getTableNamesByFilter(databaseName, filter)); - } - })); - } - catch (UnknownDBException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional getDatabase(String user, String databaseName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getDatabase", stats.getGetDatabase().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getDatabase(databaseName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public void addPartitions(String user, String databaseName, String tableName, List partitions) - { - if (partitions.isEmpty()) { - return; - } - try { - retry() - .exceptionMapper(getExceptionMapper()) - .stopOn(org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class, org.apache.hadoop.hive.metastore.api.InvalidObjectException.class, MetaException.class, org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, PrestoException.class) - .stopOnIllegalExceptions() - .run("addPartitions", stats.getAddPartitions().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - int partitionsAdded = client.addPartitions(partitions); - if (partitionsAdded != partitions.size()) { - throw new PrestoException(HIVE_METASTORE_ERROR, - format("Hive metastore only added %s of %s partitions", partitionsAdded, partitions.size())); - } - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException | org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - // todo partition already exists exception - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void dropPartition(String user, String databaseName, String tableName, List parts) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("dropPartition", stats.getDropPartition().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropPartition(databaseName, tableName, parts, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public void dropPartitionByName(String user, String databaseName, String tableName, String partitionName) - { - try { - retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, MetaException.class) - .stopOnIllegalExceptions() - .run("dropPartitionByName", stats.getDropPartitionByName().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - // It is observed that: (examples below assumes a table with one partition column `ds`) - // * When a partition doesn't exist (e.g. ds=2015-09-99), this thrift call is a no-op. It doesn't throw any exception. - // * When a typo exists in partition column name (e.g. dxs=2015-09-01), this thrift call will delete ds=2015-09-01. - client.getClient().set_ugi(user, ImmutableList.of()); - client.dropPartitionByName(databaseName, tableName, partitionName, true); - } - return null; - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw com.google.common.base.Throwables.propagate(e); - } - } - - @Override - public Optional> getPartitionNames(String user, String databaseName, String tableName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionNames", stats.getGetPartitionNames().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionNames(databaseName, tableName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getPartitionNamesByParts(String user, String databaseName, String tableName, List parts) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionNamesByParts", stats.getGetPartitionNamesPs().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionNamesFiltered(databaseName, tableName, parts)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional getPartition(String user, String databaseName, String tableName, String partitionName) - { - requireNonNull(partitionName, "partitionName is null"); - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionsByNames", stats.getGetPartitionByName().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - return Optional.of(client.getPartitionByName(databaseName, tableName, partitionName)); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getPartitionsByNames(String user, String databaseName, String tableName, List partitionNames) - { - requireNonNull(partitionNames, "partitionNames is null"); - checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); - - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class) - .stopOnIllegalExceptions() - .run("getPartitionsByNames", stats.getGetPartitionsByNames().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - List partitionColumnNames = ImmutableList.copyOf(Warehouse.makeSpecFromName(partitionNames.get(0)).keySet()); - ImmutableMap.Builder partitions = ImmutableMap.builder(); - client.getClient().set_ugi(user, ImmutableList.of()); - for (Partition partition : client.getPartitionsByNames(databaseName, tableName, partitionNames)) { - String partitionId = FileUtils.makePartName(partitionColumnNames, partition.getValues(), null); - partitions.put(partitionId, partition); - } - return Optional.of(partitions.build()); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - // assume none of the partitions in the batch are available - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional
getTable(String user, String databaseName, String tableName) - { - try { - return retry() - .stopOn(org.apache.hadoop.hive.metastore.api.NoSuchObjectException.class, HiveViewNotSupportedException.class) - .stopOnIllegalExceptions() - .run("getTable", stats.getGetTable().wrap(() -> { - try (ThriftHiveMetastoreClient client = (ThriftHiveMetastoreClient) clientProvider.createMetastoreClient()) { - client.getClient().set_ugi(user, ImmutableList.of()); - Table table = client.getTable(databaseName, tableName); - if (table.getTableType().equals(TableType.VIRTUAL_VIEW.name()) && (!isPrestoView(table))) { - throw new HiveViewNotSupportedException(new SchemaTableName(databaseName, tableName)); - } - return Optional.of(table); - } - })); - } - catch (org.apache.hadoop.hive.metastore.api.NoSuchObjectException e) { - return Optional.empty(); - } - catch (Exception e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Set getRoles(String user) - { - try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { - List roles = client.listRoles(user, USER); - if (roles == null) { - return ImmutableSet.of(); - } - return ImmutableSet.copyOf(roles.stream() - .map(Role::getRoleName) - .collect(toSet())); - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Set getDatabasePrivileges(String user, String databaseName) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - - if (isDatabaseOwner(user, databaseName)) { - privileges.add(OWNERSHIP); - } - privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.DATABASE, databaseName, null, null, null))); - - return privileges.build(); - } - - @Override - public Set getTablePrivileges(String user, String databaseName, String tableName) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - - if (isTableOwner(user, databaseName, tableName)) { - privileges.add(OWNERSHIP); - } - privileges.addAll(getPrivileges(user, new HiveObjectRef(HiveObjectType.TABLE, databaseName, tableName, null, null))); - - return privileges.build(); - } - - private Set getPrivileges(String user, HiveObjectRef objectReference) - { - ImmutableSet.Builder privileges = ImmutableSet.builder(); - try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { - PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null); - - if (privilegeSet != null) { - Map> userPrivileges = privilegeSet.getUserPrivileges(); - if (userPrivileges != null) { - privileges.addAll(toGrants(userPrivileges.get(user))); - } - for (List rolePrivileges : privilegeSet.getRolePrivileges().values()) { - privileges.addAll(toGrants(rolePrivileges)); - } - // We do not add the group permissions as Hive does not seem to process these - } - } - catch (TException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - - return privileges.build(); - } - - private static Set toGrants(List userGrants) - { - if (userGrants == null) { - return ImmutableSet.of(); - } - - ImmutableSet.Builder privileges = ImmutableSet.builder(); - for (PrivilegeGrantInfo userGrant : userGrants) { - privileges.addAll(parsePrivilege(userGrant)); - if (userGrant.isGrantOption()) { - privileges.add(HivePrivilege.GRANT); - } - } - return privileges.build(); - } -} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java deleted file mode 100644 index 7778e2ba9010..000000000000 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserBasedHiveMetastoreStats.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive.metastore; - -import org.weakref.jmx.Managed; -import org.weakref.jmx.Nested; - -public class UserBasedHiveMetastoreStats -{ - private final HiveMetastoreApiStats getAllDatabases = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getDatabase = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getAllTables = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getAllViews = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionNames = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionNamesPs = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionByName = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats getPartitionsByNames = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats createTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats alterTable = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats addPartitions = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropPartition = new HiveMetastoreApiStats(); - private final HiveMetastoreApiStats dropPartitionByName = new HiveMetastoreApiStats(); - - @Managed - @Nested - public HiveMetastoreApiStats getGetAllDatabases() - { - return getAllDatabases; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetDatabase() - { - return getDatabase; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetAllTables() - { - return getAllTables; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAllViews() - { - return getAllViews; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetTable() - { - return getTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionNames() - { - return getPartitionNames; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionNamesPs() - { - return getPartitionNamesPs; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionByName() - { - return getPartitionByName; - } - - @Managed - @Nested - public HiveMetastoreApiStats getGetPartitionsByNames() - { - return getPartitionsByNames; - } - - @Managed - @Nested - public HiveMetastoreApiStats getCreateTable() - { - return createTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropTable() - { - return dropTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAlterTable() - { - return alterTable; - } - - @Managed - @Nested - public HiveMetastoreApiStats getAddPartitions() - { - return addPartitions; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropPartition() - { - return dropPartition; - } - - @Managed - @Nested - public HiveMetastoreApiStats getDropPartitionByName() - { - return dropPartitionByName; - } -} From fe37227c083ee62b209845b90799d4aa0ea3a238 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 23 Nov 2015 11:45:41 -0800 Subject: [PATCH 023/151] Prepare for release 0.126-tw-0.7. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 52e309f6df08..414fc84a115a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126 + 0.126-tw-0.7 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 0cc6cf09234b..9a133d8e8f0f 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 9e0761b15f3d..3f218464e80f 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 3866f58c52ee..96fc271fb37c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index e91a54b509c9..5344477eda37 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 03ef5b2a0230..0a882093d15c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 46f0657d2713..ad4ec98b26bc 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 93a19ec2c54b..ac9647ce93ab 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 55dc44d383e4..f675e09c07fa 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 52f83773c3fd..e459541f3f49 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index a5a2688113f8..3a153832ad5b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 38793b1cc542..4b412e4133e1 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 7fcd9c617283..c1bfac684b13 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 0c6c17a88228..29994c4839ac 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 9a80e1608146..5498b04ad31e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d1b20af9b4b8..275b9946f55d 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5934d498d5f8..5c7cd51fbd15 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 0ed7ee924f7d..bac1e1f514d0 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ea9e6d448a2a..013447fe2411 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c0c6cf3bdba7..ddaf72fbbe69 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 2d4c8893a501..45c9d8a3ee59 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 224f07d32d5b..892e52942c88 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index baa497e03aa1..a04a1810cacf 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 31111245dbea..7e2c6e0aafe6 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index ada21024306c..d1056e3df9a7 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 71f1ea8d79e3..d9a3f6ef1fbb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 7fcfc62e80c6..2bfe3c924b40 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e438bee2d5a7..e512dc5d32f8 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d6dc67beb915..e053c3aac08b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index e919eea24cdf..5832a24c82d5 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index df3aaba4e79f..5c9bf5584c84 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index b069db9c4c96..a9344873b22f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 59fbc9d4518f..966add9b8b0a 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 5842d70b3388..016b69ffc3b1 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126 + 0.126-tw-0.7 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8362c3721e8a..60821655b780 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index cc46b69468d1..08dafd617da6 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126 + 0.126-tw-0.7 presto-verifier From 344c8f0b4e90e38a26f443b9ad090f632ae54478 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 11:19:49 -0800 Subject: [PATCH 024/151] HiveSplitManager to start HiveSplitLoader as session user --- .../presto/hive/HiveSplitManager.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index 8f63692545bc..fcca3ab5bdc1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -34,9 +34,12 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.security.UserGroupInformation; import javax.inject.Inject; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -180,9 +183,22 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa recursiveDfsWalkerEnabled); HiveSplitSource splitSource = new HiveSplitSource(connectorId, maxOutstandingSplits, hiveSplitLoader, executor); - hiveSplitLoader.start(splitSource); - return splitSource; + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + + try { + ugi.doAs((PrivilegedExceptionAction) + () -> { + hiveSplitLoader.start(splitSource); + return null; + } + ); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); + } + + return splitSource; } private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) From 378c4ed24ff10ca8954d1857aa1fd55d949c7873 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 12:30:36 -0800 Subject: [PATCH 025/151] fixing indentation --- .../facebook/presto/hive/HiveSplitManager.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index fcca3ab5bdc1..b04d13ae99cf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -187,18 +187,18 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); try { - ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } - ); + ugi.doAs((PrivilegedExceptionAction) + () -> { + hiveSplitLoader.start(splitSource); + return null; + } + ); } catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); + throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); } - return splitSource; + return splitSource; } private Iterable getPartitionMetadata(Table table, SchemaTableName tableName, List hivePartitions) From 484e2554ecd3f8695231f2ca624a7578c33d6686 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 12:48:49 -0800 Subject: [PATCH 026/151] fixing indentation --- .../java/com/facebook/presto/hive/HiveSplitManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index b04d13ae99cf..aa6ba715b7f6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -188,10 +188,10 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa try { ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } + () -> { + hiveSplitLoader.start(splitSource); + return null; + } ); } catch (IOException | InterruptedException e) { From 4300a90172c84b8f5cd3587d1c0a4358b5114e40 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 8 Dec 2015 21:17:06 -0800 Subject: [PATCH 027/151] Updating version from to 0.126-tw-0.8 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 414fc84a115a..87c608a57ce1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126-tw-0.7 + 0.126-tw-0.8 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 9a133d8e8f0f..2b54b534ec61 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3f218464e80f..24f3f314d8cf 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 96fc271fb37c..60b0e963643a 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5344477eda37..44d712cb2f74 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 0a882093d15c..bda12d19948e 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index ad4ec98b26bc..995e766931ac 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index ac9647ce93ab..235a29672d56 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f675e09c07fa..f890cc393d9c 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e459541f3f49..e3f6b094975c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 3a153832ad5b..19a597ed590d 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 4b412e4133e1..1b00d6fbbc66 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index c1bfac684b13..e6884f7c1c4e 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 29994c4839ac..3cec99c7cbf4 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 5498b04ad31e..e837f4b0c0d6 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 275b9946f55d..38847ba75baa 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5c7cd51fbd15..f7776398f801 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index bac1e1f514d0..a99b08424438 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 013447fe2411..195758c2b1ed 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index ddaf72fbbe69..1645784011d6 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 45c9d8a3ee59..3fdeff83cdaa 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 892e52942c88..85d19916b955 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a04a1810cacf..be183abcd5ec 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7e2c6e0aafe6..7f27364dea63 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index d1056e3df9a7..6b82404a6e8b 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index d9a3f6ef1fbb..e7fbecebc5bc 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 2bfe3c924b40..a47ea27e5f17 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e512dc5d32f8..0493d38e3e4b 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index e053c3aac08b..67c18607589f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 5832a24c82d5..cd423b82ff29 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 5c9bf5584c84..93a4a5070b16 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a9344873b22f..b7542fc5167d 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 966add9b8b0a..969316f4e454 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 016b69ffc3b1..d1f9516597a3 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.7 + 0.126-tw-0.8 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 60821655b780..4bfbd1272ef5 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 08dafd617da6..80e27a5c2df0 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.7 + 0.126-tw-0.8 presto-verifier From b00c53a3a350b295bdddaaabbaf4dd9806045cbf Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:12:53 -0800 Subject: [PATCH 028/151] Revert runAsUser in preparation for 0.130 upgrade --- .../facebook/presto/hive/HiveSplitManager.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index aa6ba715b7f6..8f63692545bc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -34,12 +34,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.security.UserGroupInformation; import javax.inject.Inject; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -183,20 +180,7 @@ public ConnectorSplitSource getSplits(ConnectorSession session, ConnectorTableLa recursiveDfsWalkerEnabled); HiveSplitSource splitSource = new HiveSplitSource(connectorId, maxOutstandingSplits, hiveSplitLoader, executor); - - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - - try { - ugi.doAs((PrivilegedExceptionAction) - () -> { - hiveSplitLoader.start(splitSource); - return null; - } - ); - } - catch (IOException | InterruptedException e) { - throw new RuntimeException("Could not runAs " + ugi.getUserName(), e); - } + hiveSplitLoader.start(splitSource); return splitSource; } From adb149c1a125ef6ddc7c97dc24084727b2396766 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:29:15 -0800 Subject: [PATCH 029/151] Changing pom version to 0.130 to avoid merge conflicts when merging in 0.130 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 87c608a57ce1..7b7b6132193d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.126-tw-0.8 + 0.130 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2b54b534ec61..2238de516139 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 24f3f314d8cf..c9055bbaf619 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 60b0e963643a..1503671c800f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 44d712cb2f74..a4db98340a48 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index bda12d19948e..e91e1350506c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 995e766931ac..b1478999daae 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 235a29672d56..68904e2d5b5f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f890cc393d9c..a3fea8108719 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e3f6b094975c..3c9c1d32c05c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 19a597ed590d..fd26c464678e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 1b00d6fbbc66..1ba9d816d070 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index e6884f7c1c4e..146eee7554fc 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 3cec99c7cbf4..476ff742e5fe 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index e837f4b0c0d6..c36826cb6065 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 38847ba75baa..511b51e67e91 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index f7776398f801..5781bb8854fe 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index a99b08424438..88092f60e0e7 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 195758c2b1ed..8c3f7eaa4550 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1645784011d6..46f9c3b44552 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 3fdeff83cdaa..1f284d530c36 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 85d19916b955..e59590ac04ef 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index be183abcd5ec..7ebf10590ad7 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7f27364dea63..23733201a738 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6b82404a6e8b..d4bcbca7ca86 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e7fbecebc5bc..dfabac002bf5 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index a47ea27e5f17..f8e944107093 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 0493d38e3e4b..91b8272d48a6 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 67c18607589f..8747490178eb 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index cd423b82ff29..d2cf4984cc61 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 93a4a5070b16..aaf670e2913d 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index b7542fc5167d..c11cac6fa8a6 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 969316f4e454..986d71502f89 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index d1f9516597a3..624160ecc0c8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.126-tw-0.8 + 0.130 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 4bfbd1272ef5..3be286cebe3a 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 80e27a5c2df0..6963319d3361 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.126-tw-0.8 + 0.130 presto-verifier From e84c4a4e50a5dd1555c4dcdde2531313429fe713 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 21:36:07 -0800 Subject: [PATCH 030/151] Prepare for release 0.130-tw-0.9 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 53aa0c6deed1..c627fc780d5a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130 + 0.130-tw-0.9 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2238de516139..b1cdc92b7133 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c9055bbaf619..2c1cf1e8232d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 1503671c800f..71ac5b6b6a71 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index a4db98340a48..7edb8a4b301f 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index e91e1350506c..70adcfa34e45 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index b1478999daae..9dcd4c01479e 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 68904e2d5b5f..7cee57559957 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index a3fea8108719..0a2a80999767 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 3c9c1d32c05c..c794568a3064 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index fd26c464678e..31a0e5011eba 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 1ba9d816d070..46480d8d82e9 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 146eee7554fc..64deba3e5bc7 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 476ff742e5fe..134384e326a8 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 71d9f68ff8b8..8bcbb3075b9d 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d2555a1ddba6..c7d937faa290 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5781bb8854fe..baa24b1ef8de 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 88092f60e0e7..ff01a6845134 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 8c3f7eaa4550..bcf29c7aa29a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 46f9c3b44552..4b195e0d7925 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 1f284d530c36..d8639669a4bc 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 223ecf28ad9f..073f5f3cafa3 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 7ebf10590ad7..bbfefe90b54c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 23733201a738..8801d9be3117 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index d4bcbca7ca86..7ff1c69c918d 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 0cf57ffa36e9..b95b1d27d1fb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index f8e944107093..4d22fb8ad760 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 91b8272d48a6..4f95e5638e5d 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 8747490178eb..db15e14079e3 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index d2cf4984cc61..8c6a691ba085 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 81ceb2227166..618475375aaf 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index c11cac6fa8a6..cecc9e929c4f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 986d71502f89..e3aa177c836c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 624160ecc0c8..07ecc82a8563 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130 + 0.130-tw-0.9 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 3be286cebe3a..2ce2e06261ff 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 9ffdd566bcd6..5464d09e2217 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130 + 0.130-tw-0.9 presto-verifier From 3861dc30f59e8db824d84931694ce413564b5cb4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 16 Dec 2015 23:41:42 -0800 Subject: [PATCH 031/151] Remove empty line to fix stylecheck error --- .../com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java index 9ccf999edac4..6b82efa22f97 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java @@ -77,7 +77,6 @@ public void destroy() zkMetastoreMonitor.close(); zkClient.close(); zkServer.close(); - } @BeforeTest From 7dd60e54e7d853ea16e3d7473b2af0097037349c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 22 Dec 2015 17:10:01 -0800 Subject: [PATCH 032/151] Allow Hive queries to read as the querying user --- .../hive/BackgroundHiveSplitLoader.java | 24 +++++++++++++++++++ .../presto/hive/HiveClientConfig.java | 15 ++++++++++++ .../presto/hive/HiveSessionProperties.java | 11 +++++++++ 3 files changed, 50 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 0c854f66abcd..d15374abcf75 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -38,8 +38,10 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -154,6 +156,28 @@ private class HiveSplitLoaderTask { @Override public TaskStatus process() + { + if (HiveSessionProperties.getReadAsQueryUser(session)) { + try { + // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users and + // change to the approach below. + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); + + return ugi.doAs((PrivilegedExceptionAction) this::doProcess); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + session.getUser(), e); + } + } + else { + return doProcess(); + } + } + + private TaskStatus doProcess() { while (true) { if (stopped) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 21cf49ce0b87..8ddca5cd58ba 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -67,6 +67,8 @@ public class HiveClientConfig private boolean allowCorruptWritesForTesting; + private boolean readAsQueryUser = false; + private Duration metastoreCacheTtl = new Duration(1, TimeUnit.HOURS); private Duration metastoreRefreshInterval = new Duration(1, TimeUnit.SECONDS); private int maxMetastoreRefreshThreads = 100; @@ -291,6 +293,19 @@ public HiveClientConfig setAllowCorruptWritesForTesting(boolean allowCorruptWrit return this; } + public boolean getReadAsQueryUser() + { + return readAsQueryUser; + } + + @Config("hive.read-as-query-user") + @ConfigDescription("When querying hive read data as the user submitting the query instead of as the presto daemon user") + public HiveClientConfig setReadAsQueryUser(boolean readAsQueryUser) + { + this.readAsQueryUser = readAsQueryUser; + return this; + } + public boolean getAllowAddColumn() { return this.allowAddColumn; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 37bf715c825f..6dd1bc9852a0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -28,6 +28,7 @@ public final class HiveSessionProperties { private static final String FORCE_LOCAL_SCHEDULING = "force_local_scheduling"; + private static final String READ_AS_QUERY_USER = "read_as_query_user"; private static final String OPTIMIZED_READER_ENABLED = "optimized_reader_enabled"; private static final String ORC_MAX_MERGE_DISTANCE = "orc_max_merge_distance"; private static final String ORC_MAX_BUFFER_SIZE = "orc_max_buffer_size"; @@ -51,6 +52,11 @@ public HiveSessionProperties(HiveClientConfig config) "Enable optimized readers", config.isOptimizedReaderEnabled(), true), + booleanSessionProperty( + READ_AS_QUERY_USER, + "Query reads happen as the user submitting the query", + config.getReadAsQueryUser(), + true), dataSizeSessionProperty( ORC_MAX_MERGE_DISTANCE, "ORC: Maximum size of gap between two reads to merge into a single read", @@ -93,6 +99,11 @@ public static boolean isOptimizedReaderEnabled(ConnectorSession session) return session.getProperty(OPTIMIZED_READER_ENABLED, Boolean.class); } + public static boolean getReadAsQueryUser(ConnectorSession session) + { + return session.getProperty(READ_AS_QUERY_USER, Boolean.class); + } + public static boolean isParquetOptimizedReaderEnabled(ConnectorSession session) { return session.getProperty(PARQUET_OPTIMIZED_READER_ENABLED, Boolean.class); From efc25af93ecf4c79f60aa874dc20c8781e735f1e Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 23 Dec 2015 09:45:28 -0800 Subject: [PATCH 033/151] add links to jiras --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index d15374abcf75..1123e33a1f42 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -159,8 +159,8 @@ public TaskStatus process() { if (HiveSessionProperties.getReadAsQueryUser(session)) { try { - // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users and - // change to the approach below. + // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users (HADOOPINFRA-7081) + // and then change to the approach below (IQ-85). // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html // UserGroupInformation ugi = UserGroupInformation.createProxyUser( // session.getUser(), UserGroupInformation.getLoginUser()); From ef06e47416518d7478ab721e01cc5a59cbeb0f81 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 23 Dec 2015 15:16:45 -0800 Subject: [PATCH 034/151] Clean up comments and constant order --- .../presto/hive/BackgroundHiveSplitLoader.java | 4 ++-- .../facebook/presto/hive/HiveSessionProperties.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 1123e33a1f42..82b6d7cb54da 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -159,8 +159,8 @@ public TaskStatus process() { if (HiveSessionProperties.getReadAsQueryUser(session)) { try { - // TODO: Configure hadoop to allow sqlsystem to impersonate all presto users (HADOOPINFRA-7081) - // and then change to the approach below (IQ-85). + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html // UserGroupInformation ugi = UserGroupInformation.createProxyUser( // session.getUser(), UserGroupInformation.getLoginUser()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 6dd1bc9852a0..ab1d4d2c59bf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -28,13 +28,13 @@ public final class HiveSessionProperties { private static final String FORCE_LOCAL_SCHEDULING = "force_local_scheduling"; - private static final String READ_AS_QUERY_USER = "read_as_query_user"; private static final String OPTIMIZED_READER_ENABLED = "optimized_reader_enabled"; private static final String ORC_MAX_MERGE_DISTANCE = "orc_max_merge_distance"; private static final String ORC_MAX_BUFFER_SIZE = "orc_max_buffer_size"; private static final String ORC_STREAM_BUFFER_SIZE = "orc_stream_buffer_size"; private static final String PARQUET_PREDICATE_PUSHDOWN_ENABLED = "parquet_predicate_pushdown_enabled"; private static final String PARQUET_OPTIMIZED_READER_ENABLED = "parquet_optimized_reader_enabled"; + private static final String READ_AS_QUERY_USER = "read_as_query_user"; private final List> sessionProperties; @@ -99,11 +99,6 @@ public static boolean isOptimizedReaderEnabled(ConnectorSession session) return session.getProperty(OPTIMIZED_READER_ENABLED, Boolean.class); } - public static boolean getReadAsQueryUser(ConnectorSession session) - { - return session.getProperty(READ_AS_QUERY_USER, Boolean.class); - } - public static boolean isParquetOptimizedReaderEnabled(ConnectorSession session) { return session.getProperty(PARQUET_OPTIMIZED_READER_ENABLED, Boolean.class); @@ -129,6 +124,11 @@ public static boolean isParquetPredicatePushdownEnabled(ConnectorSession session return session.getProperty(PARQUET_PREDICATE_PUSHDOWN_ENABLED, Boolean.class); } + public static boolean getReadAsQueryUser(ConnectorSession session) + { + return session.getProperty(READ_AS_QUERY_USER, Boolean.class); + } + public static PropertyMetadata dataSizeSessionProperty(String name, String description, DataSize defaultValue, boolean hidden) { return new PropertyMetadata<>( From 2200cdef2bbed2e6260dfd2d2abdaccc9bab3f80 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Jan 2016 10:52:56 -0800 Subject: [PATCH 035/151] Upgrading to 0.130-tw-0.10 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index c627fc780d5a..45f1aab240a9 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.9 + 0.130-tw-0.10 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index b1cdc92b7133..254d1a0a97df 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 2c1cf1e8232d..3551e5b9ba79 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 71ac5b6b6a71..758f46b6829f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 7edb8a4b301f..3098d1160fb0 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 70adcfa34e45..1e6d4890fab2 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 9dcd4c01479e..e7cb584603c0 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 7cee57559957..25be93acdfe5 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0a2a80999767..5cd6892eb6ed 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index c794568a3064..32c2f9e32945 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 31a0e5011eba..593000f9eb64 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 46480d8d82e9..938bb689ee59 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 64deba3e5bc7..06b43819b485 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 134384e326a8..a23dc2923fe8 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 8bcbb3075b9d..0fc8dfc22def 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index c7d937faa290..fe3d827a1628 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index baa24b1ef8de..cfeeb3122bda 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index ff01a6845134..afe37061a74c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index bcf29c7aa29a..19aa2a3e7f7b 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4b195e0d7925..4e15544c3e32 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index d8639669a4bc..eb7bb0041273 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 073f5f3cafa3..58ed96057649 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index bbfefe90b54c..659f51f9e7ee 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8801d9be3117..8b73a4081cee 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 7ff1c69c918d..1868af2c227c 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index b95b1d27d1fb..47a876a3f1a1 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 4d22fb8ad760..bedfad6e0542 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 4f95e5638e5d..67ac2b629ac3 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index db15e14079e3..d9e657b67322 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 8c6a691ba085..1369629d997b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 618475375aaf..be03619e8409 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index cecc9e929c4f..300a83451b02 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index e3aa177c836c..b826dfab48a9 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 07ecc82a8563..0efab43b7181 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.9 + 0.130-tw-0.10 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 2ce2e06261ff..34b329a0af18 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 5464d09e2217..a34d0649d123 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.9 + 0.130-tw-0.10 presto-verifier From 1b38bdf471eb43e90127200d828f8d2ea02b3101 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Mon, 4 Jan 2016 11:46:32 -0800 Subject: [PATCH 036/151] Updating unit tests for ReadAsQueryUser flag --- .../java/com/facebook/presto/hive/TestHiveClientConfig.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index 08311e3c09e1..b7647ee99677 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -66,6 +66,7 @@ public void testDefaults() .setResourceConfigFiles((String) null) .setHiveStorageFormat(HiveStorageFormat.RCBINARY) .setRespectTableFormat(true) + .setReadAsQueryUser(false) .setImmutablePartitions(false) .setMaxPartitionsPerWriter(100) .setUseParquetColumnNames(false) @@ -132,6 +133,7 @@ public void testExplicitPropertyMappings() .put("hive.max-concurrent-file-renames", "100") .put("hive.assume-canonical-partition-keys", "true") .put("hive.parquet.use-column-names", "true") + .put("hive.read-as-query-user", "true") .put("hive.s3.aws-access-key", "abc123") .put("hive.s3.aws-secret-key", "secret") .put("hive.s3.use-instance-credentials", "false") @@ -185,6 +187,7 @@ public void testExplicitPropertyMappings() .setVerifyChecksum(false) .setResourceConfigFiles(ImmutableList.of("/foo.xml", "/bar.xml")) .setHiveStorageFormat(HiveStorageFormat.SEQUENCEFILE) + .setReadAsQueryUser(true) .setRespectTableFormat(false) .setImmutablePartitions(true) .setMaxPartitionsPerWriter(222) From 9f646d1af11a8c43e350d4d4fbc4a9ca31d83019 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 6 Jan 2016 12:33:56 -0800 Subject: [PATCH 037/151] Avoid NPE when ZK changed state is related to connection change. --- .../presto/hive/ZookeeperMetastoreMonitor.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java index 002eefe5cc13..3789ca3c3271 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java @@ -94,31 +94,25 @@ private HostAndPort deserialize(byte[] bytes) @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - String node = ZKPaths.getNodeFromPath(event.getData().getPath()); - switch (event.getType()) { - case CHILD_ADDED: { - HostAndPort hostPort = deserialize(event.getData().getData()); - log.info("child added: " + node + ": " + hostPort); - servers.put(node, hostPort); - break; - } - + case CHILD_ADDED: case CHILD_UPDATED: { HostAndPort hostPort = deserialize(event.getData().getData()); + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); log.info("child updated: " + node + ": " + hostPort); servers.put(node, hostPort); break; } case CHILD_REMOVED: { + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); log.info("child removed: " + node); servers.remove(node); break; } default: - log.info("connection state changed: " + node); + log.info("connection state changed: " + event.getType()); break; } } From 48973308e108c13354c5c7b33823584fd32cad90 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 12 Jan 2016 10:38:13 -0800 Subject: [PATCH 038/151] Cache the UGI object per user in BackgroundHiveSplitLoader --- .../hive/BackgroundHiveSplitLoader.java | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 82b6d7cb54da..4c3c6e2e1312 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -51,6 +51,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -72,6 +73,11 @@ public class BackgroundHiveSplitLoader { public static final CompletableFuture COMPLETED_FUTURE = CompletableFuture.completedFuture(null); + // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due + // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak + // in the PrestoFileSystemCache. + private static final Map UGI_CACHE = new ConcurrentHashMap<>(); + private final String connectorId; private final Table table; private final Optional bucket; @@ -140,8 +146,26 @@ public BackgroundHiveSplitLoader( public void start(HiveSplitSource splitSource) { this.hiveSplitSource = splitSource; + + UserGroupInformation ugi = null; + + if (HiveSessionProperties.getReadAsQueryUser(session)) { + String user = session.getUser(); + ugi = UGI_CACHE.get(user); + + if (ugi == null) { + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + ugi = UserGroupInformation.createRemoteUser(user); + UGI_CACHE.put(user, ugi); + } + } + for (int i = 0; i < maxPartitionBatchSize; i++) { - ResumableTasks.submit(executor, new HiveSplitLoaderTask()); + ResumableTasks.submit(executor, new HiveSplitLoaderTask(ugi)); } } @@ -154,18 +178,18 @@ public void stop() private class HiveSplitLoaderTask implements ResumableTask { + private UserGroupInformation ugi; + + public HiveSplitLoaderTask(UserGroupInformation ugi) + { + this.ugi = ugi; + } + @Override public TaskStatus process() { - if (HiveSessionProperties.getReadAsQueryUser(session)) { + if (ugi != null) { try { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). - // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(session.getUser()); - return ugi.doAs((PrivilegedExceptionAction) this::doProcess); } catch (IOException | InterruptedException e) { From 53ae65b2def1cf7c7dc64e2d46037d1869db5453 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 12 Jan 2016 11:44:36 -0800 Subject: [PATCH 039/151] Upgrading version to 0.130-tw-0.11 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 45f1aab240a9..18a82637339e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.10 + 0.130-tw-0.11 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 254d1a0a97df..38f39e5bed6e 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3551e5b9ba79..c0681a377e35 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 758f46b6829f..ad5f16c3d1ce 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 3098d1160fb0..9d9927732b12 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 1e6d4890fab2..57ede93b43d9 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e7cb584603c0..d7332844143c 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 25be93acdfe5..e9448441e73f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 5cd6892eb6ed..ac21d14d80ec 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 32c2f9e32945..854bb94bd03a 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 593000f9eb64..0c1b2d9b6de5 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 938bb689ee59..0425c49408ed 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 06b43819b485..bfcc07f1f1e8 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index a23dc2923fe8..e82ef33a456f 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 0fc8dfc22def..f5e8c4c1953f 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index fe3d827a1628..3602a338b4f5 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index cfeeb3122bda..897fbb6cad37 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index afe37061a74c..82f8ab1f75e9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 19aa2a3e7f7b..b88f23e6c6a6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4e15544c3e32..9531e434a54a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index eb7bb0041273..dd68037b9691 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 58ed96057649..3f4f329dc9e9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 659f51f9e7ee..20cf118a617c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8b73a4081cee..81edf97a3c5d 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 1868af2c227c..db8f461db9f6 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 47a876a3f1a1..879bbc03bcf0 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index bedfad6e0542..224ea2174faf 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 67ac2b629ac3..d22265c2e95c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d9e657b67322..562371fbc934 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 1369629d997b..399083032f49 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index be03619e8409..64f4a5aae122 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 300a83451b02..a1b8f7b4b9f7 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index b826dfab48a9..0aa645c9613e 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0efab43b7181..7332273d684a 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.10 + 0.130-tw-0.11 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 34b329a0af18..bef17e132820 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index a34d0649d123..e894a2a8420b 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.10 + 0.130-tw-0.11 presto-verifier From eff466b617cc1d1dbf1f25b805b2ecdde33c184c Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 13 Jan 2016 10:56:49 -0800 Subject: [PATCH 040/151] Change version to 0.132. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 18a82637339e..0505afdef136 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.130-tw-0.11 + 0.132 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 38f39e5bed6e..0757573fdeac 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c0681a377e35..7cfc90dd1d09 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index ad5f16c3d1ce..4e97c070cf5b 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 9d9927732b12..1288733d1437 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 57ede93b43d9..3dca182b2de4 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index d7332844143c..135f8845791f 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index e9448441e73f..e6b2500dbed6 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index ac21d14d80ec..892071a00abe 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 854bb94bd03a..1f88313cc127 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 0c1b2d9b6de5..ad4b4a2a0de7 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 0425c49408ed..add0c37510b7 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index bfcc07f1f1e8..8f3ed0661a7a 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index e82ef33a456f..753f94c30bdf 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f5e8c4c1953f..a05910678f98 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 3602a338b4f5..9306604a38b3 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 897fbb6cad37..57c23c70c40a 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 82f8ab1f75e9..311ff49ce349 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b88f23e6c6a6..b0438d65c2ae 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 9531e434a54a..d3f04d7dd2aa 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index dd68037b9691..c0a1a1a32cae 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 3f4f329dc9e9..0b88fc338d2b 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 20cf118a617c..0d05fa046030 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 81edf97a3c5d..a4009d5b00ad 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index db8f461db9f6..4423dbd7e9e0 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 879bbc03bcf0..3b69b26d948f 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 224ea2174faf..ea37942e5e63 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index d22265c2e95c..498a260bd5d2 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 562371fbc934..2ff6d2c1a64e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 399083032f49..9d135250631d 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 64f4a5aae122..b277f10bf706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a1b8f7b4b9f7..36717f601a71 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 0aa645c9613e..27a65f8c462b 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 7332273d684a..23b0093b7898 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.130-tw-0.11 + 0.132 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index bef17e132820..bcc4768dedf1 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index e894a2a8420b..7cfaddeadd23 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.130-tw-0.11 + 0.132 presto-verifier From 14fdca19c6357d2cae3bf743f6f5cce69dee38e8 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Wed, 13 Jan 2016 11:04:41 -0800 Subject: [PATCH 041/151] Change version to 0.132-tw-0.12. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 09213bb52fad..0a7310f24e82 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132 + 0.132-tw-0.12 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 0757573fdeac..1baffbd7964d 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 7cfc90dd1d09..c5d1e97aed38 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 4e97c070cf5b..720c269b8da6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index e51c2780dd01..8e9f44465cd2 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index bbf43c188d3a..a1ca82981c41 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 3dca182b2de4..228bef817de6 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 135f8845791f..f17279106bfa 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index e6b2500dbed6..80c85709d67f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 892071a00abe..3201c365c7eb 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 1f88313cc127..07bb8a3fedfd 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index ad4b4a2a0de7..d2c77960ab34 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index add0c37510b7..e5c4dbf0ded8 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 8f3ed0661a7a..9b50071e2147 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 753f94c30bdf..c33ce09bbf18 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index a05910678f98..062cd3b3acc7 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9306604a38b3..9c99be4db232 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 57c23c70c40a..7d1d123ba81d 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 311ff49ce349..5ab9e91e369c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index e5c954b4aea4..ba634e0b5580 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index d3f04d7dd2aa..f41514dd6603 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index c0a1a1a32cae..dc92ae9833ce 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 0b88fc338d2b..09c469a95f67 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 0d05fa046030..4842899667bb 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index a4009d5b00ad..345c15453b13 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 4423dbd7e9e0..a9844a05f653 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 3b69b26d948f..d9b862a8316a 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index ea37942e5e63..ebfd82dfe86f 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 498a260bd5d2..3098dbba128e 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 2ff6d2c1a64e..c8bad3c5fedd 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9d135250631d..c3e87fb73c03 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index b277f10bf706..80d537ff1e7f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 36717f601a71..7021f5b56d1b 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 27a65f8c462b..2169be00f880 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 8113b83bbc4e..0e30a1f586fb 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132 + 0.132-tw-0.12 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index bcc4768dedf1..8a103229074e 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7cfaddeadd23..18c5b2961b4a 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132 + 0.132-tw-0.12 presto-verifier From 310441d3775ddacce9c69c98c602e75262822718 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 14 Jan 2016 23:31:43 -0800 Subject: [PATCH 042/151] HivePageSourceProvider to support query as user from workers --- .../hive/BackgroundHiveSplitLoader.java | 20 +------- .../presto/hive/HivePageSourceProvider.java | 22 +++++++++ .../facebook/presto/hive/util/UgiUtils.java | 49 +++++++++++++++++++ 3 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 4c3c6e2e1312..676b856e6f7c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.util.HiveFileIterator; import com.facebook.presto.hive.util.ResumableTask; import com.facebook.presto.hive.util.ResumableTasks; +import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; @@ -51,7 +52,6 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -73,11 +73,6 @@ public class BackgroundHiveSplitLoader { public static final CompletableFuture COMPLETED_FUTURE = CompletableFuture.completedFuture(null); - // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due - // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak - // in the PrestoFileSystemCache. - private static final Map UGI_CACHE = new ConcurrentHashMap<>(); - private final String connectorId; private final Table table; private final Optional bucket; @@ -150,18 +145,7 @@ public void start(HiveSplitSource splitSource) UserGroupInformation ugi = null; if (HiveSessionProperties.getReadAsQueryUser(session)) { - String user = session.getUser(); - ugi = UGI_CACHE.get(user); - - if (ugi == null) { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). - // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - ugi = UserGroupInformation.createRemoteUser(user); - UGI_CACHE.put(user, ugi); - } + ugi = UgiUtils.getUgi(session.getUser()); } for (int i = 0; i < maxPartitionBatchSize; i++) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index 73abe1477345..ff1c81f19367 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorPageSourceProvider; @@ -26,10 +27,13 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTimeZone; import javax.inject.Inject; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -67,6 +71,24 @@ public HivePageSourceProvider( @Override public ConnectorPageSource createPageSource(ConnectorSession session, ConnectorSplit split, List columns) + { + if (HiveSessionProperties.getReadAsQueryUser(session)) { + UserGroupInformation ugi = UgiUtils.getUgi(session.getUser()); + try { + return ugi.doAs((PrivilegedExceptionAction) () -> + doCreatePageSource(session, split, columns) + ); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException("Could not runAs " + session.getUser(), e); + } + } + else { + return doCreatePageSource(session, split, columns); + } + } + + private ConnectorPageSource doCreatePageSource(ConnectorSession session, ConnectorSplit split, List columns) { HiveSplit hiveSplit = checkType(split, HiveSplit.class, "split"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java new file mode 100644 index 000000000000..5c8e99e75efa --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.util; + +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Utility class to handle creating and caching the UserGroupInformation object. + */ +public class UgiUtils +{ + private UgiUtils() {} + + // Every instance of a UserGroupInformation object for a given user has a unique hashcode, due + // to the hashCode() impl. If we don't cache the UGI per-user here, there will be a memory leak + // in the PrestoFileSystemCache. + private static final Map UGI_CACHE = new ConcurrentHashMap<>(); + + public static UserGroupInformation getUgi(String user) + { + UserGroupInformation ugi = UGI_CACHE.get(user); + + if (ugi == null) { + // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users + // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html + // UserGroupInformation ugi = UserGroupInformation.createProxyUser( + // session.getUser(), UserGroupInformation.getLoginUser()); + ugi = UserGroupInformation.createRemoteUser(user); + UGI_CACHE.put(user, ugi); + } + + return ugi; + } +} From cab14e9c61d7c756810d7612e559e9bc0fafbc64 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Fri, 15 Jan 2016 13:21:38 -0800 Subject: [PATCH 043/151] Upgrading presto to 0.132-tw-13 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 0a7310f24e82..2ce743f8efef 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.12 + 0.132-tw-0.13 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1baffbd7964d..2a013d29b68b 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c5d1e97aed38..1f9ec5acebed 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 720c269b8da6..be910a6326bd 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8e9f44465cd2..40e60d9700dc 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index a1ca82981c41..c9cb7dcf52a9 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 228bef817de6..49eb6dbe2b24 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index f17279106bfa..49626b15d567 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 80c85709d67f..82197cc42286 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 3201c365c7eb..93686b08e969 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 07bb8a3fedfd..e0149f18d330 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index d2c77960ab34..ae357e508d3b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index e5c4dbf0ded8..11449f92c28c 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 9b50071e2147..33937c62d142 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index c33ce09bbf18..345bfa73fded 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 062cd3b3acc7..ccc723f4dac2 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9c99be4db232..9f34b46f98e2 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 7d1d123ba81d..28f602bc8314 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 5ab9e91e369c..9da3ebf652eb 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ba634e0b5580..e2da193b04d9 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index f41514dd6603..1d893ba7d3e4 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index dc92ae9833ce..93f1cd722e4c 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 09c469a95f67..79af769f34dd 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 4842899667bb..58bd4e5c904a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 345c15453b13..443b69f5a434 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index a9844a05f653..47c86b3a1ba4 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index d9b862a8316a..0d10ea4c729b 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index ebfd82dfe86f..02d01b2c3d3c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 3098dbba128e..c945bda7597c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index c8bad3c5fedd..33b551a1306f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index c3e87fb73c03..a01062233738 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 80d537ff1e7f..73363ed81103 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 7021f5b56d1b..175862f3fc8c 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 2169be00f880..8a36e2d5602f 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0e30a1f586fb..dab3686e6aa7 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.12 + 0.132-tw-0.13 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8a103229074e..8158dc7e5c8e 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 18c5b2961b4a..7ddccd9f434d 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.12 + 0.132-tw-0.13 presto-verifier From cfc21131975bdfe0756aa0bf3f4d42ba820e033f Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Sun, 31 Jan 2016 04:48:12 -0800 Subject: [PATCH 044/151] Revert PR#3, case insensitive types, since it is now supported. --- .../java/com/facebook/presto/metadata/FunctionRegistry.java | 2 +- .../main/java/com/facebook/presto/type/RowParametricType.java | 2 +- .../main/java/com/facebook/presto/spi/type/TypeSignature.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java index 8b30e3f8b760..4ebac76ce3af 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java @@ -508,7 +508,7 @@ private SqlFunction getRowFieldReference(String field, TypeSignature rowTypeSign checkState(rowType.getTypeSignature().getBase().equals(StandardTypes.ROW), "rowType is not a ROW type"); SqlFunction match = null; for (SqlFunction function : RowParametricType.ROW.createFunctions(rowType)) { - if (!function.getSignature().getName().equalsIgnoreCase(field)) { + if (!function.getSignature().getName().equals(field)) { continue; } checkArgument(match == null, "Ambiguous field %s in type %s", field, rowType.getDisplayName()); diff --git a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java index bc08fd1c3045..4705d16f87a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java +++ b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java @@ -68,7 +68,7 @@ public List createFunctions(Type type) RowField field = fields.get(i); int index = i; field.getName() - .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getType(), index, field.getName().get().toLowerCase()))); + .ifPresent(name -> builder.add(new RowFieldReference(rowType, field.getType(), index, field.getName().get()))); } return builder.build(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java index 75ffe4906f81..8e5d924fe399 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java @@ -189,7 +189,7 @@ private static Object parseLiteral(String literal) { if (literal.startsWith("'") || literal.endsWith("'")) { checkArgument(literal.startsWith("'") && literal.endsWith("'"), "Bad literal: '%s'", literal); - return literal.substring(1, literal.length() - 1).toLowerCase(); + return literal.substring(1, literal.length() - 1); } else { return Long.parseLong(literal); From b25b9f0ba1bba64009d3c393cf35ed2c60724fa1 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 2 Feb 2016 17:58:46 -0800 Subject: [PATCH 045/151] Update to use UGI.createProxyUser instead of createRemoteUser --- .../com/facebook/presto/hive/util/UgiUtils.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index 5c8e99e75efa..08f2dde30ff2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -15,6 +15,7 @@ import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -35,12 +36,14 @@ public static UserGroupInformation getUgi(String user) UserGroupInformation ugi = UGI_CACHE.get(user); if (ugi == null) { - // TODO: Configure hadoop to allow presto daemon user to impersonate all presto users - // (HADOOPINFRA-7081) and then change to the approach below (IQ-85). + // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html - // UserGroupInformation ugi = UserGroupInformation.createProxyUser( - // session.getUser(), UserGroupInformation.getLoginUser()); - ugi = UserGroupInformation.createRemoteUser(user); + try { + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + } + catch (IOException e) { + throw new RuntimeException("Could not get login user from UserGroupInformation", e); + } UGI_CACHE.put(user, ugi); } From ea2c8c36829f4cf7971e7f27555c0157bc126534 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 11:30:26 -0800 Subject: [PATCH 046/151] Upgrade to 0.132-tw-0.14 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2ce743f8efef..2416e615dfbe 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.13 + 0.132-tw-0.14 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2a013d29b68b..1f6bbc8b9648 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 1f9ec5acebed..114bf047c473 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index be910a6326bd..339f20cd8a98 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 40e60d9700dc..83e0d9c4fbc3 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index c9cb7dcf52a9..8dbaae2f8ae7 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 49eb6dbe2b24..7d5c36b52a5f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 49626b15d567..68ca8e164066 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 82197cc42286..766de9cc7394 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 93686b08e969..d6b9e4b865f0 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e0149f18d330..4383a60d3042 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index ae357e508d3b..2bd05749d80b 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 11449f92c28c..a795a1c8141d 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 33937c62d142..0339fdc41b7c 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 345bfa73fded..861da65b5d72 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index ccc723f4dac2..83d79d919e62 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f34b46f98e2..97675a00841a 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 28f602bc8314..af34740a897f 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 9da3ebf652eb..c5511c6b3d64 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index e2da193b04d9..9fa2fef79ff5 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1d893ba7d3e4..48948e33ab0b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 93f1cd722e4c..94fff311cc77 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 79af769f34dd..5749e3cfca24 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 58bd4e5c904a..a84aeb8f9a8c 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 443b69f5a434..c15dd15cd386 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 47c86b3a1ba4..3f982d5f5342 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 0d10ea4c729b..5630396e0e01 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 02d01b2c3d3c..4689f17b5d6c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index c945bda7597c..47ee9d810e78 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 33b551a1306f..92a6f8c8ce62 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index a01062233738..3d7badbc6e9b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 73363ed81103..c5c7ee7d4f3e 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 175862f3fc8c..4b718d149db5 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8a36e2d5602f..c5aaefea41c0 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index dab3686e6aa7..fbe89c2ee7f0 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.13 + 0.132-tw-0.14 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 8158dc7e5c8e..9f39f8e46511 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7ddccd9f434d..d1ee25a4fac7 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.13 + 0.132-tw-0.14 presto-verifier From 6ac4bc4686e2474eaf7804554e50fdae91458843 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 16:47:44 -0800 Subject: [PATCH 047/151] Whitelist zeppelin temporarily --- .../main/java/com/facebook/presto/hive/util/UgiUtils.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index 08f2dde30ff2..d5af9fb53d4c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -39,7 +39,13 @@ public static UserGroupInformation getUgi(String user) // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html try { - ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + // TODO: IQ-152 roll this back once zeppelin runsAs user + if ("zeppelin".equals(user)) { + ugi = UserGroupInformation.createRemoteUser(user); + } + else { + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + } } catch (IOException e) { throw new RuntimeException("Could not get login user from UserGroupInformation", e); From 6d857e619eab832f58ef868bf2aa9d414f9d42cf Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 3 Feb 2016 17:13:39 -0800 Subject: [PATCH 048/151] Upgrade to 0.132-tw-0.15 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 2416e615dfbe..f14af7bda23b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.14 + 0.132-tw-0.15 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1f6bbc8b9648..bf932e890e47 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 114bf047c473..b0115ab5c45a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 339f20cd8a98..787722a101b6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 83e0d9c4fbc3..afa61e58fbe7 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 8dbaae2f8ae7..b0ef31fe590b 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 7d5c36b52a5f..c07316019a33 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 68ca8e164066..410fd2ea0307 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 766de9cc7394..6816ba6073ab 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index d6b9e4b865f0..3ab195df116b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 4383a60d3042..363e6ec51409 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2bd05749d80b..84d401d9faa7 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a795a1c8141d..a7a279c4a475 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 0339fdc41b7c..5f0851c1f5d0 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 861da65b5d72..431cee305c72 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 83d79d919e62..610809a45d5e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 97675a00841a..65ef3e4a3779 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index af34740a897f..39edd5685be9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index c5511c6b3d64..2ca95314cfef 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 9fa2fef79ff5..0932dae34401 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 48948e33ab0b..4e331b4bb39a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 94fff311cc77..ff77684f5e9d 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 5749e3cfca24..f6b84dc54dc0 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a84aeb8f9a8c..9ed3bf4c0d63 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index c15dd15cd386..331e7c2d27d1 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 3f982d5f5342..95fce0f974a5 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 5630396e0e01..f3e78e9044d3 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 4689f17b5d6c..adf30dff5fb0 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 47ee9d810e78..73cfbf6524a5 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 92a6f8c8ce62..fee591fefc2b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 3d7badbc6e9b..cac03417ee1b 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c5c7ee7d4f3e..fcc758ea371f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 4b718d149db5..49fdfa2d07c0 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index c5aaefea41c0..9ef399614934 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index fbe89c2ee7f0..c2614a98b3cb 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.14 + 0.132-tw-0.15 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9f39f8e46511..12409b46a8d4 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index d1ee25a4fac7..b4c38be3a538 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.14 + 0.132-tw-0.15 presto-verifier From 207a2cca84a79e348ea43a277add1d44981e7605 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 11 Feb 2016 14:07:07 -0800 Subject: [PATCH 049/151] Rolling back Zeppelin auth hack, add some logging --- .../java/com/facebook/presto/hive/util/UgiUtils.java | 8 +------- .../java/com/facebook/presto/server/HttpRemoteTask.java | 9 +++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java index d5af9fb53d4c..08f2dde30ff2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java @@ -39,13 +39,7 @@ public static UserGroupInformation getUgi(String user) // Configure hadoop to allow presto daemon user to impersonate all presto users // See https://hadoop.apache.org/docs/r2.4.1/hadoop-project-dist/hadoop-common/Superusers.html try { - // TODO: IQ-152 roll this back once zeppelin runsAs user - if ("zeppelin".equals(user)) { - ugi = UserGroupInformation.createRemoteUser(user); - } - else { - ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); - } + ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); } catch (IOException e) { throw new RuntimeException("Could not get login user from UserGroupInformation", e); diff --git a/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java index c21451070dd1..529b337add8b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/HttpRemoteTask.java @@ -53,6 +53,7 @@ import io.airlift.http.client.HttpClient; import io.airlift.http.client.HttpStatus; import io.airlift.http.client.Request; +import io.airlift.http.client.StaticBodyGenerator; import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; @@ -454,6 +455,14 @@ private synchronized void scheduleUpdate() updateErrorTracker.startRequest(); + // TODO: (billg) remove this logging or contribute it upstream + if (log.isDebugEnabled()) { + String size = "unknown"; + if (request.getBodyGenerator() instanceof StaticBodyGenerator) { + size = Integer.toString(((StaticBodyGenerator) request.getBodyGenerator()).getBody().length); + } + log.debug(String.format("scheduleUpdate POST %s, bodySize=%s sourcesSize=%s", request.getUri(), size, sources.size())); + } ListenableFuture> future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec)); currentRequest = future; currentRequestStartNanos = System.nanoTime(); From daa37735883f0d2b2220a77664f2919bf7dd4cd4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 11 Feb 2016 15:40:29 -0800 Subject: [PATCH 050/151] Upgrading to 0.132-tw-0.16 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f14af7bda23b..f22df98fd73e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.15 + 0.132-tw-0.16 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index bf932e890e47..632f2069eff1 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index b0115ab5c45a..cc0eee6d5965 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 787722a101b6..05bf2895362a 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index afa61e58fbe7..0a271e785de8 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index b0ef31fe590b..6425232da79b 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index c07316019a33..8aa9693ec1da 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 410fd2ea0307..121e534b2954 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 6816ba6073ab..ece17fc3749b 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 3ab195df116b..f25c91f065bd 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 363e6ec51409..6c0df00ff71c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 84d401d9faa7..4068bf2c3f06 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a7a279c4a475..455d8aa2580c 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 5f0851c1f5d0..ac7c172f3d98 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 431cee305c72..fd48780e8e96 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 610809a45d5e..f9c8aeeb7330 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 65ef3e4a3779..47bd019010f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 39edd5685be9..c63e02329855 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2ca95314cfef..f67ddcb8f3c9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 0932dae34401..eb67f46ce25a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 4e331b4bb39a..09846fc4988d 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index ff77684f5e9d..f8fc748cb600 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index f6b84dc54dc0..47c4e2b73fe3 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 9ed3bf4c0d63..2bcb0bc67c03 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 331e7c2d27d1..4ca861f83971 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 95fce0f974a5..45afe82f6e0c 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index f3e78e9044d3..af428336232c 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index adf30dff5fb0..371a08a68b28 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 73cfbf6524a5..2d440900ff36 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index fee591fefc2b..4c77ae0aaeed 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index cac03417ee1b..09f7c9e30b0f 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index fcc758ea371f..06b939aef70d 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 49fdfa2d07c0..1d365f90c836 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 9ef399614934..50270820bc3c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index c2614a98b3cb..d82075da681d 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.15 + 0.132-tw-0.16 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 12409b46a8d4..846d8f330385 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index b4c38be3a538..ff14bb0ff9ee 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.15 + 0.132-tw-0.16 presto-verifier From f4fed25634728d7a03010e5073b2614b213a8a36 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 09:38:56 -0800 Subject: [PATCH 051/151] Change version to 0.139 to not conflict with merge --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f22df98fd73e..afb115e56e7b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.132-tw-0.16 + 0.139 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 632f2069eff1..721e5197c68d 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index cc0eee6d5965..bee89f2aca0c 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 05bf2895362a..e6b9dfe148ef 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 0a271e785de8..24d70e0bd523 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 6425232da79b..3cbadb8cb50c 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 8aa9693ec1da..eee77f26b681 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 121e534b2954..053443888316 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index ece17fc3749b..7e609c58d169 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f25c91f065bd..bbdb2780c86f 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 6c0df00ff71c..74cc530dc2d6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 4068bf2c3f06..3909da01a3e5 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 455d8aa2580c..3617329133c4 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index ac7c172f3d98..b99adacef6d4 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index fd48780e8e96..5ebdcab6d24a 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f9c8aeeb7330..db7651abb449 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 47bd019010f6..77e51c1592c6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index c63e02329855..459740752199 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index f67ddcb8f3c9..cd5b16486e33 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index eb67f46ce25a..881acee972aa 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 09846fc4988d..2691474bab1b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f8fc748cb600..f7f94917820f 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 47c4e2b73fe3..e52248e66964 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 2bcb0bc67c03..bf2af5acd118 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 4ca861f83971..54d95b03d34b 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 45afe82f6e0c..4c01f1eb9160 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index af428336232c..4248921cc55a 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 371a08a68b28..7b6d3f420fa9 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 2d440900ff36..7b545ce1b177 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 4c77ae0aaeed..2a8ef6926ffc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 09f7c9e30b0f..7e3151746a48 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 06b939aef70d..6460358c5dba 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 1d365f90c836..0ab884da5c84 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 50270820bc3c..cc0f53d979cb 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index d82075da681d..6d2ca1720a56 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.132-tw-0.16 + 0.139 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 846d8f330385..db45c2072959 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index ff14bb0ff9ee..adba2eee4163 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.132-tw-0.16 + 0.139 presto-verifier From 2c9da6093b6e72f8634b7c874b517dc6e729ad31 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 10:12:18 -0800 Subject: [PATCH 052/151] upgrade pom version to 0.139-tw-0.17 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 67358e743bdc..632225d50fe1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139 + 0.139-tw-0.17 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 721e5197c68d..d84fe856c920 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index bee89f2aca0c..80781a476f81 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index e6b9dfe148ef..073888296318 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 24d70e0bd523..8117fa6549ae 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 3cbadb8cb50c..5493f10852bd 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index eee77f26b681..4339db66dc64 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 053443888316..dd2800d32a73 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4e7fc011a28b..907443ee18ec 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index bbdb2780c86f..7df4d2893179 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 74cc530dc2d6..85b2c4f1524d 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 3909da01a3e5..1c254fb609c8 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 3617329133c4..0e58e0b8a334 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index b99adacef6d4..f75f642ef484 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 5ebdcab6d24a..22e73c96b1d0 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index db7651abb449..cc32b3878947 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 77e51c1592c6..9f748e0114ee 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 803afb3bbbd0..8be6aa536b7c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index cd5b16486e33..ef29485ce5c9 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 73bdb896bc4d..749ca56c4191 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 2691474bab1b..34b47c0df123 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f7f94917820f..7754bb50b25c 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index e52248e66964..422be0c545d9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index bf2af5acd118..48218aa78fb5 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 54d95b03d34b..5891c6b4c875 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 4c01f1eb9160..6c78d18b97ca 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e1959313da8a..8504a4df3bc8 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 7b6d3f420fa9..0719269862f1 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 7b545ce1b177..e15a8e9be8c5 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 2a8ef6926ffc..23bd57d1aef9 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 7e3151746a48..eb130f83e50d 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 6460358c5dba..f23575bcf202 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 0ab884da5c84..85bc734c1bf3 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index cc0f53d979cb..97a45f3fce0f 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 6d2ca1720a56..75edb19713ae 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139 + 0.139-tw-0.17 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index db45c2072959..730294962ce5 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index ad246ed83646..507e3ebe0c51 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139 + 0.139-tw-0.17 presto-verifier From 582df38cca94dbe0f447673c34973badc96397e6 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 24 Feb 2016 14:48:28 -0800 Subject: [PATCH 053/151] Adding query logging to presto queries --- .../java/com/facebook/presto/execution/SqlQueryManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index e004f2024012..52a0751cc3d1 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -306,6 +306,10 @@ public QueryInfo createQuery(Session session, String query) stats.queryFinished(info); queryMonitor.completionEvent(info); expirationQueue.add(queryExecution); + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + info.getQueryId(), newValue, info.getErrorType(), info.getErrorCode(), + session.getUser(), info.getQueryStats().getElapsedTime(), + info.getQuery().replace(System.getProperty("line.separator"), " "))); } }); From 04e60cce32c2a06bf77818fb6b522c0ab0864eea Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 29 Feb 2016 14:38:01 -0800 Subject: [PATCH 054/151] Add a few logs in DiscoveryNodeManager to debug 'No worker nodes' error. --- .../facebook/presto/metadata/DiscoveryNodeManager.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index f6d470e88352..7238f4f3f94b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -26,6 +26,7 @@ import io.airlift.discovery.client.ServiceSelector; import io.airlift.discovery.client.ServiceType; import io.airlift.http.client.HttpClient; +import io.airlift.log.Logger; import io.airlift.node.NodeInfo; import io.airlift.units.Duration; @@ -60,6 +61,8 @@ public final class DiscoveryNodeManager implements InternalNodeManager { + private static final Logger log = Logger.get(DiscoveryNodeManager.class); + private static final Duration MAX_AGE = new Duration(5, TimeUnit.SECONDS); private static final Splitter DATASOURCES_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); @@ -124,12 +127,18 @@ public void startPollingNodeStates() Set deadNodes = difference(nodeStates.keySet(), aliveNodeIds).immutableCopy(); nodeStates.keySet().removeAll(deadNodes); + if (deadNodes.size() > 0) { + log.warn("Dead nodes: %s", deadNodes); + } + // Add new nodes for (Node node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } + log.debug("Number of alive nodes: %d", nodeStates.size()); + // Schedule refresh nodeStates.values().forEach(RemoteNodeState::asyncRefresh); }, 1, 5, TimeUnit.SECONDS); From 4ca01a760e2610e19562eea4c4e2e5ff749283c9 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Mon, 29 Feb 2016 15:27:05 -0800 Subject: [PATCH 055/151] Upgrade to 0.139-tw-0.18 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 632225d50fe1..9f3597db7a77 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.17 + 0.139-tw-0.18 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d84fe856c920..1b0a10a8cbd6 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 80781a476f81..1629008c4a62 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 073888296318..671b065d02f8 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8117fa6549ae..fa8e9ef3fc0e 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 5493f10852bd..031b25c708c4 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 4339db66dc64..b9c5632037af 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index dd2800d32a73..a8b37a95d835 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 907443ee18ec..23b87a6a0851 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 7df4d2893179..0033399e5eb7 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 85b2c4f1524d..9a212eb99757 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 1c254fb609c8..fafbe305fc71 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 0e58e0b8a334..f2ef547e83e6 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index f75f642ef484..9b784c66bf4b 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 22e73c96b1d0..a984ac81da6e 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index cc32b3878947..02f321b1d5e3 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f748e0114ee..9f1258948e5d 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 8be6aa536b7c..09d2e4ec341c 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index ef29485ce5c9..cc0875536dae 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 749ca56c4191..498eaaabdfc8 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 34b47c0df123..a53264ad7f8b 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 7754bb50b25c..9b547152999b 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 422be0c545d9..7ded555176ad 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 48218aa78fb5..518f2dd91b55 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 5891c6b4c875..aed0b1c31591 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6c78d18b97ca..6b1a9d92df35 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 8504a4df3bc8..e96cab0b52f0 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 0719269862f1..88ea917ca2d8 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e15a8e9be8c5..e38908baa260 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 23bd57d1aef9..94886cb04ccc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index eb130f83e50d..84c31bc33f0f 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index f23575bcf202..edfa5063e36a 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 85bc734c1bf3..4c737d4e9be3 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 97a45f3fce0f..83afc4fe5eaa 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 75edb19713ae..aac5464df7de 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.17 + 0.139-tw-0.18 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 730294962ce5..539fb12adb9a 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 507e3ebe0c51..d81371230049 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.17 + 0.139-tw-0.18 presto-verifier From f1085e08d059e10ec18706e1a0927e31bce34edc Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 12:16:25 -0800 Subject: [PATCH 056/151] Look up parquet columns by name case-insensitive --- .../facebook/presto/hive/parquet/ParquetTypeUtils.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index bcf9027daeba..0fce1d81eced 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.HiveColumnHandle; import parquet.schema.MessageType; +import parquet.schema.Type; public final class ParquetTypeUtils { @@ -28,6 +29,13 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag if (messageType.containsField(column.getName())) { return messageType.getType(column.getName()); } + // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase + // check for direct match above but if no match found, try case-insensitive match + for (Type type : messageType.getFields()) { + if (type.getName().equalsIgnoreCase(column.getName())) { + return type; + } + } return null; } From 5b4647a1bb4e8638d38e9e376469bec5ae26ee27 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 16:47:02 -0800 Subject: [PATCH 057/151] Upgrading to 0.139-tw-0.19 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 9f3597db7a77..f9431fab7e4c 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.18 + 0.139-tw-0.19 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1b0a10a8cbd6..48bdb7c717c6 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 1629008c4a62..9b83d6abcd3d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 671b065d02f8..55598291a33c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index fa8e9ef3fc0e..9491d2c60eaa 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 031b25c708c4..47d7c6fef678 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index b9c5632037af..c4fd099e44d0 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index a8b37a95d835..9ae21a132d6f 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 23b87a6a0851..64003da568a8 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0033399e5eb7..508ba4cca260 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 9a212eb99757..5bb50b9fedf9 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index fafbe305fc71..85a7d855d03d 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f2ef547e83e6..c796d9bc36d5 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 9b784c66bf4b..8784ec38bc75 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index a984ac81da6e..83cf942d5b8d 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 02f321b1d5e3..19c29f7401b8 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 9f1258948e5d..4d3016e99c94 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 09d2e4ec341c..a85718e4708a 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index cc0875536dae..1fe37919474c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 498eaaabdfc8..b04fed59a0ff 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index a53264ad7f8b..5359608986e9 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 9b547152999b..30a708b8d147 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 7ded555176ad..74512ff0e849 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 518f2dd91b55..0d57737392af 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index aed0b1c31591..6384f163283e 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6b1a9d92df35..693d2bb9c135 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e96cab0b52f0..2952839013c2 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 88ea917ca2d8..53c00bb94f9a 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index e38908baa260..490cd8218967 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 94886cb04ccc..da1094e39d4b 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 84c31bc33f0f..de827fc9fd70 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index edfa5063e36a..33c89198a37c 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 4c737d4e9be3..c627552ef199 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 83afc4fe5eaa..083bb3564d3c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index aac5464df7de..0d21f45e190e 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.18 + 0.139-tw-0.19 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 539fb12adb9a..b10f3a011cab 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index d81371230049..1a30c0800d0d 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.18 + 0.139-tw-0.19 presto-verifier From f766a0c616d437eca61bb9b4f1d9e7712373341c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 18:07:59 -0800 Subject: [PATCH 058/151] Handle hive keywords when doing a name-based parquet field lookup --- .../presto/hive/parquet/ParquetTypeUtils.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index 0fce1d81eced..8891fb88afcd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -26,17 +26,15 @@ private ParquetTypeUtils() public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { - if (messageType.containsField(column.getName())) { - return messageType.getType(column.getName()); - } - // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase - // check for direct match above but if no match found, try case-insensitive match - for (Type type : messageType.getFields()) { - if (type.getName().equalsIgnoreCase(column.getName())) { - return type; - } + String name = column.getName(); + Type type = getParquetTypeByName(name, messageType); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); } - return null; + return type; } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { @@ -44,4 +42,20 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag } return null; } + + private static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) + { + if (messageType.containsField(columnName)) { + return messageType.getType(columnName); + } + // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase + // check for direct match above but if no match found, try case-insensitive match + for (Type type : messageType.getFields()) { + if (type.getName().equalsIgnoreCase(columnName)) { + return type; + } + } + + return null; + } } From ca9961e8deeca4f1640673511870fbb08d908b72 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 1 Mar 2016 20:40:26 -0800 Subject: [PATCH 059/151] Refactor to make diff cleaner --- .../presto/hive/parquet/ParquetTypeUtils.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index 8891fb88afcd..bdd315f17218 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -26,15 +26,7 @@ private ParquetTypeUtils() public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { - String name = column.getName(); - Type type = getParquetTypeByName(name, messageType); - - // when a parquet field is a hive keyword we append an _ to it in hive. When doing - // a name-based lookup, we need to strip it off again if we didn't get a direct match. - if (type == null && name.endsWith("_")) { - type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); - } - return type; + return findParquetTypeByName(column, messageType); } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { @@ -43,6 +35,28 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag return null; } + /** + * Find the column type by name using returning the first match with the following logic: + *
    + *
  • direct match
    • + *
  • case-insensitive match
    • + *
  • if the name ends with _, remove it and direct match
    • + *
  • if the name ends with _, remove it and case-insensitive match
    • + *
+ */ + private static parquet.schema.Type findParquetTypeByName(HiveColumnHandle column, MessageType messageType) + { + String name = column.getName(); + Type type = getParquetTypeByName(name, messageType); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); + } + return type; + } + private static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) { if (messageType.containsField(columnName)) { From d7d66a28954049e9d41baf0d7eefd0bfd245637e Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 11 Mar 2016 11:25:21 -0800 Subject: [PATCH 060/151] Upgrade to 0.141. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f9431fab7e4c..1cc3360c33ef 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.139-tw-0.19 + 0.141 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 48bdb7c717c6..aefbe525d11c 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 9b83d6abcd3d..883a041146d2 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 55598291a33c..546f6323162c 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 9491d2c60eaa..7637c11bcb13 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 47d7c6fef678..f24bb402f6f5 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index c4fd099e44d0..db5650329053 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 9ae21a132d6f..755f4b0c0bd0 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 64003da568a8..1bff918f4e79 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 508ba4cca260..ad3c2de908f1 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 5bb50b9fedf9..5bd58c4889a2 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 85a7d855d03d..360175ba7f1f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index c796d9bc36d5..fc5b3c6bb0cc 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 8784ec38bc75..2e2d0c2e6938 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 83cf942d5b8d..f9cedd62b887 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 19c29f7401b8..77c66becb7c1 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 4d3016e99c94..80c89914cb60 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index a85718e4708a..8271359203ce 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 1fe37919474c..057819d9734b 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b04fed59a0ff..ecbbd667febc 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 5359608986e9..1aa7767621ae 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 30a708b8d147..f02eced794de 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 74512ff0e849..906c40b97ba9 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 0d57737392af..b30f2e739442 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 6384f163283e..8f130d06d809 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 693d2bb9c135..ec7b54712bed 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 2952839013c2..17601b1b1823 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 53c00bb94f9a..f6f597487812 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 490cd8218967..31eb7a942001 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index da1094e39d4b..eb44ebac42f7 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index de827fc9fd70..67b64fdc2156 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 33c89198a37c..5b61cec72b21 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index c627552ef199..e9ab9fba2bba 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 083bb3564d3c..05bdf43a0a08 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0d21f45e190e..696256a3122b 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.139-tw-0.19 + 0.141 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index b10f3a011cab..72aaa2acb3dd 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 1a30c0800d0d..5f6c00cf18f5 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.139-tw-0.19 + 0.141 presto-verifier From 94bcf3e46c886684e45a5a796d0ae9167e5ad6f8 Mon Sep 17 00:00:00 2001 From: Sailesh Mittal Date: Fri, 11 Mar 2016 11:29:59 -0800 Subject: [PATCH 061/151] Upgrade to 0.141-tw-0.20. --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 411996e53ee0..130f0ff9eb9d 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.141 + 0.141-tw-0.20 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index def710c5c2c6..00cc629d364c 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 883a041146d2..3652e17bd06f 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 546f6323162c..8f86dabd3e25 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 7637c11bcb13..1c0d41086973 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index f24bb402f6f5..1bdeaecdcea2 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index db5650329053..4f0675eda340 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 755f4b0c0bd0..bdcff6cdb591 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 1bff918f4e79..f54c94db8398 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index ad3c2de908f1..4b4ad310d3e1 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 5bd58c4889a2..2c4cc361f6fb 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 360175ba7f1f..7bb7379f70b6 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index fc5b3c6bb0cc..09e13a19e68d 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 2e2d0c2e6938..6e304d8b10bd 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index f9cedd62b887..cfa615c8d2dc 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 77c66becb7c1..cebd8614c068 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 80c89914cb60..24ca263be84e 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 8271359203ce..4ae769912294 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 057819d9734b..31620ae9fe5a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index ecbbd667febc..b1eed0066f04 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1aa7767621ae..b5e67298855a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index f02eced794de..4722581cd3a5 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 906c40b97ba9..d710b58886f5 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index b30f2e739442..830af643de31 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 8f130d06d809..c8e057ed14aa 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index ec7b54712bed..bfe2383a567f 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 17601b1b1823..3b527cbaa5dc 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index f6f597487812..5c724e169c0c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 31eb7a942001..4c226d7edc39 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index eb44ebac42f7..14038c648f0f 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 67b64fdc2156..2c4b2c1f49eb 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 5b61cec72b21..a977f2d08420 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index e9ab9fba2bba..301e4c4efe55 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 05bdf43a0a08..dc8b11426992 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 696256a3122b..25c2414ce83f 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141 + 0.141-tw-0.20 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 72aaa2acb3dd..3979e5a18555 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 5f6c00cf18f5..9dbeee8b8252 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141 + 0.141-tw-0.20 presto-verifier From 3de1ca483acb8944e714631df79fe62a3023f450 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 09:35:41 -0700 Subject: [PATCH 062/151] Use modules and query events for logging --- .../presto/execution/SqlQueryManager.java | 4 - .../facebook/presto/server/PrestoServer.java | 2 +- .../facebook/presto/twitter/ModuleLoader.java | 37 ++++++++ .../presto/twitter/logging/QueryLogger.java | 91 +++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index 51e261aeeb10..6722c701fd5e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -309,10 +309,6 @@ public QueryInfo createQuery(Session session, String query) stats.queryFinished(info); queryMonitor.completionEvent(info); expirationQueue.add(queryExecution); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - info.getQueryId(), newValue, info.getErrorType(), info.getErrorCode(), - session.getUser(), info.getQueryStats().getElapsedTime(), - info.getQuery().replace(System.getProperty("line.separator"), " "))); } }); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index de138dc25cba..685a0454c73a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -154,7 +154,7 @@ public void run() protected Iterable getAdditionalModules() { - return ImmutableList.of(); + return com.facebook.presto.twitter.ModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java new file mode 100644 index 000000000000..aee0a6ca2825 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter; + +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; +import io.airlift.event.client.EventClient; + +public class ModuleLoader +{ + private ModuleLoader() + { + } + + public static Iterable getAdditionalModules() + { + return ImmutableList.of(binder -> + Multibinder.newSetBinder(binder, EventClient.class) + .addBinding() + .to(QueryLogger.class) + .in(Scopes.SINGLETON)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java new file mode 100644 index 000000000000..863847ab73c0 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.spi.StandardErrorCode; +import io.airlift.event.client.AbstractEventClient; +import io.airlift.event.client.EventType; +import io.airlift.log.Logger; +import io.airlift.units.Duration; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public class QueryLogger extends AbstractEventClient +{ + private static final int MAX_QUERY_LENGTH = 1000; + private static final String DASH = "-"; + private static final String COLON = ":"; + private static final String SPACE = " "; + private static final String ELIPSIS = "..."; + + private static final Logger log = Logger.get(QueryLogger.class); + + @Override + protected void postEvent(T event) + throws IOException + { + EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); + if (eventTypeAnnotation == null) { + return; + } + + // other event types exist, like QueryCreatedEvent and SplitCompletionEvent + if (eventTypeAnnotation.value().equals("QueryCompletion")) { + logQueryComplete((QueryCompletionEvent) event); + } + } + + private static void logQueryComplete(QueryCompletionEvent event) + { + String errorType = DASH; + String errorCode = DASH; + if (event.getErrorCode() != null) { + errorType = StandardErrorCode.toErrorType(event.getErrorCode()).toString(); + if (event.getErrorCodeName() != null) { + errorCode = event.getErrorCodeName() + COLON + event.getErrorCode(); + } + } + + Duration duration = (new Duration( + event.getEndTime().getMillis() - + event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) + .convertToMostSuccinctTimeUnit(); + + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + event.getQueryId(), toLogValue(event.getRemoteClientAddress()), + event.getQueryState(), errorType, errorCode, + event.getUser(), duration, + cleanseAndTrimQuery(event.getQuery()))); + } + + private static String toLogValue(Object object) + { + if (object == null) { + return DASH; + } + else { + return object.toString(); + } + } + + private static String cleanseAndTrimQuery(String query) + { + if (query.length() > MAX_QUERY_LENGTH) { + query = query.substring(0, MAX_QUERY_LENGTH) + ELIPSIS; + } + return query.replace(System.getProperty("line.separator"), SPACE); + } +} From f055db69eaf6901002e0d017e85223cc5cdede80 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:25:55 -0700 Subject: [PATCH 063/151] Add javadocs --- .../facebook/presto/server/PrestoServer.java | 3 +- .../facebook/presto/twitter/ModuleLoader.java | 37 ------------------- .../presto/twitter/logging/QueryLogger.java | 3 ++ 3 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 685a0454c73a..78b01e8cd4fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -24,6 +24,7 @@ import com.facebook.presto.security.AccessControlModule; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.twitter.TwitterModuleLoader; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -154,7 +155,7 @@ public void run() protected Iterable getAdditionalModules() { - return com.facebook.presto.twitter.ModuleLoader.getAdditionalModules(); + return TwitterModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java deleted file mode 100644 index aee0a6ca2825..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/ModuleLoader.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter; - -import com.facebook.presto.twitter.logging.QueryLogger; -import com.google.common.collect.ImmutableList; -import com.google.inject.Module; -import com.google.inject.Scopes; -import com.google.inject.multibindings.Multibinder; -import io.airlift.event.client.EventClient; - -public class ModuleLoader -{ - private ModuleLoader() - { - } - - public static Iterable getAdditionalModules() - { - return ImmutableList.of(binder -> - Multibinder.newSetBinder(binder, EventClient.class) - .addBinding() - .to(QueryLogger.class) - .in(Scopes.SINGLETON)); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 863847ab73c0..b70eda37da7c 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; +/** + * Class that listens for query completion events and logs them to a file + */ public class QueryLogger extends AbstractEventClient { private static final int MAX_QUERY_LENGTH = 1000; From 3c1c7b661cf0b79abeb33448eafd9b0fe2c421b0 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:45:11 -0700 Subject: [PATCH 064/151] Add javadocs --- .../presto/twitter/TwitterModuleLoader.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java new file mode 100644 index 000000000000..44bfc995b462 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter; + +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; +import io.airlift.event.client.EventClient; + +/** + * Loader that initializes custom Twitter code to inject into Presto. Whenever + * possible we should use this pattern to inject custom functionality, since it + * makes it easier to differentiate our patches from the core OS code. + * + * If the functionality we wish to add/override isn't currently possible to via + * overriding a guice module, we should contribute the necessary modules/interfaces + * into the OS Presto code base to make it possible. + */ +public class TwitterModuleLoader +{ + private TwitterModuleLoader() + { + } + + public static Iterable getAdditionalModules() + { + return ImmutableList.of(binder -> + Multibinder.newSetBinder(binder, EventClient.class) + .addBinding() + .to(QueryLogger.class) + .in(Scopes.SINGLETON)); + } +} From d8afd6223cc9d99d3cfd430b0752b53c0cb66c75 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 10:47:05 -0700 Subject: [PATCH 065/151] fix imports --- .../src/main/java/com/facebook/presto/server/PrestoServer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 78b01e8cd4fb..4bac276668ef 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -24,7 +24,6 @@ import com.facebook.presto.security.AccessControlModule; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.parser.SqlParserOptions; -import com.facebook.presto.twitter.TwitterModuleLoader; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -155,7 +154,7 @@ public void run() protected Iterable getAdditionalModules() { - return TwitterModuleLoader.getAdditionalModules(); + return com.facebook.presto.twitter.TwitterModuleLoader.getAdditionalModules(); } private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) From 251db4d7c15df61b8e86eb1bd3e21ad904ecac04 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 11:01:00 -0700 Subject: [PATCH 066/151] add splits, rows and bytes --- .../com/facebook/presto/twitter/logging/QueryLogger.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index b70eda37da7c..1ad52efb0d67 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -67,10 +67,10 @@ private static void logQueryComplete(QueryCompletionEvent event) event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", event.getQueryId(), toLogValue(event.getRemoteClientAddress()), - event.getQueryState(), errorType, errorCode, - event.getUser(), duration, + event.getQueryState(), errorType, errorCode, event.getUser(), duration, + event.getSplits(), event.getTotalRows(), event.getTotalBytes(), cleanseAndTrimQuery(event.getQuery()))); } From 95035ecf3eb7a1a9569fa9a13a15b00463992349 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 24 Mar 2016 11:56:17 -0700 Subject: [PATCH 067/151] Change to use QueryComplete --- .../com/facebook/presto/twitter/logging/QueryLogger.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 1ad52efb0d67..0a6f5c3e105c 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -33,6 +33,7 @@ public class QueryLogger extends AbstractEventClient private static final String COLON = ":"; private static final String SPACE = " "; private static final String ELIPSIS = "..."; + private static final String QUERY_COMPLETION = "QueryCompletion"; private static final Logger log = Logger.get(QueryLogger.class); @@ -46,7 +47,7 @@ protected void postEvent(T event) } // other event types exist, like QueryCreatedEvent and SplitCompletionEvent - if (eventTypeAnnotation.value().equals("QueryCompletion")) { + if (eventTypeAnnotation.value().equals(QUERY_COMPLETION)) { logQueryComplete((QueryCompletionEvent) event); } } @@ -67,8 +68,8 @@ private static void logQueryComplete(QueryCompletionEvent event) event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); - log.info(String.format("Query complete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - event.getQueryId(), toLogValue(event.getRemoteClientAddress()), + log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + QUERY_COMPLETION, event.getQueryId(), toLogValue(event.getRemoteClientAddress()), event.getQueryState(), errorType, errorCode, event.getUser(), duration, event.getSplits(), event.getTotalRows(), event.getTotalBytes(), cleanseAndTrimQuery(event.getQuery()))); From dbb4c42e330e42815a097da727f26cad68d55141 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 12:29:07 -0700 Subject: [PATCH 068/151] Add event process and handlers and scribe query completion events --- pom.xml | 55 ++++++ presto-main/pom.xml | 18 ++ .../facebook/presto/event/EventProcessor.java | 90 ++++++++++ .../event/query/QueryCompletionEvent.java | 2 +- .../presto/event/query/QueryCreatedEvent.java | 2 +- .../presto/event/query/QueryEvent.java | 22 +++ .../presto/event/query/QueryEventHandler.java | 19 ++ .../event/query/SplitCompletionEvent.java | 2 +- .../presto/twitter/TwitterModuleLoader.java | 26 ++- .../presto/twitter/logging/QueryLogger.java | 27 +-- .../presto/twitter/logging/QueryScriber.java | 169 ++++++++++++++++++ 11 files changed, 403 insertions(+), 29 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java create mode 100644 presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java create mode 100644 presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java diff --git a/pom.xml b/pom.xml index 130f0ff9eb9d..ae36b8cf3d9c 100644 --- a/pom.xml +++ b/pom.xml @@ -730,6 +730,61 @@ hive-apache-jdbc 0.13.1-1 + + + + com.twitter + presto-thrift-java + 0.0.1 + + + com.twitter + util-core_2.11 + + + com.twitter + util-core-java + + + com.twitter + util-function_2.10 + + + com.twitter + util-function-java + + + commons-logging + commons-logging + + + org.scala-lang.modules + scala-parser-combinators_2.11 + + + + + com.twitter + util-logging_2.11 + 6.33.0 + + + commons-logging + commons-logging + + + + + org.scala-lang + scala-library + 2.11.7 + + + commons-logging + commons-logging + + + diff --git a/presto-main/pom.xml b/presto-main/pom.xml index b1eed0066f04..dfe5c5d08d7a 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -296,6 +296,24 @@ tpch test + + + + com.twitter + presto-thrift-java + + + com.twitter + util-logging_2.11 + + + org.apache.thrift + libthrift + + + org.scala-lang + scala-library + diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java new file mode 100644 index 000000000000..edaabc5437de --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryCreatedEvent; +import com.facebook.presto.event.query.QueryEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.event.query.SplitCompletionEvent; +import com.facebook.presto.twitter.logging.QueryLogger; +import com.google.inject.Inject; +import io.airlift.event.client.AbstractEventClient; +import io.airlift.event.client.EventType; +import io.airlift.log.Logger; + +import java.io.IOException; +import java.util.Set; + +public class EventProcessor extends AbstractEventClient +{ + private static final String QUERY_CREATED = "QueryCreated"; + private static final String QUERY_COMPLETION = "QueryCompletion"; + private static final String SPLIT_COMPLETION = "SplitCompletion"; + private static final Logger log = Logger.get(QueryLogger.class); + + private Set> queryCreatedEventHandlers; + private Set> queryCompletionEventHandlers; + private Set> splitCompletionEventHandlers; + + @Inject + public EventProcessor( + Set> queryCreatedEventHandlers, + Set> queryCompletionEventHandlers, + Set> splitCompletionEventHandlers) + { + this.queryCreatedEventHandlers = queryCreatedEventHandlers; + this.queryCompletionEventHandlers = queryCompletionEventHandlers; + this.splitCompletionEventHandlers = splitCompletionEventHandlers; + } + + @Override + protected void postEvent(T event) + throws IOException + { + EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); + if (eventTypeAnnotation == null) { + return; + } + + String type = eventTypeAnnotation.value(); + + switch (type) { + case QUERY_CREATED: + handle(queryCreatedEventHandlers, type, (QueryCreatedEvent) event); + break; + case QUERY_COMPLETION: + handle(queryCompletionEventHandlers, type, (QueryCompletionEvent) event); + break; + case SPLIT_COMPLETION: + handle(splitCompletionEventHandlers, type, (SplitCompletionEvent) event); + break; + default: + log.warn("Unrecognized event found: " + type); + } + } + + private void handle(Set> handlers, String type, E event) + { + for (QueryEventHandler handler : handlers) { + try { + handler.handle(event); + } + catch (Throwable e) { + log.error(e, String.format( + "Exception processing %s event for query %s", type, event.getQueryId())); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java index c005649e77e2..3c95ffe66ef0 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCompletionEvent.java @@ -31,7 +31,7 @@ @Immutable @EventType("QueryCompletion") -public class QueryCompletionEvent +public class QueryCompletionEvent implements QueryEvent { private final QueryId queryId; private final String transactionId; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java index 64871433ecee..6be296bba8e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryCreatedEvent.java @@ -24,7 +24,7 @@ @Immutable @EventType("QueryCreated") -public class QueryCreatedEvent +public class QueryCreatedEvent implements QueryEvent { private final QueryId queryId; private final String transactionId; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java new file mode 100644 index 000000000000..33ca974b826c --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event.query; + +/** + * + */ +public interface QueryEvent +{ + String getQueryId(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java new file mode 100644 index 000000000000..2aa0bd381a5f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEventHandler.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.event.query; + +public interface QueryEventHandler +{ + void handle(T event); +} diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java index f9374c21b09d..4a9e206b9962 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/SplitCompletionEvent.java @@ -29,7 +29,7 @@ @Immutable @EventType("SplitCompletion") -public class SplitCompletionEvent +public class SplitCompletionEvent implements QueryEvent { private final QueryId queryId; private final StageId stageId; diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java index 44bfc995b462..eb23a7650a09 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java @@ -13,10 +13,17 @@ */ package com.facebook.presto.twitter; +import com.facebook.presto.event.EventProcessor; +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryCreatedEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.event.query.SplitCompletionEvent; import com.facebook.presto.twitter.logging.QueryLogger; +import com.facebook.presto.twitter.logging.QueryScriber; import com.google.common.collect.ImmutableList; import com.google.inject.Module; import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; import io.airlift.event.client.EventClient; @@ -37,10 +44,21 @@ private TwitterModuleLoader() public static Iterable getAdditionalModules() { - return ImmutableList.of(binder -> - Multibinder.newSetBinder(binder, EventClient.class) + return ImmutableList.of( + binder -> Multibinder.newSetBinder(binder, EventClient.class) .addBinding() - .to(QueryLogger.class) - .in(Scopes.SINGLETON)); + .to(EventProcessor.class) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(new TypeLiteral(){}) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(QueryScriber.class) + .in(Scopes.SINGLETON) + ); } } diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java index 0a6f5c3e105c..02edd8da2440 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -14,19 +14,17 @@ package com.facebook.presto.twitter.logging; import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; import com.facebook.presto.spi.StandardErrorCode; -import io.airlift.event.client.AbstractEventClient; -import io.airlift.event.client.EventType; import io.airlift.log.Logger; import io.airlift.units.Duration; -import java.io.IOException; import java.util.concurrent.TimeUnit; /** - * Class that listens for query completion events and logs them to a file + * Class that logs query events to a file */ -public class QueryLogger extends AbstractEventClient +public class QueryLogger implements QueryEventHandler { private static final int MAX_QUERY_LENGTH = 1000; private static final String DASH = "-"; @@ -38,21 +36,7 @@ public class QueryLogger extends AbstractEventClient private static final Logger log = Logger.get(QueryLogger.class); @Override - protected void postEvent(T event) - throws IOException - { - EventType eventTypeAnnotation = event.getClass().getAnnotation(EventType.class); - if (eventTypeAnnotation == null) { - return; - } - - // other event types exist, like QueryCreatedEvent and SplitCompletionEvent - if (eventTypeAnnotation.value().equals(QUERY_COMPLETION)) { - logQueryComplete((QueryCompletionEvent) event); - } - } - - private static void logQueryComplete(QueryCompletionEvent event) + public void handle(QueryCompletionEvent event) { String errorType = DASH; String errorCode = DASH; @@ -64,8 +48,7 @@ private static void logQueryComplete(QueryCompletionEvent event) } Duration duration = (new Duration( - event.getEndTime().getMillis() - - event.getCreateTime().getMillis(), TimeUnit.MILLISECONDS)) + event.getQueryWallTimeMs(), TimeUnit.MILLISECONDS)) .convertToMostSuccinctTimeUnit(); log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java new file mode 100644 index 000000000000..0110afdf8a61 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.google.common.base.Optional; +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; +import com.twitter.presto.thriftjava.QueryState; +import io.airlift.log.Logger; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.logging.LogRecord; + +/** + * Class that scribes query completion events + */ +public class QueryScriber implements QueryEventHandler +{ + private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; + private static final int MAX_QUEUE_SIZE = 1000; + + private static final Logger log = Logger.get(QueryScriber.class); + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() + { + @Override protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public QueryScriber() + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + SCRIBE_CATEGORY, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + @Override + public void handle(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); + Optional message = serializeThriftToString(thriftEvent); + + if (message.isPresent()) { + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); + } + else { + log.warn("Unable to serialize QueryCompletionEvent: " + event); + } + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + */ + private Optional serializeThriftToString(TBase thriftMessage) + { + try { + return Optional.of( + Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); + } + catch (TException e) { + log.warn(e, "Could not serialize thrift object" + thriftMessage); + return Optional.absent(); + } + } + + private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); + + thriftEvent.query_id = event.getQueryId(); + thriftEvent.transaction_id = event.getTransactionId(); + thriftEvent.user = event.getUser(); + thriftEvent.principal = event.getPrincipal(); + thriftEvent.source = event.getSource(); + thriftEvent.server_version = event.getServerVersion(); + thriftEvent.environment = event.getEnvironment(); + thriftEvent.catalog = event.getCatalog(); + thriftEvent.schema = event.getSchema(); + thriftEvent.remote_client_address = event.getRemoteClientAddress(); + thriftEvent.user_agent = event.getUserAgent(); + thriftEvent.query_state = QueryState.valueOf(event.getQueryState()); + thriftEvent.uri = event.getUri(); + thriftEvent.field_names = event.getFieldNames(); + thriftEvent.query = event.getQuery(); + thriftEvent.create_time_ms = event.getCreateTime().getMillis(); + thriftEvent.execution_start_time_ms = event.getExecutionStartTime().getMillis(); + thriftEvent.end_time_ms = event.getEndTime().getMillis(); + thriftEvent.queued_time_ms = event.getQueuedTimeMs(); + if (event.getAnalysisTimeMs() != null) { + thriftEvent.analysis_time_ms = event.getAnalysisTimeMs(); + } + if (event.getDistributedPlanningTimeMs() != null) { + thriftEvent.distributed_planning_time_ms = event.getDistributedPlanningTimeMs(); + } + if (event.getTotalSplitWallTimeMs() != null) { + thriftEvent.total_split_wall_time_ms = event.getTotalSplitWallTimeMs(); + } + if (event.getTotalSplitCpuTimeMs() != null) { + thriftEvent.total_split_cpu_time_ms = event.getTotalSplitCpuTimeMs(); + } + if (event.getTotalBytes() != null) { + thriftEvent.total_bytes = event.getTotalBytes(); + } + if (event.getTotalRows() != null) { + thriftEvent.total_rows = event.getTotalRows(); + } + thriftEvent.splits = event.getSplits(); + if (event.getErrorCode() != null) { + thriftEvent.error_code_id = event.getErrorCode(); + } + thriftEvent.error_code_name = event.getErrorCodeName(); + thriftEvent.failure_type = event.getFailureType(); + thriftEvent.failure_message = event.getFailureMessage(); + thriftEvent.failure_task = event.getFailureTask(); + thriftEvent.failure_host = event.getFailureHost(); + thriftEvent.output_stage_json = event.getOutputStageJson(); + thriftEvent.failures_json = event.getFailuresJson(); + thriftEvent.inputs_json = event.getInputsJson(); + thriftEvent.session_properties_json = event.getSessionPropertiesJson(); + thriftEvent.query_wall_time_ms = event.getQueryWallTimeMs(); + if (event.getBytesPerSec() != null) { + thriftEvent.bytes_per_sec = event.getBytesPerSec(); + } + if (event.getBytesPerCpuSec() != null) { + thriftEvent.bytes_per_cpu_sec = event.getBytesPerCpuSec(); + } + if (event.getRowsPerSec() != null) { + thriftEvent.rows_per_sec = event.getRowsPerSec(); + } + if (event.getRowsPerCpuSec() != null) { + thriftEvent.rows_per_cpu_sec = event.getRowsPerCpuSec(); + } + + return thriftEvent; + } +} From 38e6d09f7a30f27c21b2402bd5f0895ffbf08199 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 13:39:30 -0700 Subject: [PATCH 069/151] code clean-up --- .../facebook/presto/event/EventProcessor.java | 3 +++ .../facebook/presto/event/query/QueryEvent.java | 3 --- .../presto/twitter/logging/QueryScriber.java | 17 ++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index edaabc5437de..1ac55ea6d8a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -27,6 +27,9 @@ import java.io.IOException; import java.util.Set; +/** + * Class that listens for airlift events and sends presto events to handlers + */ public class EventProcessor extends AbstractEventClient { private static final String QUERY_CREATED = "QueryCreated"; diff --git a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java index 33ca974b826c..da43e387ee22 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java +++ b/presto-main/src/main/java/com/facebook/presto/event/query/QueryEvent.java @@ -13,9 +13,6 @@ */ package com.facebook.presto.event.query; -/** - * - */ public interface QueryEvent { String getQueryId(); diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index 0110afdf8a61..d4450aaf6a38 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.twitter.logging; -import com.facebook.presto.event.query.QueryCompletionEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.google.common.base.Optional; import com.twitter.logging.BareFormatter$; import com.twitter.logging.Level; import com.twitter.logging.QueueingHandler; import com.twitter.logging.ScribeHandler; +import com.twitter.presto.thriftjava.QueryCompletionEvent; import com.twitter.presto.thriftjava.QueryState; import io.airlift.log.Logger; import org.apache.thrift.TBase; @@ -32,7 +32,7 @@ /** * Class that scribes query completion events */ -public class QueryScriber implements QueryEventHandler +public class QueryScriber implements QueryEventHandler { private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; @@ -66,17 +66,17 @@ public QueryScriber() } @Override - public void handle(QueryCompletionEvent event) + public void handle(com.facebook.presto.event.query.QueryCompletionEvent event) { com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); Optional message = serializeThriftToString(thriftEvent); if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); } else { - log.warn("Unable to serialize QueryCompletionEvent: " + event); + log.warn("Unable to serialize QueryCompletionEvent: " + event); } } @@ -95,10 +95,9 @@ private Optional serializeThriftToString(TBase thriftMessage) } } - private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) + private static QueryCompletionEvent toThriftQueryCompletionEvent(com.facebook.presto.event.query.QueryCompletionEvent event) { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = - new com.twitter.presto.thriftjava.QueryCompletionEvent(); + QueryCompletionEvent thriftEvent = new QueryCompletionEvent(); thriftEvent.query_id = event.getQueryId(); thriftEvent.transaction_id = event.getTransactionId(); From 2ac4c4225977b175d2ecab181aa1f4d9a4cb1907 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 14:02:47 -0700 Subject: [PATCH 070/151] Fix package leakage --- .../main/java/com/facebook/presto/event/EventProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index 1ac55ea6d8a7..c575a6a49818 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -18,7 +18,6 @@ import com.facebook.presto.event.query.QueryEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.facebook.presto.event.query.SplitCompletionEvent; -import com.facebook.presto.twitter.logging.QueryLogger; import com.google.inject.Inject; import io.airlift.event.client.AbstractEventClient; import io.airlift.event.client.EventType; @@ -35,7 +34,7 @@ public class EventProcessor extends AbstractEventClient private static final String QUERY_CREATED = "QueryCreated"; private static final String QUERY_COMPLETION = "QueryCompletion"; private static final String SPLIT_COMPLETION = "SplitCompletion"; - private static final Logger log = Logger.get(QueryLogger.class); + private static final Logger log = Logger.get(EventProcessor.class); private Set> queryCreatedEventHandlers; private Set> queryCompletionEventHandlers; From a8d1ffc0bdadbd5bed81db42135f1ed2ea1479e5 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 15:15:11 -0700 Subject: [PATCH 071/151] Move twitter classes to a twitter package --- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 2 +- .../java/com/facebook/presto/hive/HiveConnectorFactory.java | 3 +++ .../com/facebook/presto/hive/HivePageSourceProvider.java | 2 +- .../{ => twitter}/hive/MetastoreStaticClusterModule.java | 5 ++++- .../{ => twitter}/hive/MetastoreZkDiscoveryBasedModule.java | 3 ++- .../presto/{ => twitter}/hive/ZookeeperMetastoreMonitor.java | 2 +- .../{ => twitter}/hive/ZookeeperServersetHiveCluster.java | 4 +++- .../hive/ZookeeperServersetMetastoreConfig.java | 2 +- .../facebook/presto/{ => twitter}/hive/util/UgiUtils.java | 2 +- .../{ => twitter}/hive/TestZookeeperMetastoreMonitor.java | 4 ++-- .../hive/TestZookeeperServersetMetastoreConfig.java | 2 +- .../facebook/presto/{ => twitter}/hive/util/TestUtils.java | 2 +- 12 files changed, 21 insertions(+), 12 deletions(-) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/MetastoreStaticClusterModule.java (84%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/MetastoreZkDiscoveryBasedModule.java (92%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperMetastoreMonitor.java (99%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperServersetHiveCluster.java (94%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/ZookeeperServersetMetastoreConfig.java (98%) rename presto-hive/src/main/java/com/facebook/presto/{ => twitter}/hive/util/UgiUtils.java (97%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/TestZookeeperMetastoreMonitor.java (98%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/TestZookeeperServersetMetastoreConfig.java (98%) rename presto-hive/src/test/java/com/facebook/presto/{ => twitter}/hive/util/TestUtils.java (94%) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 676b856e6f7c..cd8d88e0d9e8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -16,10 +16,10 @@ import com.facebook.presto.hive.util.HiveFileIterator; import com.facebook.presto.hive.util.ResumableTask; import com.facebook.presto.hive.util.ResumableTasks; -import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.twitter.hive.util.UgiUtils; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index f544a553bdbb..7aa27c3ea11f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -29,6 +29,9 @@ import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.security.ConnectorAccessControl; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.twitter.hive.MetastoreStaticClusterModule; +import com.facebook.presto.twitter.hive.MetastoreZkDiscoveryBasedModule; +import com.facebook.presto.twitter.hive.ZookeeperServersetMetastoreConfig; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.inject.Injector; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index ff1c81f19367..1edee9f8d7d6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.util.UgiUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorPageSourceProvider; @@ -23,6 +22,7 @@ import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.twitter.hive.util.UgiUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java similarity index 84% rename from presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java index 18bfa3d1eff1..b8c1b7dc9a60 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreStaticClusterModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java @@ -11,8 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.StaticHiveCluster; +import com.facebook.presto.hive.StaticMetastoreConfig; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java similarity index 92% rename from presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java index a4d84813ab76..775a5afaf4c8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/MetastoreZkDiscoveryBasedModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java @@ -11,8 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java index 3789ca3c3271..e1d0f2011468 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperMetastoreMonitor.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.google.common.net.HostAndPort; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java similarity index 94% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 48f869ec8ed0..83642e41c50f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -11,8 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; +import com.facebook.presto.hive.HiveCluster; +import com.facebook.presto.hive.HiveMetastoreClientFactory; import com.facebook.presto.hive.metastore.HiveMetastoreClient; import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java index 44c6f9d19188..26e36b469d0c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java rename to presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java index 08f2dde30ff2..6d540bbe3c5a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/UgiUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/UgiUtils.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive.util; +package com.facebook.presto.twitter.hive.util; import org.apache.hadoop.security.UserGroupInformation; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java index 6b82efa22f97..d0eba9f4e6e5 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java @@ -11,10 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; -import com.facebook.presto.hive.util.TestUtils; +import com.facebook.presto.twitter.hive.util.TestUtils; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.I0Itec.zkclient.ZkClient; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java index bb1117697296..6992a752dea1 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.twitter.hive; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java similarity index 94% rename from presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java rename to presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java index 4315bac667c2..379ad3877e32 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive.util; +package com.facebook.presto.twitter.hive.util; import java.io.IOException; import java.net.ServerSocket; From 73c53029594b351bc9f593b657e6bd86c0475a82 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Tue, 29 Mar 2016 15:26:47 -0700 Subject: [PATCH 072/151] Default package is presto and thrift is custom --- .../facebook/presto/twitter/logging/QueryScriber.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index d4450aaf6a38..306029550ed6 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.twitter.logging; +import com.facebook.presto.event.query.QueryCompletionEvent; import com.facebook.presto.event.query.QueryEventHandler; import com.google.common.base.Optional; import com.twitter.logging.BareFormatter$; import com.twitter.logging.Level; import com.twitter.logging.QueueingHandler; import com.twitter.logging.ScribeHandler; -import com.twitter.presto.thriftjava.QueryCompletionEvent; import com.twitter.presto.thriftjava.QueryState; import io.airlift.log.Logger; import org.apache.thrift.TBase; @@ -32,7 +32,7 @@ /** * Class that scribes query completion events */ -public class QueryScriber implements QueryEventHandler +public class QueryScriber implements QueryEventHandler { private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; @@ -66,7 +66,7 @@ public QueryScriber() } @Override - public void handle(com.facebook.presto.event.query.QueryCompletionEvent event) + public void handle(QueryCompletionEvent event) { com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); Optional message = serializeThriftToString(thriftEvent); @@ -95,9 +95,10 @@ private Optional serializeThriftToString(TBase thriftMessage) } } - private static QueryCompletionEvent toThriftQueryCompletionEvent(com.facebook.presto.event.query.QueryCompletionEvent event) + private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) { - QueryCompletionEvent thriftEvent = new QueryCompletionEvent(); + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); thriftEvent.query_id = event.getQueryId(); thriftEvent.transaction_id = event.getTransactionId(); From fadd0f13f702a516a67b29c82dc8880ef19fce42 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:08:39 -0700 Subject: [PATCH 073/151] Removing test from scribe category --- .../java/com/facebook/presto/twitter/logging/QueryScriber.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index 306029550ed6..ad3e52fb9c21 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -34,7 +34,7 @@ */ public class QueryScriber implements QueryEventHandler { - private static final String SCRIBE_CATEGORY = "test_presto_query_complete"; + private static final String SCRIBE_CATEGORY = "presto_query_complete"; private static final int MAX_QUEUE_SIZE = 1000; private static final Logger log = Logger.get(QueryScriber.class); From 2e5c3f2ee54859b6f8d9a3a4c14c20da836175fd Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:25:12 -0700 Subject: [PATCH 074/151] change version to avoid merge conflicts --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index ae36b8cf3d9c..dbe9ff96aecd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.141-tw-0.20 + 0.143 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 00cc629d364c..f97c8bba0c51 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3652e17bd06f..428c8c649476 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 8f86dabd3e25..2d6d529d0fae 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 1c0d41086973..f939fcd80924 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 1bdeaecdcea2..b6558dd1ae17 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 4f0675eda340..fabc5079fb31 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index bdcff6cdb591..841119ce5d50 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index f54c94db8398..1afa4e2137b9 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 4b4ad310d3e1..0aa73f400de0 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 2c4cc361f6fb..e2cce02150c6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 7bb7379f70b6..6321368c053f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 09e13a19e68d..e327f50e7fd0 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 6e304d8b10bd..0b2e9211761b 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index cfa615c8d2dc..f6e99377daf2 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index cebd8614c068..2ec69e52029e 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 24ca263be84e..84c75bbc045b 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 4ae769912294..b9d05f4c16c9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 31620ae9fe5a..2aa0870b3928 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index dfe5c5d08d7a..789340a2d1fc 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index b5e67298855a..1899f5fcca69 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 4722581cd3a5..35b56aa6b724 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index d710b58886f5..160e11c44e72 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 830af643de31..a9c126945d7a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index c8e057ed14aa..018204dea0e0 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index bfe2383a567f..4b00570b97cd 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 3b527cbaa5dc..dee555abf4bb 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 5c724e169c0c..e8b911e4b433 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 4c226d7edc39..8c618e94b57c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 14038c648f0f..d4a6474c05bd 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 2c4b2c1f49eb..2a35482c59a9 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index a977f2d08420..39bbf912871a 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 301e4c4efe55..bb6947fe3db6 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index dc8b11426992..a1d92c2c2639 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 25c2414ce83f..a81e47fb1e3e 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.141-tw-0.20 + 0.143 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 3979e5a18555..62667b35f11d 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 9dbeee8b8252..21c117a922ad 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.141-tw-0.20 + 0.143 presto-verifier From 1673479f824df2bfe0c63c91233eb1709d08dad7 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 10:32:32 -0700 Subject: [PATCH 075/151] Update version to 0.143-tw-21 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index a15ff943598a..4d3e280a8b86 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143 + 0.143-tw-21 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index f97c8bba0c51..9a3b30321ab3 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 428c8c649476..3e9996b54c4a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 2d6d529d0fae..f678a84674f6 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index f939fcd80924..96374f2176cf 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index b6558dd1ae17..bb5f42df5816 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index d75efa6ae5d0..2d0b7b238afd 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 841119ce5d50..b1b1c9af3a69 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 1afa4e2137b9..89ffee3b5137 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0aa73f400de0..46f1cd60f31f 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e2cce02150c6..ba4b98f37e6a 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index d6249e4c7649..dfab48e8c1fe 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 45dafbefb8e4..a7ed3e068632 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 5dd2f1cdb7b1..d1888fde6fe7 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 24e1bd0aa0d1..3f6eae9a0306 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 0ad2ec1f4a09..f1551983ce7c 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 84c75bbc045b..5840a8d8c5db 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index b9d05f4c16c9..e29367f1e07d 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2aa0870b3928..b19205980eee 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 789340a2d1fc..955f03a0ec5b 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1899f5fcca69..04c5a023509f 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 35b56aa6b724..9aecc0f8e037 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 160e11c44e72..b132a24e22e8 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a9c126945d7a..2af8ad644181 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 018204dea0e0..0929844e2231 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6a0ad32d8c94..7f8944a0ecbd 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index dee555abf4bb..067fef0d7288 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index e8b911e4b433..b9cb797cbbc1 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 8c618e94b57c..10149cb92d1b 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index d4a6474c05bd..38e7af2f19cc 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 2a35482c59a9..9183920dfa3c 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 39bbf912871a..c6f3399b8ff7 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index bb6947fe3db6..81dc9466c696 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index a1d92c2c2639..ea738c49db14 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index a81e47fb1e3e..4376173271e8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143 + 0.143-tw-21 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 62667b35f11d..9a704e9671b1 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 21c117a922ad..3511883027ed 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143 + 0.143-tw-21 presto-verifier From 4e48cbd905adb085c1986050632beb2679796c63 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 11:07:23 -0700 Subject: [PATCH 076/151] Fixed to not use illegal import --- .../presto/twitter/hive/TestZookeeperMetastoreMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java index d0eba9f4e6e5..1f89464aedce 100644 --- a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.twitter.hive; -import com.facebook.presto.hadoop.shaded.com.google.common.collect.ImmutableList; import com.facebook.presto.twitter.hive.util.TestUtils; +import com.google.common.collect.ImmutableList; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.I0Itec.zkclient.ZkClient; From 55a39606d49d3706b98dc1fa20285dec76614b7c Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 11:22:11 -0700 Subject: [PATCH 077/151] Reduce logging of unsupported HttpRequest events --- .../src/main/java/com/facebook/presto/event/EventProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java index c575a6a49818..eef91027a4a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/EventProcessor.java @@ -72,8 +72,6 @@ protected void postEvent(T event) case SPLIT_COMPLETION: handle(splitCompletionEventHandlers, type, (SplitCompletionEvent) event); break; - default: - log.warn("Unrecognized event found: " + type); } } From 8729a4ef5268876deef4c1ebc36630ddb9607fe7 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 13:17:31 -0700 Subject: [PATCH 078/151] Fix version from 0.143-tw-21 to 0.143-tw-0.21 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 4d3e280a8b86..dcd36597ae72 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-21 + 0.143-tw-0.21 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 9a3b30321ab3..f5509ae7ace0 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 3e9996b54c4a..f4798cbed1db 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index f678a84674f6..9cdb2ba7a288 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 96374f2176cf..cee636f28457 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index bb5f42df5816..3f9d4cc315b8 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 2d0b7b238afd..1584fab21a1a 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index b1b1c9af3a69..0112a2ec8df3 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 89ffee3b5137..86694f5e8351 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 46f1cd60f31f..802a03d546d3 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index ba4b98f37e6a..93ecc58122af 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index dfab48e8c1fe..98bf09737d7f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a7ed3e068632..5a5325b78ffd 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index d1888fde6fe7..c497b8c804e2 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 3f6eae9a0306..08a42164f5a9 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f1551983ce7c..f2b6bfe18024 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 5840a8d8c5db..0649fdbca865 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index e29367f1e07d..6e482242ae9e 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index b19205980eee..5bb58fec8ffc 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 955f03a0ec5b..51a6e06ebe65 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 04c5a023509f..c017ed0e635d 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 9aecc0f8e037..a85b0b403e1f 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index b132a24e22e8..a67f058a17ed 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 2af8ad644181..97e0abd530f1 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 0929844e2231..7b2409dcb2ea 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 7f8944a0ecbd..b14447d4255d 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 067fef0d7288..98098e7bca6b 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index b9cb797cbbc1..007a962d5113 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 10149cb92d1b..d0e5fcb1a355 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 38e7af2f19cc..8c699831a585 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9183920dfa3c..0c4200ec3f96 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c6f3399b8ff7..26814c291cf9 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 81dc9466c696..9d0629fca8e8 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index ea738c49db14..fca9e2640ba5 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 4376173271e8..4ab85400cfe8 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-21 + 0.143-tw-0.21 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9a704e9671b1..38342b08d9b7 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 3511883027ed..2d075a28a214 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-21 + 0.143-tw-0.21 presto-verifier From e6579abad2f31e5966c5edffda2ca19ed3deb5a2 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Wed, 30 Mar 2016 22:40:07 -0700 Subject: [PATCH 079/151] Fix name of log category --- .../java/com/facebook/presto/twitter/logging/QueryScriber.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java index ad3e52fb9c21..5e55aa1ac195 100644 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -34,7 +34,7 @@ */ public class QueryScriber implements QueryEventHandler { - private static final String SCRIBE_CATEGORY = "presto_query_complete"; + private static final String SCRIBE_CATEGORY = "presto_query_completion"; private static final int MAX_QUEUE_SIZE = 1000; private static final Logger log = Logger.get(QueryScriber.class); From a1ba15bbfbadc8c46d5cc2118b8a53c9e5365455 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 21 Jul 2016 17:38:02 -0700 Subject: [PATCH 080/151] Fix scala dependency issues with twitter-master; removed scala 2.11 dependencies from thrift-java; downgraded util-logging to 2.10 --- pom.xml | 19 ++++++++++++++++--- presto-main/pom.xml | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index dcd36597ae72..7998b80610f2 100644 --- a/pom.xml +++ b/pom.xml @@ -768,12 +768,25 @@ org.scala-lang.modules scala-parser-combinators_2.11 + + com.twitter + scrooge-core + + + org.scala-lang + scala-library + + + org.scala-lang + scala-reflect + + com.twitter - util-logging_2.11 - 6.33.0 + util-logging_2.10 + 6.34.0 commons-logging @@ -784,7 +797,7 @@ org.scala-lang scala-library - 2.11.7 + 2.10.6 commons-logging diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 51a6e06ebe65..f7aeef580f94 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -304,7 +304,7 @@ com.twitter - util-logging_2.11 + util-logging_2.10 org.apache.thrift From 484459328f2161d2e31d34112d0105cda72ec342 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 9 Aug 2016 11:42:56 -0700 Subject: [PATCH 081/151] Cherry pick Tableau connector fix with bad defaults --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- .../src/main/resources/webapp/tableau/presto-client.js | 10 ++++++++-- .../resources/webapp/tableau/presto-connector.html | 6 ++---- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 39 files changed, 48 insertions(+), 44 deletions(-) diff --git a/pom.xml b/pom.xml index 7998b80610f2..f1e99d883be1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-0.21 + 0.143-tw-0.22 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index f5509ae7ace0..f723feaac0d2 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index f4798cbed1db..f2b8bbbc0f79 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 9cdb2ba7a288..7a40052b3460 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index cee636f28457..5127b74af96c 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 3f9d4cc315b8..8ecc331b8526 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 1584fab21a1a..cbf7b6065d3d 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 0112a2ec8df3..7a808d8896b8 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 86694f5e8351..09d5d5d01fbb 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 802a03d546d3..d293241818a3 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 93ecc58122af..59bd652ed59d 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 98bf09737d7f..b10f1b9646fe 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 5a5325b78ffd..1e0c40197d7a 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index c497b8c804e2..f9f34bfbe92e 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 08a42164f5a9..04cc2752f767 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f2b6bfe18024..7602d1fe2634 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 0649fdbca865..36bfcdf41432 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 6e482242ae9e..e23c28c1df56 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 5bb58fec8ffc..7bced3cb454b 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index f7aeef580f94..6f9499075e9f 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-main diff --git a/presto-main/src/main/resources/webapp/tableau/presto-client.js b/presto-main/src/main/resources/webapp/tableau/presto-client.js index e0f0a14f4ebc..3bc357dd1b65 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-client.js +++ b/presto-main/src/main/resources/webapp/tableau/presto-client.js @@ -28,11 +28,17 @@ function StatementClient(connectionData, headerCallback, dataCallback, errorCall this.headers = { "X-Presto-User": this.user ? this.user : 'N/A', "X-Presto-Source": this.source, - "X-Presto-Catalog": this.catalog, - "X-Presto-Schema": this.schema, "X-Presto-Session": this.sessionParameters }; + if (!(this.catalog === undefined)) { + this.headers["X-Presto-Catalog"] = this.catalog + } + + if (!(this.schema === undefined)) { + this.headers["X-Presto-Schema"] = this.schema + } + // lastRecordNumber starts with 0 according to Tableau web connector docs this.submitQuery(0); } diff --git a/presto-main/src/main/resources/webapp/tableau/presto-connector.html b/presto-main/src/main/resources/webapp/tableau/presto-connector.html index 1b00dd795845..3b915fdfb4b2 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-connector.html +++ b/presto-main/src/main/resources/webapp/tableau/presto-connector.html @@ -135,8 +135,6 @@ function populateCatalogs() { var catalogClient = new StatementClient({ - 'catalog': 'hive', - 'schema': 'default', 'query': 'show catalogs' }, function() {}, @@ -191,8 +189,8 @@ } populateCatalogs(); - // initially load the schemas of the hive catalog - populateSchemas('hive'); + // initially load the schemas of the first catalog + populateSchemas($('#catalog option:selected').val()); document.getElementById("catalog").addEventListener('change', function() { diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c017ed0e635d..e7283f16351e 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index a85b0b403e1f..9f01fee99131 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index a67f058a17ed..fdd0ceab14f0 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 97e0abd530f1..817ffe5fc72a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7b2409dcb2ea..910721d13e73 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index b14447d4255d..b558349d3f8f 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 98098e7bca6b..dc6480483553 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 007a962d5113..1d6638c663f7 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index d0e5fcb1a355..d53c3390fa7f 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 8c699831a585..653ed487fc2e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 0c4200ec3f96..117ed5996b80 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 26814c291cf9..e424ee3d0725 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 9d0629fca8e8..de117022e736 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index fca9e2640ba5..5dc0a25029fd 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 4ab85400cfe8..8a66e9e37e43 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 38342b08d9b7..d861a447aafc 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 2d075a28a214..63bf1e044ee3 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-verifier From 1df4359c90121bd58773d9c77b2320e728aab130 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Mon, 15 Aug 2016 11:17:27 -0700 Subject: [PATCH 082/151] Cherry pick Tableau connector fix with bad defaults --- .../src/main/resources/webapp/tableau/presto-client.js | 10 ++++++++-- .../resources/webapp/tableau/presto-connector.html | 6 ++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/presto-main/src/main/resources/webapp/tableau/presto-client.js b/presto-main/src/main/resources/webapp/tableau/presto-client.js index e0f0a14f4ebc..3bc357dd1b65 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-client.js +++ b/presto-main/src/main/resources/webapp/tableau/presto-client.js @@ -28,11 +28,17 @@ function StatementClient(connectionData, headerCallback, dataCallback, errorCall this.headers = { "X-Presto-User": this.user ? this.user : 'N/A', "X-Presto-Source": this.source, - "X-Presto-Catalog": this.catalog, - "X-Presto-Schema": this.schema, "X-Presto-Session": this.sessionParameters }; + if (!(this.catalog === undefined)) { + this.headers["X-Presto-Catalog"] = this.catalog + } + + if (!(this.schema === undefined)) { + this.headers["X-Presto-Schema"] = this.schema + } + // lastRecordNumber starts with 0 according to Tableau web connector docs this.submitQuery(0); } diff --git a/presto-main/src/main/resources/webapp/tableau/presto-connector.html b/presto-main/src/main/resources/webapp/tableau/presto-connector.html index 1b00dd795845..3b915fdfb4b2 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-connector.html +++ b/presto-main/src/main/resources/webapp/tableau/presto-connector.html @@ -135,8 +135,6 @@ function populateCatalogs() { var catalogClient = new StatementClient({ - 'catalog': 'hive', - 'schema': 'default', 'query': 'show catalogs' }, function() {}, @@ -191,8 +189,8 @@ } populateCatalogs(); - // initially load the schemas of the hive catalog - populateSchemas('hive'); + // initially load the schemas of the first catalog + populateSchemas($('#catalog option:selected').val()); document.getElementById("catalog").addEventListener('change', function() { From 1e07aeaa7fee3a719f0875ff853c1ae495cd9d33 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 9 Aug 2016 11:42:56 -0700 Subject: [PATCH 083/151] Cherry pick Tableau connector fix with bad defaults --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 7998b80610f2..f1e99d883be1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-0.21 + 0.143-tw-0.22 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index f5509ae7ace0..f723feaac0d2 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index f4798cbed1db..f2b8bbbc0f79 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 9cdb2ba7a288..7a40052b3460 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index cee636f28457..5127b74af96c 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 3f9d4cc315b8..8ecc331b8526 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 1584fab21a1a..cbf7b6065d3d 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 0112a2ec8df3..7a808d8896b8 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 86694f5e8351..09d5d5d01fbb 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 802a03d546d3..d293241818a3 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 93ecc58122af..59bd652ed59d 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 98bf09737d7f..b10f1b9646fe 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 5a5325b78ffd..1e0c40197d7a 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index c497b8c804e2..f9f34bfbe92e 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 08a42164f5a9..04cc2752f767 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f2b6bfe18024..7602d1fe2634 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 0649fdbca865..36bfcdf41432 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 6e482242ae9e..e23c28c1df56 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 5bb58fec8ffc..7bced3cb454b 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index f7aeef580f94..6f9499075e9f 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c017ed0e635d..e7283f16351e 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index a85b0b403e1f..9f01fee99131 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index a67f058a17ed..fdd0ceab14f0 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 97e0abd530f1..817ffe5fc72a 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7b2409dcb2ea..910721d13e73 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index b14447d4255d..b558349d3f8f 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 98098e7bca6b..dc6480483553 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 007a962d5113..1d6638c663f7 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index d0e5fcb1a355..d53c3390fa7f 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 8c699831a585..653ed487fc2e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 0c4200ec3f96..117ed5996b80 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 26814c291cf9..e424ee3d0725 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 9d0629fca8e8..de117022e736 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index fca9e2640ba5..5dc0a25029fd 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 4ab85400cfe8..8a66e9e37e43 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.21 + 0.143-tw-0.22 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 38342b08d9b7..d861a447aafc 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 2d075a28a214..63bf1e044ee3 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.21 + 0.143-tw-0.22 presto-verifier From 6e9bfea61ae214640c764d57da42aeaefced1387 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 18 Aug 2016 11:46:43 -0700 Subject: [PATCH 084/151] add hack to javascript to make requests in https if page was visited in https --- .../src/main/resources/webapp/tableau/presto-client.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/presto-main/src/main/resources/webapp/tableau/presto-client.js b/presto-main/src/main/resources/webapp/tableau/presto-client.js index 3bc357dd1b65..abd6c8b045e9 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-client.js +++ b/presto-main/src/main/resources/webapp/tableau/presto-client.js @@ -17,6 +17,8 @@ function StatementClient(connectionData, headerCallback, dataCallback, errorCall this.currentResults = null; this.valid = true; + this.isHttps = window.location.protocol === "https:" + if (!(connectionData.sessionParameters === undefined)) { var parameterMap = JSON.parse(connectionData.sessionParameters); for (var name in parameterMap) { @@ -72,7 +74,7 @@ StatementClient.prototype.advance = function(lastRecordNumber) { var statementClient = this; $.ajax({ type: "GET", - url: this.currentResults.nextUri, + url: this.isHttps ? this.currentResults.nextUri.replace("http", "https") : this.currentResults.nextUri, headers: this.headers, dataType: 'json', // FIXME having problems when async: true From bf4dec27e18098a0a8c29d5cbdba57c84cb3b6df Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 18 Aug 2016 13:14:08 -0700 Subject: [PATCH 085/151] address Zhihao's comment to mind case where nextUri might already have https --- presto-main/src/main/resources/webapp/tableau/presto-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/resources/webapp/tableau/presto-client.js b/presto-main/src/main/resources/webapp/tableau/presto-client.js index abd6c8b045e9..14de90193895 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-client.js +++ b/presto-main/src/main/resources/webapp/tableau/presto-client.js @@ -74,7 +74,7 @@ StatementClient.prototype.advance = function(lastRecordNumber) { var statementClient = this; $.ajax({ type: "GET", - url: this.isHttps ? this.currentResults.nextUri.replace("http", "https") : this.currentResults.nextUri, + url: this.isHttps ? this.currentResults.nextUri.replace(/^http:/, 'https:') : this.currentResults.nextUri, headers: this.headers, dataType: 'json', // FIXME having problems when async: true From 443f76b73ef3131e7383656d86ae5721cac7ca4e Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 18 Aug 2016 14:41:16 -0700 Subject: [PATCH 086/151] update tag to 0.143-tw-0.23 --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index f1e99d883be1..56888bf9ec0e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-0.22 + 0.143-tw-0.23 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index f723feaac0d2..91f462cfadb0 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index f2b8bbbc0f79..b3eb4d56e859 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 7a40052b3460..ca72cfe3d5d3 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.22 + 0.143-tw-0.23 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5127b74af96c..8aa14bce49fd 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 8ecc331b8526..7ade60a4cb0c 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index cbf7b6065d3d..990c3da0dba6 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 7a808d8896b8..0dbadefd62ae 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 09d5d5d01fbb..2d99c244cf0b 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index d293241818a3..e5db3aa84671 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 59bd652ed59d..08404aab6545 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index b10f1b9646fe..d9036e8f3d95 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 1e0c40197d7a..f9d45157f9fc 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index f9f34bfbe92e..ea710e04f016 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 04cc2752f767..f6f9554cc4f1 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 7602d1fe2634..deaf1b6f7ce2 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 36bfcdf41432..299cfbd382a3 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index e23c28c1df56..782d4fc3894f 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 7bced3cb454b..875a03aa63f7 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 6f9499075e9f..88d36739d70c 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index e7283f16351e..05a809b78044 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 9f01fee99131..19104889d0e1 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index fdd0ceab14f0..ff25cd66c973 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 817ffe5fc72a..0040747083d3 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 910721d13e73..c0d7e1b5c67c 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index b558349d3f8f..d3c75dc22d20 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.22 + 0.143-tw-0.23 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index dc6480483553..f6df5f755d74 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 1d6638c663f7..85e013e3bc61 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index d53c3390fa7f..38200cc68020 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 653ed487fc2e..aa2792d3b0d6 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 117ed5996b80..9b4d72aa6ddf 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index e424ee3d0725..20dcbb999b82 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index de117022e736..9f0efb881244 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 5dc0a25029fd..11cf1b90fe7c 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 8a66e9e37e43..574825fa8a61 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.22 + 0.143-tw-0.23 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index d861a447aafc..28dfc515bf06 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 63bf1e044ee3..430df7fc5da0 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.22 + 0.143-tw-0.23 presto-verifier From 67d22d7e4b8431fe3ea69734ceaad93e2cc0468e Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 25 Oct 2016 16:52:05 -0700 Subject: [PATCH 087/151] Added shuffle for metastores in order to balance connection load --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 83642e41c50f..1885f515848d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import java.util.Collections; import java.util.List; import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; @@ -51,6 +52,7 @@ public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, H public HiveMetastoreClient createMetastoreClient() { List metastores = zkMetastoreMonitor.getServers(); + Collections.shuffle(metastores); TTransportException lastException = null; for (HostAndPort metastore : metastores) { try { From ecfa6c0a656f3f59864fd5f49efd1e7c84220479 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 27 Oct 2016 17:49:54 -0700 Subject: [PATCH 088/151] Added log for hive connection and log for RetryDriver --- .../src/main/java/com/facebook/presto/hive/RetryDriver.java | 4 +++- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java index d969a522a750..c2c1fac24ffc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java @@ -136,16 +136,18 @@ public V run(String callableName, Callable callable) return callable.call(); } catch (Exception e) { + log.debug("Failed on executing %s with attempt %d, Exception: %s", callableName, attempt, e.getMessage()); e = exceptionMapper.apply(e); for (Class clazz : exceptionWhiteList) { if (clazz.isInstance(e)) { + log.debug("Exception is in whitelist."); throw e; } } if (attempt >= maxAttempts || Duration.nanosSince(startTime).compareTo(maxRetryTime) >= 0) { + log.debug("Maximum attempts or maximum retry time reached. attempt: %d, maxAttempts: %d, duration: [%s] maxRetryTime: [%s]", attempt, maxAttempts, Duration.nanosSince(startTime).toString(), maxRetryTime.toString()); throw e; } - log.debug("Failed on executing %s with attempt %d, will retry. Exception: %s", callableName, attempt, e.getMessage()); int delayInMs = (int) Math.min(minSleepTime.toMillis() * Math.pow(scaleFactor, attempt - 1), maxSleepTime.toMillis()); TimeUnit.MILLISECONDS.sleep(delayInMs); diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 1885f515848d..1f1b6cad01a4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -60,10 +60,11 @@ public HiveMetastoreClient createMetastoreClient() return clientFactory.create(metastore.getHostText(), metastore.getPort()); } catch (TTransportException e) { + log.debug("Failed connecting to Hive metastore at: " + metastore.toString()); lastException = e; } } - throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore", lastException); + throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore: " + metastores, lastException); } } From dee633688c4aa2c49da0fa7c3549d22087eb54b1 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 31 Oct 2016 15:45:52 -0700 Subject: [PATCH 089/151] Used RuntimeException to replace PrestoException, prevent whitelist stops RetryDriver --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 1f1b6cad01a4..ca684b269cf5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -16,7 +16,6 @@ import com.facebook.presto.hive.HiveCluster; import com.facebook.presto.hive.HiveMetastoreClientFactory; import com.facebook.presto.hive.metastore.HiveMetastoreClient; -import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.apache.thrift.transport.TTransportException; @@ -26,7 +25,6 @@ import java.util.Collections; import java.util.List; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static java.util.Objects.requireNonNull; public class ZookeeperServersetHiveCluster @@ -65,6 +63,6 @@ public HiveMetastoreClient createMetastoreClient() } } - throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore: " + metastores, lastException); + throw new RuntimeException("Failed connecting to Hive metastore: " + metastores, lastException); } } From 56d7c362a1cdf4ec76d5b6975f8227fff853390a Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 31 Oct 2016 18:22:10 -0700 Subject: [PATCH 090/151] Used formatter for logs --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index ca684b269cf5..379352d4100e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -54,15 +54,15 @@ public HiveMetastoreClient createMetastoreClient() TTransportException lastException = null; for (HostAndPort metastore : metastores) { try { - log.info("Connecting to metastore at: " + metastore.toString()); + log.info("Connecting to metastore at: %s", metastore.toString()); return clientFactory.create(metastore.getHostText(), metastore.getPort()); } catch (TTransportException e) { - log.debug("Failed connecting to Hive metastore at: " + metastore.toString()); + log.debug("Failed connecting to Hive metastore at: %s", metastore.toString()); lastException = e; } } - throw new RuntimeException("Failed connecting to Hive metastore: " + metastores, lastException); + throw new RuntimeException("Failed connecting to Hive metastore.", lastException); } } From 36175ec6d47d24ff3d7519f68d3e5c0d0af0fc98 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Wed, 2 Nov 2016 15:19:05 -0700 Subject: [PATCH 091/151] remove obsolete event listener implementation - part 1 of 3 for upgrading to 0.155 --- pom.xml | 68 ------- presto-main/pom.xml | 18 -- .../facebook/presto/server/PrestoServer.java | 7 - .../presto/twitter/TwitterModuleLoader.java | 64 ------- .../presto/twitter/logging/QueryLogger.java | 78 -------- .../presto/twitter/logging/QueryScriber.java | 169 ------------------ 6 files changed, 404 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java diff --git a/pom.xml b/pom.xml index 56888bf9ec0e..73a2b815da39 100644 --- a/pom.xml +++ b/pom.xml @@ -737,74 +737,6 @@ hive-apache-jdbc 0.13.1-1 - - - - com.twitter - presto-thrift-java - 0.0.1 - - - com.twitter - util-core_2.11 - - - com.twitter - util-core-java - - - com.twitter - util-function_2.10 - - - com.twitter - util-function-java - - - commons-logging - commons-logging - - - org.scala-lang.modules - scala-parser-combinators_2.11 - - - com.twitter - scrooge-core - - - org.scala-lang - scala-library - - - org.scala-lang - scala-reflect - - - - - - com.twitter - util-logging_2.10 - 6.34.0 - - - commons-logging - commons-logging - - - - - org.scala-lang - scala-library - 2.10.6 - - - commons-logging - commons-logging - - - diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 88d36739d70c..c6f9b388ceab 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -296,24 +296,6 @@ tpch test - - - - com.twitter - presto-thrift-java - - - com.twitter - util-logging_2.10 - - - org.apache.thrift - libthrift - - - org.scala-lang - scala-library - diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index f7185a279dc7..fda5706d9ebe 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -119,8 +119,6 @@ public void run() ); - modules.addAll(getAdditionalModules()); - Bootstrap app = new Bootstrap(modules.build()); try { @@ -149,11 +147,6 @@ public void run() } } - protected Iterable getAdditionalModules() - { - return com.facebook.presto.twitter.TwitterModuleLoader.getAdditionalModules(); - } - private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) { // get existing announcement diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java deleted file mode 100644 index eb23a7650a09..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter; - -import com.facebook.presto.event.EventProcessor; -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryCreatedEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.facebook.presto.event.query.SplitCompletionEvent; -import com.facebook.presto.twitter.logging.QueryLogger; -import com.facebook.presto.twitter.logging.QueryScriber; -import com.google.common.collect.ImmutableList; -import com.google.inject.Module; -import com.google.inject.Scopes; -import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.Multibinder; -import io.airlift.event.client.EventClient; - -/** - * Loader that initializes custom Twitter code to inject into Presto. Whenever - * possible we should use this pattern to inject custom functionality, since it - * makes it easier to differentiate our patches from the core OS code. - * - * If the functionality we wish to add/override isn't currently possible to via - * overriding a guice module, we should contribute the necessary modules/interfaces - * into the OS Presto code base to make it possible. - */ -public class TwitterModuleLoader -{ - private TwitterModuleLoader() - { - } - - public static Iterable getAdditionalModules() - { - return ImmutableList.of( - binder -> Multibinder.newSetBinder(binder, EventClient.class) - .addBinding() - .to(EventProcessor.class) - .in(Scopes.SINGLETON), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) - .addBinding() - .to(new TypeLiteral(){}) - .in(Scopes.SINGLETON), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) - .addBinding() - .to(QueryScriber.class) - .in(Scopes.SINGLETON) - ); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java deleted file mode 100644 index 02edd8da2440..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter.logging; - -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.facebook.presto.spi.StandardErrorCode; -import io.airlift.log.Logger; -import io.airlift.units.Duration; - -import java.util.concurrent.TimeUnit; - -/** - * Class that logs query events to a file - */ -public class QueryLogger implements QueryEventHandler -{ - private static final int MAX_QUERY_LENGTH = 1000; - private static final String DASH = "-"; - private static final String COLON = ":"; - private static final String SPACE = " "; - private static final String ELIPSIS = "..."; - private static final String QUERY_COMPLETION = "QueryCompletion"; - - private static final Logger log = Logger.get(QueryLogger.class); - - @Override - public void handle(QueryCompletionEvent event) - { - String errorType = DASH; - String errorCode = DASH; - if (event.getErrorCode() != null) { - errorType = StandardErrorCode.toErrorType(event.getErrorCode()).toString(); - if (event.getErrorCodeName() != null) { - errorCode = event.getErrorCodeName() + COLON + event.getErrorCode(); - } - } - - Duration duration = (new Duration( - event.getQueryWallTimeMs(), TimeUnit.MILLISECONDS)) - .convertToMostSuccinctTimeUnit(); - - log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - QUERY_COMPLETION, event.getQueryId(), toLogValue(event.getRemoteClientAddress()), - event.getQueryState(), errorType, errorCode, event.getUser(), duration, - event.getSplits(), event.getTotalRows(), event.getTotalBytes(), - cleanseAndTrimQuery(event.getQuery()))); - } - - private static String toLogValue(Object object) - { - if (object == null) { - return DASH; - } - else { - return object.toString(); - } - } - - private static String cleanseAndTrimQuery(String query) - { - if (query.length() > MAX_QUERY_LENGTH) { - query = query.substring(0, MAX_QUERY_LENGTH) + ELIPSIS; - } - return query.replace(System.getProperty("line.separator"), SPACE); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java deleted file mode 100644 index 5e55aa1ac195..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter.logging; - -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.google.common.base.Optional; -import com.twitter.logging.BareFormatter$; -import com.twitter.logging.Level; -import com.twitter.logging.QueueingHandler; -import com.twitter.logging.ScribeHandler; -import com.twitter.presto.thriftjava.QueryState; -import io.airlift.log.Logger; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; - -import java.util.Base64; -import java.util.logging.LogRecord; - -/** - * Class that scribes query completion events - */ -public class QueryScriber implements QueryEventHandler -{ - private static final String SCRIBE_CATEGORY = "presto_query_completion"; - private static final int MAX_QUEUE_SIZE = 1000; - - private static final Logger log = Logger.get(QueryScriber.class); - - private QueueingHandler queueingHandler; - - // TSerializer is not thread safe - private final ThreadLocal serializer = new ThreadLocal() - { - @Override protected TSerializer initialValue() - { - return new TSerializer(); - } - }; - - public QueryScriber() - { - ScribeHandler scribeHandler = new ScribeHandler( - ScribeHandler.DefaultHostname(), - ScribeHandler.DefaultPort(), - SCRIBE_CATEGORY, - ScribeHandler.DefaultBufferTime(), - ScribeHandler.DefaultConnectBackoff(), - ScribeHandler.DefaultMaxMessagesPerTransaction(), - ScribeHandler.DefaultMaxMessagesToBuffer(), - BareFormatter$.MODULE$, - scala.Option.apply((Level) null)); - queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); - } - - @Override - public void handle(QueryCompletionEvent event) - { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); - Optional message = serializeThriftToString(thriftEvent); - - if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); - } - else { - log.warn("Unable to serialize QueryCompletionEvent: " + event); - } - } - - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - */ - private Optional serializeThriftToString(TBase thriftMessage) - { - try { - return Optional.of( - Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); - } - catch (TException e) { - log.warn(e, "Could not serialize thrift object" + thriftMessage); - return Optional.absent(); - } - } - - private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) - { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = - new com.twitter.presto.thriftjava.QueryCompletionEvent(); - - thriftEvent.query_id = event.getQueryId(); - thriftEvent.transaction_id = event.getTransactionId(); - thriftEvent.user = event.getUser(); - thriftEvent.principal = event.getPrincipal(); - thriftEvent.source = event.getSource(); - thriftEvent.server_version = event.getServerVersion(); - thriftEvent.environment = event.getEnvironment(); - thriftEvent.catalog = event.getCatalog(); - thriftEvent.schema = event.getSchema(); - thriftEvent.remote_client_address = event.getRemoteClientAddress(); - thriftEvent.user_agent = event.getUserAgent(); - thriftEvent.query_state = QueryState.valueOf(event.getQueryState()); - thriftEvent.uri = event.getUri(); - thriftEvent.field_names = event.getFieldNames(); - thriftEvent.query = event.getQuery(); - thriftEvent.create_time_ms = event.getCreateTime().getMillis(); - thriftEvent.execution_start_time_ms = event.getExecutionStartTime().getMillis(); - thriftEvent.end_time_ms = event.getEndTime().getMillis(); - thriftEvent.queued_time_ms = event.getQueuedTimeMs(); - if (event.getAnalysisTimeMs() != null) { - thriftEvent.analysis_time_ms = event.getAnalysisTimeMs(); - } - if (event.getDistributedPlanningTimeMs() != null) { - thriftEvent.distributed_planning_time_ms = event.getDistributedPlanningTimeMs(); - } - if (event.getTotalSplitWallTimeMs() != null) { - thriftEvent.total_split_wall_time_ms = event.getTotalSplitWallTimeMs(); - } - if (event.getTotalSplitCpuTimeMs() != null) { - thriftEvent.total_split_cpu_time_ms = event.getTotalSplitCpuTimeMs(); - } - if (event.getTotalBytes() != null) { - thriftEvent.total_bytes = event.getTotalBytes(); - } - if (event.getTotalRows() != null) { - thriftEvent.total_rows = event.getTotalRows(); - } - thriftEvent.splits = event.getSplits(); - if (event.getErrorCode() != null) { - thriftEvent.error_code_id = event.getErrorCode(); - } - thriftEvent.error_code_name = event.getErrorCodeName(); - thriftEvent.failure_type = event.getFailureType(); - thriftEvent.failure_message = event.getFailureMessage(); - thriftEvent.failure_task = event.getFailureTask(); - thriftEvent.failure_host = event.getFailureHost(); - thriftEvent.output_stage_json = event.getOutputStageJson(); - thriftEvent.failures_json = event.getFailuresJson(); - thriftEvent.inputs_json = event.getInputsJson(); - thriftEvent.session_properties_json = event.getSessionPropertiesJson(); - thriftEvent.query_wall_time_ms = event.getQueryWallTimeMs(); - if (event.getBytesPerSec() != null) { - thriftEvent.bytes_per_sec = event.getBytesPerSec(); - } - if (event.getBytesPerCpuSec() != null) { - thriftEvent.bytes_per_cpu_sec = event.getBytesPerCpuSec(); - } - if (event.getRowsPerSec() != null) { - thriftEvent.rows_per_sec = event.getRowsPerSec(); - } - if (event.getRowsPerCpuSec() != null) { - thriftEvent.rows_per_cpu_sec = event.getRowsPerCpuSec(); - } - - return thriftEvent; - } -} From f89cc403a0d6705d8d354d1f309307e5d3091b03 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 8 Nov 2016 11:34:20 -0800 Subject: [PATCH 092/151] update version first --- pom.xml | 4 ++-- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 73a2b815da39..c94af68e6215 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.143-tw-0.23 + 0.149 diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 91f462cfadb0..d5b8d1e78be0 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index b3eb4d56e859..c73d27de8a45 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index ca72cfe3d5d3..63198c198edc 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.23 + 0.149 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8aa14bce49fd..b03e8e6acc3c 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 7ade60a4cb0c..03d54389f987 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 990c3da0dba6..97f8ad0d6d2f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 0dbadefd62ae..03c448212c15 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 2d99c244cf0b..58398e0dc6f5 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index e5db3aa84671..b8d82a0b8a7c 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 08404aab6545..8b83af250254 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index d9036e8f3d95..ea8070438a1d 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f9d45157f9fc..72207865f38c 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index ea710e04f016..d305d142df4c 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index f6f9554cc4f1..e14863a1f68e 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index deaf1b6f7ce2..43ba4621569a 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 299cfbd382a3..fc710346266a 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 782d4fc3894f..d1a24c512bea 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 875a03aa63f7..142fcf2dac0c 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-kafka diff --git a/presto-main/pom.xml b/presto-main/pom.xml index c6f9b388ceab..64b6cbeffbf5 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 05a809b78044..1f28da25ceda 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-ml diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 19104889d0e1..d6e339e06e67 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index ff25cd66c973..77e592bab23d 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 0040747083d3..7a80dd13631b 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index c0d7e1b5c67c..b1095676e8d1 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index d3c75dc22d20..9502d7166a81 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.23 + 0.149 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index f6df5f755d74..8a60522ac198 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 85e013e3bc61..fb3c6827e991 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 38200cc68020..fa0916fb9bf1 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index aa2792d3b0d6..f831831e0a5d 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9b4d72aa6ddf..4ca0a9cf7397 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 20dcbb999b82..8cec5d288c1b 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 9f0efb881244..2a24fdc53a19 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 11cf1b90fe7c..c46298046c3b 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 574825fa8a61..2dc252ceb20a 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.143-tw-0.23 + 0.149 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 28dfc515bf06..d8c0c539d4e6 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 430df7fc5da0..dbfa75787fc6 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.143-tw-0.23 + 0.149 presto-verifier From 9b5d00094652c9354fb03e15f9353c6e1497c148 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 8 Nov 2016 14:23:59 -0800 Subject: [PATCH 093/151] use new twitter version --- pom.xml | 4 ++-- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- .../com/facebook/presto/hive/BackgroundHiveSplitLoader.java | 4 ++-- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 41 files changed, 43 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index ae25ca9b6ca3..01e7b2e5eddf 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.149 + 0.149-tw-0.25 diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index d31f9c4d3504..85403124dc54 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d5b8d1e78be0..dbbd0b2fd05e 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c73d27de8a45..6275ebb8259e 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 63198c198edc..706c6b66dbeb 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149 + 0.149-tw-0.25 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index b03e8e6acc3c..132567423856 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 03d54389f987..95bae83cc8a3 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index a74a27be3f8c..1911d10efcf7 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 03c448212c15..56062155bc00 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 58398e0dc6f5..ccf9b9c9eefa 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index b8d82a0b8a7c..0a38ee035f39 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 8b83af250254..2f2c445868a4 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index ea8070438a1d..8cd92ff0172f 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 72207865f38c..d81ea563c4ec 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index d305d142df4c..79da6f2cfa01 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 201d1afe7674..30147c361b34 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 8e40fc0b237d..3dfa747efde6 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-hive diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 6da153d2a898..55fcce7db92a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -41,14 +41,14 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.security.UserGroupInformation; import java.io.BufferedReader; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Deque; import java.util.List; diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index fc710346266a..162187eb1a44 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 72979a0fbfec..18d6ea61e4f9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 0bea79df0b75..7018bfaad555 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index d962fddabf80..0f50c038f3ae 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 2be41fe12a48..0272a3419ecf 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 1f28da25ceda..a80cc82ee392 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 7387f060788c..fccde09ef8e3 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index d6e339e06e67..d7043fd17066 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 7eee85282846..a2135acd8b46 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 3c457be6f418..6914d4063014 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index b1095676e8d1..0eb5d49a788e 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 9f66fc0766f0..b39f43defae3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149 + 0.149-tw-0.25 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 7b84574e3ef9..68a69009a4b6 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index fb3c6827e991..f5be0b5b557c 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index fa0916fb9bf1..c0acace95105 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index f831831e0a5d..b91a02b4d835 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 4ca0a9cf7397..69119bfea46c 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8cec5d288c1b..83f72ae8a5a3 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2a24fdc53a19..f4c18070879f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 355e72d34716..e22cd4d33df3 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index cae36d3a457c..8494dc7183d0 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149 + 0.149-tw-0.25 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index d8c0c539d4e6..5cae2b5c833a 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index dbfa75787fc6..3f7b41d22be8 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149 + 0.149-tw-0.25 presto-verifier From 53dcebe3014bc91f3db8a8c009e55b189f43dcd7 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 8 Nov 2016 15:08:12 -0800 Subject: [PATCH 094/151] avoid java version check for twitter --- .../com/facebook/presto/server/PrestoSystemRequirements.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java index 5bbef6bc4f09..a9066b4cad2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java @@ -45,7 +45,7 @@ public static void verifyJvmRequirements() failRequirement("Presto requires an Oracle or OpenJDK JVM (found %s)", vendor); } - verifyJavaVersion(); + //verifyJavaVersion(); String dataModel = System.getProperty("sun.arch.data.model"); if (!"64".equals(dataModel)) { From 488d786f3a6d9e6638113806d3ea4174bab19fdb Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 25 Oct 2016 16:52:05 -0700 Subject: [PATCH 095/151] Added shuffle for metastores in order to balance connection load --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 83642e41c50f..1885f515848d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import java.util.Collections; import java.util.List; import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; @@ -51,6 +52,7 @@ public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, H public HiveMetastoreClient createMetastoreClient() { List metastores = zkMetastoreMonitor.getServers(); + Collections.shuffle(metastores); TTransportException lastException = null; for (HostAndPort metastore : metastores) { try { From b33652e517b10e19567b89e0893e64e0c586760c Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 27 Oct 2016 17:49:54 -0700 Subject: [PATCH 096/151] Added log for hive connection and log for RetryDriver --- .../src/main/java/com/facebook/presto/hive/RetryDriver.java | 4 +++- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java index d969a522a750..c2c1fac24ffc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java @@ -136,16 +136,18 @@ public V run(String callableName, Callable callable) return callable.call(); } catch (Exception e) { + log.debug("Failed on executing %s with attempt %d, Exception: %s", callableName, attempt, e.getMessage()); e = exceptionMapper.apply(e); for (Class clazz : exceptionWhiteList) { if (clazz.isInstance(e)) { + log.debug("Exception is in whitelist."); throw e; } } if (attempt >= maxAttempts || Duration.nanosSince(startTime).compareTo(maxRetryTime) >= 0) { + log.debug("Maximum attempts or maximum retry time reached. attempt: %d, maxAttempts: %d, duration: [%s] maxRetryTime: [%s]", attempt, maxAttempts, Duration.nanosSince(startTime).toString(), maxRetryTime.toString()); throw e; } - log.debug("Failed on executing %s with attempt %d, will retry. Exception: %s", callableName, attempt, e.getMessage()); int delayInMs = (int) Math.min(minSleepTime.toMillis() * Math.pow(scaleFactor, attempt - 1), maxSleepTime.toMillis()); TimeUnit.MILLISECONDS.sleep(delayInMs); diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 1885f515848d..1f1b6cad01a4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -60,10 +60,11 @@ public HiveMetastoreClient createMetastoreClient() return clientFactory.create(metastore.getHostText(), metastore.getPort()); } catch (TTransportException e) { + log.debug("Failed connecting to Hive metastore at: " + metastore.toString()); lastException = e; } } - throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore", lastException); + throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore: " + metastores, lastException); } } From c43856986a1be1b148c602ebf41b7bfd9b77795a Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 31 Oct 2016 15:45:52 -0700 Subject: [PATCH 097/151] Used RuntimeException to replace PrestoException, prevent whitelist stops RetryDriver --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 1f1b6cad01a4..ca684b269cf5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -16,7 +16,6 @@ import com.facebook.presto.hive.HiveCluster; import com.facebook.presto.hive.HiveMetastoreClientFactory; import com.facebook.presto.hive.metastore.HiveMetastoreClient; -import com.facebook.presto.spi.PrestoException; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.apache.thrift.transport.TTransportException; @@ -26,7 +25,6 @@ import java.util.Collections; import java.util.List; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static java.util.Objects.requireNonNull; public class ZookeeperServersetHiveCluster @@ -65,6 +63,6 @@ public HiveMetastoreClient createMetastoreClient() } } - throw new PrestoException(HIVE_METASTORE_ERROR, "Failed connecting to Hive metastore: " + metastores, lastException); + throw new RuntimeException("Failed connecting to Hive metastore: " + metastores, lastException); } } From 43eff7d14a0fa11ef19bd526100b9216131cc664 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 31 Oct 2016 18:22:10 -0700 Subject: [PATCH 098/151] Used formatter for logs --- .../presto/twitter/hive/ZookeeperServersetHiveCluster.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index ca684b269cf5..379352d4100e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -54,15 +54,15 @@ public HiveMetastoreClient createMetastoreClient() TTransportException lastException = null; for (HostAndPort metastore : metastores) { try { - log.info("Connecting to metastore at: " + metastore.toString()); + log.info("Connecting to metastore at: %s", metastore.toString()); return clientFactory.create(metastore.getHostText(), metastore.getPort()); } catch (TTransportException e) { - log.debug("Failed connecting to Hive metastore at: " + metastore.toString()); + log.debug("Failed connecting to Hive metastore at: %s", metastore.toString()); lastException = e; } } - throw new RuntimeException("Failed connecting to Hive metastore: " + metastores, lastException); + throw new RuntimeException("Failed connecting to Hive metastore.", lastException); } } From abe1d34774ac0d9e387e9ab24437d332a3b06031 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 18 Nov 2016 14:15:40 -0800 Subject: [PATCH 099/151] Fix bug for tableau webapp --- .../src/main/resources/webapp/tableau/presto-connector.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-main/src/main/resources/webapp/tableau/presto-connector.html b/presto-main/src/main/resources/webapp/tableau/presto-connector.html index 3b915fdfb4b2..f4a48c6466f9 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-connector.html +++ b/presto-main/src/main/resources/webapp/tableau/presto-connector.html @@ -2,8 +2,8 @@ - - + + + diff --git a/presto-main/src/main/resources/webapp/vendor/tableau/tableauwdc-1.1.0.js b/presto-main/src/main/resources/webapp/vendor/tableau/tableauwdc-1.1.0.js new file mode 100644 index 000000000000..11e72f9288e0 --- /dev/null +++ b/presto-main/src/main/resources/webapp/vendor/tableau/tableauwdc-1.1.0.js @@ -0,0 +1,202 @@ +//The MIT License (MIT) +// +//Copyright (c) 2015 Tableau +// +//Permission is hereby granted, free of charge, to any person obtaining a copy +//of this software and associated documentation files (the "Software"), to deal +//in the Software without restriction, including without limitation the rights +//to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +//copies of the Software, and to permit persons to whom the Software is +//furnished to do so, subject to the following conditions: +// +//The above copyright notice and this permission notice shall be included in all +//copies or substantial portions of the Software. +// +//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +//SOFTWARE. +(function() { + + var versionNumber = "1.1.0"; + var _sourceWindow; + + if (typeof tableauVersionBootstrap === 'undefined') { + // tableau version bootstrap isn't defined. We are likely running in the simulator so init up our tableau object + tableau = { + connectionName: "", + connectionData: "", + password: "", + username: "", + incrementalExtractColumn: "", + + initCallback: function () { + _sendMessage("initCallback"); + }, + + shutdownCallback: function () { + _sendMessage("shutdownCallback"); + }, + + submit: function () { + _sendMessage("submit"); + }, + + log: function (msg) { + _sendMessage("log", {"logMsg": msg}); + }, + + headersCallback: function (fieldNames, types) { + _sendMessage("headersCallback", {"fieldNames": fieldNames, "types":types}); + }, + + dataCallback: function (data, lastRecordToken, moreData) { + _sendMessage("dataCallback", {"data": data, "lastRecordToken": lastRecordToken, "moreData": moreData}); + }, + + abortWithError: function (errorMsg) { + _sendMessage("abortWithError", {"errorMsg": errorMsg}); + } + }; + } else { // Tableau version bootstrap is defined. Let's use it + tableauVersionBootstrap.ReportVersionNumber(versionNumber); + } + + // Check if something weird happened during bootstraping. If so, just define a tableau object to we don't + // throw errors all over the place because tableau isn't defined + if (typeof tableau === "undefined") { + tableau = {} + } + + tableau.versionNumber = versionNumber; + + tableau.phaseEnum = { + interactivePhase: "interactive", + authPhase: "auth", + gatherDataPhase: "gatherData" + }; + + if (!tableau.phase) { + tableau.phase = tableau.phaseEnum.interactivePhase; + } + + // Assign the functions we always want to have available on the tableau object + tableau.makeConnector = function() { + var defaultImpls = { + init: function() { tableau.initCallback(); }, + shutdown: function() { tableau.shutdownCallback(); } + }; + return defaultImpls; + }; + + tableau.registerConnector = function (wdc) { + // do some error checking on the wdc + var functionNames = ["init", "shutdown", "getColumnHeaders", "getTableData"] + for (var ii = functionNames.length - 1; ii >= 0; ii--) { + if (typeof(wdc[functionNames[ii]]) !== "function") { + throw "The connector did not define the required function: " + functionNames[ii]; + } + }; + window._wdc = wdc; + }; + + function _sendMessage(msgName, msgData) { + var messagePayload = _buildMessagePayload(msgName, msgData); + + // Check first to see if we have a messageHandler defined to post the message to + if (typeof window.webkit != 'undefined' && + typeof window.webkit.messageHandlers != 'undefined' && + typeof window.webkit.messageHandlers.wdcHandler != 'undefined') { + + window.webkit.messageHandlers.wdcHandler.postMessage(messagePayload); + } else if (!_sourceWindow) { + throw "Looks like the WDC is calling a tableau function before tableau.init() has been called." + } else { + _sourceWindow.postMessage(messagePayload, "*"); + } + } + + function _buildMessagePayload(msgName, msgData) { + var msgObj = {"msgName": msgName, + "props": _packagePropertyValues(), + "msgData": msgData}; + return JSON.stringify(msgObj); + } + + function _packagePropertyValues() { + var propValues = {"connectionName": tableau.connectionName, + "connectionData": tableau.connectionData, + "password": tableau.password, + "username": tableau.username, + "incrementalExtractColumn": tableau.incrementalExtractColumn, + "versionNumber": tableau.versionNumber}; + return propValues; + } + + function _applyPropertyValues(props) { + if (props) { + tableau.connectionName = props.connectionName; + tableau.connectionData = props.connectionData; + tableau.password = props.password; + tableau.username = props.username; + tableau.incrementalExtractColumn = props.incrementalExtractColumn; + } + } + + function _receiveMessage(evnt) { + var wdc = window._wdc; + if (!wdc) { + throw "No WDC registered. Did you forget to call tableau.registerConnector?"; + } + if (!_sourceWindow) { + _sourceWindow = evnt.source + } + var payloadObj = JSON.parse(evnt.data); + var msgData = payloadObj.msgData; + _applyPropertyValues(payloadObj.props); + + switch(payloadObj.msgName) { + case "init": + tableau.phase = msgData.phase; + wdc.init(); + break; + case "shutdown": + wdc.shutdown(); + break; + case "getColumnHeaders": + wdc.getColumnHeaders(); + break; + case "getTableData": + wdc.getTableData(msgData.lastRecordToken); + break; + } + }; + + // Add global error handler. If there is a javascript error, this will report it to Tableau + // so that it can be reported to the user. + window.onerror = function (message, file, line, column, errorObj) { + if (tableau._hasAlreadyThrownErrorSoDontThrowAgain) { + return true; + } + var msg = message; + if(errorObj) { + msg += " stack:" + errorObj.stack; + } else { + msg += " file: " + file; + msg += " line: " + line; + } + + if (tableau && tableau.abortWithError) { + tableau.abortWithError(msg); + } else { + throw msg; + } + tableau._hasAlreadyThrownErrorSoDontThrowAgain = true; + return true; + } + + window.addEventListener('message', _receiveMessage, false); +})(); \ No newline at end of file From 35f71d297105d707c608160ac3753dae9126133e Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 22 Nov 2016 14:33:52 -0800 Subject: [PATCH 102/151] Undo erroneous merge of commit 36175ec6d47d24ff3d7519f68d3e5c0d0af0fc98 which removed the Twitter module that scribes QueryCompletionEvent --- pom.xml | 68 +++++++ presto-main/pom.xml | 18 ++ .../operator/.HttpPageBufferClient.java.swp | Bin 0 -> 16384 bytes .../facebook/presto/server/PrestoServer.java | 6 + .../presto/twitter/TwitterModuleLoader.java | 64 +++++++ .../presto/twitter/logging/QueryLogger.java | 78 ++++++++ .../presto/twitter/logging/QueryScriber.java | 169 ++++++++++++++++++ 7 files changed, 403 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java create mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java diff --git a/pom.xml b/pom.xml index 4b36010586f6..574f605cb947 100644 --- a/pom.xml +++ b/pom.xml @@ -761,6 +761,74 @@ 2.1.7 + + + com.twitter + presto-thrift-java + 0.0.1 + + + com.twitter + util-core_2.11 + + + com.twitter + util-core-java + + + com.twitter + util-function_2.10 + + + com.twitter + util-function-java + + + commons-logging + commons-logging + + + org.scala-lang.modules + scala-parser-combinators_2.11 + + + com.twitter + scrooge-core + + + org.scala-lang + scala-library + + + org.scala-lang + scala-reflect + + + + + + com.twitter + util-logging_2.10 + 6.34.0 + + + commons-logging + commons-logging + + + + + org.scala-lang + scala-library + 2.10.6 + + + commons-logging + commons-logging + + + + diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 94b5a1b08642..d244d702d5a5 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -252,6 +252,24 @@ jgrapht-core + + + com.twitter + presto-thrift-java + + + com.twitter + util-logging_2.10 + + + org.apache.thrift + libthrift + + + org.scala-lang + scala-library + + org.testng diff --git a/presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp b/presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp new file mode 100644 index 0000000000000000000000000000000000000000..4527e97497ec4ac48a06fca9116b4c57bdad80d0 GIT binary patch literal 16384 zcmeHN%Wot{8E;quAtVG5ffNZuVJ_Za*DML)R)E{KE!2oQ&`Z~`PwhyxN*A|Y`|{s2I5Ah20})sNYE?RZu!7q(jZ zP4`rN_0?CuuO44@SNGBtXODcEK4al}pJjdb_WHHQKk)p8KYh!x)R7#pG}So^FEv~r zbmR4N#lKcc)P^!SPGjctTAc7y$?~>7U?QreiC@bfZm<(p^W~t{gJf6A<4Rh_JYh;E zwH>A6eb(ojS+B>FMksiss4lk9T$5=Sct9C=%zEbOPh8TC|Jce$$rJBCctCkD%{B}e z1`Gp+0mFb{z%XDKFbq7H45;{F>v=T&?Pc@#%KO`GHFkl!k3>XIfFBxEgWnDqbUeX(7eE+ZZ|F<5utX}{pz%H-@ zd=Pj7_}zOg>l?rZ@b||o>kq*1fgb=b178O^z>~mV9<{9B0xtp)@GNivcnurz?*Iv~ z1$-Fz$0Lvhz6nU+^T3CIH{WepuL3uLp93!fN5E%+&j3#Xw;#5w*MVOFKLoxF41sIF zCxN%#1s%Xkzz{eAE&+dgr)7N)_#*Hq@auhA90p5yM-y~fV+y3$zao)iTv3G5`1Nkayl&eOS!Wec@! z9&B%;t&L{ewGX%JPSf66MrU;>D0lhHp&*?w8XDA3IJ)sf+W)4CyuaLYJ>_`b&wE1E zd-prG>$MC2IW5<#w;J}LvlKOp^zGL#qf@kZ?bb33oU5s2U&8>qAL{n%*A89h3v);6 z^k7+JR@I%_d;3k>TS{Fwl>RZz-RANH{T$sT$p0?GQQ@{d9th^0#TdM$=QPu9Ir&tx zus*9+40T^dewHNq02_|B4TMkY`K>9V{;1aT#hpm;{@4sNw}dAK{2&r{rs26#u7AXX zEaZWG%KZ$-?ZCx}@bAq2b4zV!3NzLmi`%ghGGgIf8dPPY0dopBMLLzNC-;MnGbxLO z(TUhs*1nY>0QQ_i~KR0E*5IAX|yO6j-Ite=$NHE1*@uj&s(hwbhy81UKdY+a-X3P(Z#Y5SrP1zssv|~n($2V0_;z`+L zNuSTU@tlHXNHovrdG(wxm&Z5fiRlpJAQ5F0w2X8SvMnRoC?Y>yMa9n40K5w&x$`C4&q!D4xs?Yr9cc@^n(XT!2^ zTKX7IWS&?R`fOO_sTx2+F%ZRcl#(n$@JrMYCryl6Yz(NDa5*1OBRyHjq%rqJ4||M| zMg2@y*q0|fiO@WP$aNsn6q83v5D98ER~>Jsec+L+^-ibW@*LYG?G8bpz2$gLyM=$- zq~5wtcAeJN8sP$FB!r*p7&j%T5}Oh9;2<|Cz#N~5Zl-uJYcvrl85fWdOducHdBv8~nBP>Q8zIHzvHPp4Te!`KbBoPrACPJZgYb2FDHN^e| zYP2SO=~>}(wc2nvqzn!?;uO4w9wX8)#xrz)GGkPuey(8@98^=VS=(#>l^z}EtDf6g`Ff39 z!OjOBBe0RI>Rzx?7FxNuVbz|p^!{I9AOBN8@Bgt4&#ymYpZ`bTN5Bh!3w#8419HCs zZUR38bh)1Z<}wTz1`Gp+0mFb{z%XDKFbo(53 vX7s$MfGUOa`=nsP@uEG#ON{X^i`e0d^p4z@NA|~A7q2ZyAv-FxdeQn1&2=?# literal 0 HcmV?d00001 diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 3aa94c7f9317..7e1bbcd031c3 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -103,6 +103,7 @@ public void run() new ServerMainModule(sqlParserOptions), new GracefulShutdownModule()); + modules.addAll(getAdditionalModules()); Bootstrap app = new Bootstrap(modules.build()); try { @@ -131,6 +132,11 @@ public void run() } } + protected Iterable getAdditionalModules() + { + return com.facebook.presto.twitter.TwitterModuleLoader.getAdditionalModules(); + } + private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) { // get existing announcement diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java new file mode 100644 index 000000000000..eb23a7650a09 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter; + +import com.facebook.presto.event.EventProcessor; +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryCreatedEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.event.query.SplitCompletionEvent; +import com.facebook.presto.twitter.logging.QueryLogger; +import com.facebook.presto.twitter.logging.QueryScriber; +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.Multibinder; +import io.airlift.event.client.EventClient; + +/** + * Loader that initializes custom Twitter code to inject into Presto. Whenever + * possible we should use this pattern to inject custom functionality, since it + * makes it easier to differentiate our patches from the core OS code. + * + * If the functionality we wish to add/override isn't currently possible to via + * overriding a guice module, we should contribute the necessary modules/interfaces + * into the OS Presto code base to make it possible. + */ +public class TwitterModuleLoader +{ + private TwitterModuleLoader() + { + } + + public static Iterable getAdditionalModules() + { + return ImmutableList.of( + binder -> Multibinder.newSetBinder(binder, EventClient.class) + .addBinding() + .to(EventProcessor.class) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(new TypeLiteral(){}) + .in(Scopes.SINGLETON), + binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) + .addBinding() + .to(QueryScriber.class) + .in(Scopes.SINGLETON) + ); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java new file mode 100644 index 000000000000..02edd8da2440 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.facebook.presto.spi.StandardErrorCode; +import io.airlift.log.Logger; +import io.airlift.units.Duration; + +import java.util.concurrent.TimeUnit; + +/** + * Class that logs query events to a file + */ +public class QueryLogger implements QueryEventHandler +{ + private static final int MAX_QUERY_LENGTH = 1000; + private static final String DASH = "-"; + private static final String COLON = ":"; + private static final String SPACE = " "; + private static final String ELIPSIS = "..."; + private static final String QUERY_COMPLETION = "QueryCompletion"; + + private static final Logger log = Logger.get(QueryLogger.class); + + @Override + public void handle(QueryCompletionEvent event) + { + String errorType = DASH; + String errorCode = DASH; + if (event.getErrorCode() != null) { + errorType = StandardErrorCode.toErrorType(event.getErrorCode()).toString(); + if (event.getErrorCodeName() != null) { + errorCode = event.getErrorCodeName() + COLON + event.getErrorCode(); + } + } + + Duration duration = (new Duration( + event.getQueryWallTimeMs(), TimeUnit.MILLISECONDS)) + .convertToMostSuccinctTimeUnit(); + + log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", + QUERY_COMPLETION, event.getQueryId(), toLogValue(event.getRemoteClientAddress()), + event.getQueryState(), errorType, errorCode, event.getUser(), duration, + event.getSplits(), event.getTotalRows(), event.getTotalBytes(), + cleanseAndTrimQuery(event.getQuery()))); + } + + private static String toLogValue(Object object) + { + if (object == null) { + return DASH; + } + else { + return object.toString(); + } + } + + private static String cleanseAndTrimQuery(String query) + { + if (query.length() > MAX_QUERY_LENGTH) { + query = query.substring(0, MAX_QUERY_LENGTH) + ELIPSIS; + } + return query.replace(System.getProperty("line.separator"), SPACE); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java new file mode 100644 index 000000000000..5e55aa1ac195 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.logging; + +import com.facebook.presto.event.query.QueryCompletionEvent; +import com.facebook.presto.event.query.QueryEventHandler; +import com.google.common.base.Optional; +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; +import com.twitter.presto.thriftjava.QueryState; +import io.airlift.log.Logger; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.logging.LogRecord; + +/** + * Class that scribes query completion events + */ +public class QueryScriber implements QueryEventHandler +{ + private static final String SCRIBE_CATEGORY = "presto_query_completion"; + private static final int MAX_QUEUE_SIZE = 1000; + + private static final Logger log = Logger.get(QueryScriber.class); + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() + { + @Override protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public QueryScriber() + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + SCRIBE_CATEGORY, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + @Override + public void handle(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); + Optional message = serializeThriftToString(thriftEvent); + + if (message.isPresent()) { + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); + } + else { + log.warn("Unable to serialize QueryCompletionEvent: " + event); + } + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + */ + private Optional serializeThriftToString(TBase thriftMessage) + { + try { + return Optional.of( + Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); + } + catch (TException e) { + log.warn(e, "Could not serialize thrift object" + thriftMessage); + return Optional.absent(); + } + } + + private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) + { + com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); + + thriftEvent.query_id = event.getQueryId(); + thriftEvent.transaction_id = event.getTransactionId(); + thriftEvent.user = event.getUser(); + thriftEvent.principal = event.getPrincipal(); + thriftEvent.source = event.getSource(); + thriftEvent.server_version = event.getServerVersion(); + thriftEvent.environment = event.getEnvironment(); + thriftEvent.catalog = event.getCatalog(); + thriftEvent.schema = event.getSchema(); + thriftEvent.remote_client_address = event.getRemoteClientAddress(); + thriftEvent.user_agent = event.getUserAgent(); + thriftEvent.query_state = QueryState.valueOf(event.getQueryState()); + thriftEvent.uri = event.getUri(); + thriftEvent.field_names = event.getFieldNames(); + thriftEvent.query = event.getQuery(); + thriftEvent.create_time_ms = event.getCreateTime().getMillis(); + thriftEvent.execution_start_time_ms = event.getExecutionStartTime().getMillis(); + thriftEvent.end_time_ms = event.getEndTime().getMillis(); + thriftEvent.queued_time_ms = event.getQueuedTimeMs(); + if (event.getAnalysisTimeMs() != null) { + thriftEvent.analysis_time_ms = event.getAnalysisTimeMs(); + } + if (event.getDistributedPlanningTimeMs() != null) { + thriftEvent.distributed_planning_time_ms = event.getDistributedPlanningTimeMs(); + } + if (event.getTotalSplitWallTimeMs() != null) { + thriftEvent.total_split_wall_time_ms = event.getTotalSplitWallTimeMs(); + } + if (event.getTotalSplitCpuTimeMs() != null) { + thriftEvent.total_split_cpu_time_ms = event.getTotalSplitCpuTimeMs(); + } + if (event.getTotalBytes() != null) { + thriftEvent.total_bytes = event.getTotalBytes(); + } + if (event.getTotalRows() != null) { + thriftEvent.total_rows = event.getTotalRows(); + } + thriftEvent.splits = event.getSplits(); + if (event.getErrorCode() != null) { + thriftEvent.error_code_id = event.getErrorCode(); + } + thriftEvent.error_code_name = event.getErrorCodeName(); + thriftEvent.failure_type = event.getFailureType(); + thriftEvent.failure_message = event.getFailureMessage(); + thriftEvent.failure_task = event.getFailureTask(); + thriftEvent.failure_host = event.getFailureHost(); + thriftEvent.output_stage_json = event.getOutputStageJson(); + thriftEvent.failures_json = event.getFailuresJson(); + thriftEvent.inputs_json = event.getInputsJson(); + thriftEvent.session_properties_json = event.getSessionPropertiesJson(); + thriftEvent.query_wall_time_ms = event.getQueryWallTimeMs(); + if (event.getBytesPerSec() != null) { + thriftEvent.bytes_per_sec = event.getBytesPerSec(); + } + if (event.getBytesPerCpuSec() != null) { + thriftEvent.bytes_per_cpu_sec = event.getBytesPerCpuSec(); + } + if (event.getRowsPerSec() != null) { + thriftEvent.rows_per_sec = event.getRowsPerSec(); + } + if (event.getRowsPerCpuSec() != null) { + thriftEvent.rows_per_cpu_sec = event.getRowsPerCpuSec(); + } + + return thriftEvent; + } +} From 30630793e4ad467a2f9aafef4dc8ec64417a12a0 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 22 Nov 2016 14:35:59 -0800 Subject: [PATCH 103/151] update tag to 0.149-tw-0.27 --- pom.xml | 4 ++-- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 40 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pom.xml b/pom.xml index 574f605cb947..c31176988c59 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.149-tw-0.26 + 0.149-tw-0.27 diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 86a265fda82a..09465ba6c003 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 2341ce4f1b56..0339a4e61b2b 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 571824609031..7e26a4b4544a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index d676a6d61541..1bce4669b733 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.26 + 0.149-tw-0.27 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8102a09248eb..8f5e60ab8f67 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 4c5d0cd8499e..147e19a1d0be 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index f7de6cf287f6..8fca2daa6285 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index ee11160dd65b..4ce75a91f29e 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index ac42c8769498..fc675ad793c1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 1f41107e5199..f1e97a471056 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index f824320b74da..e3a043887564 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 372d9932f9a4..56a91e857267 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index a60ed7801af2..14232f3a3b5b 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index efca94d80ce5..636a011ee5e1 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 77b4e787fabf..298456be757c 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f32f810c7d41..f3b0ec562a81 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index ba717e41d02f..88e5f9bb334e 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 5d50049084fd..13f8dd9ec3f9 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index c9309a2fae26..b644db550652 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 02179d1d9e7c..8803cdbc9630 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index d244d702d5a5..02c1d2323f44 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 6a88d759675a..0260655c0298 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 73262f63ffaf..28d973ed89c7 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 7f90f9b8282a..5e407459011c 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 0efc7dac4577..30cc3fa661f8 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a20ab814f118..4ba0e816b0f5 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 40ece3041bcb..de3ff795dbec 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index e337e6cc6f01..9a9e22b52ad1 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.26 + 0.149-tw-0.27 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 357ae8bcc93d..95c4461e9845 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index dce3b54ffd2f..c7439b37d553 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 8d11658416aa..1ccdab8fce4c 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index f4ff91a09ca5..b471f8b603f2 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index dc787efaf891..6ceae8a36b78 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index f87acdfc90a1..e26987fdaf37 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2ac12cb7f520..6287071b0c71 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 58a857dfbfa6..46b9e6f1bcc4 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 9b673a003d44..54849774f6c2 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.26 + 0.149-tw-0.27 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 1e78b6278ed6..278763709289 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index ca874156cb9a..50e222cdc4fb 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.26 + 0.149-tw-0.27 presto-verifier From 5f21b8269ba4a9a180081f872db5c0af3a323b54 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Tue, 22 Nov 2016 15:08:13 -0800 Subject: [PATCH 104/151] take out swap file --- .../operator/.HttpPageBufferClient.java.swp | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp diff --git a/presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp b/presto-main/src/main/java/com/facebook/presto/operator/.HttpPageBufferClient.java.swp deleted file mode 100644 index 4527e97497ec4ac48a06fca9116b4c57bdad80d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHN%Wot{8E;quAtVG5ffNZuVJ_Za*DML)R)E{KE!2oQ&`Z~`PwhyxN*A|Y`|{s2I5Ah20})sNYE?RZu!7q(jZ zP4`rN_0?CuuO44@SNGBtXODcEK4al}pJjdb_WHHQKk)p8KYh!x)R7#pG}So^FEv~r zbmR4N#lKcc)P^!SPGjctTAc7y$?~>7U?QreiC@bfZm<(p^W~t{gJf6A<4Rh_JYh;E zwH>A6eb(ojS+B>FMksiss4lk9T$5=Sct9C=%zEbOPh8TC|Jce$$rJBCctCkD%{B}e z1`Gp+0mFb{z%XDKFbq7H45;{F>v=T&?Pc@#%KO`GHFkl!k3>XIfFBxEgWnDqbUeX(7eE+ZZ|F<5utX}{pz%H-@ zd=Pj7_}zOg>l?rZ@b||o>kq*1fgb=b178O^z>~mV9<{9B0xtp)@GNivcnurz?*Iv~ z1$-Fz$0Lvhz6nU+^T3CIH{WepuL3uLp93!fN5E%+&j3#Xw;#5w*MVOFKLoxF41sIF zCxN%#1s%Xkzz{eAE&+dgr)7N)_#*Hq@auhA90p5yM-y~fV+y3$zao)iTv3G5`1Nkayl&eOS!Wec@! z9&B%;t&L{ewGX%JPSf66MrU;>D0lhHp&*?w8XDA3IJ)sf+W)4CyuaLYJ>_`b&wE1E zd-prG>$MC2IW5<#w;J}LvlKOp^zGL#qf@kZ?bb33oU5s2U&8>qAL{n%*A89h3v);6 z^k7+JR@I%_d;3k>TS{Fwl>RZz-RANH{T$sT$p0?GQQ@{d9th^0#TdM$=QPu9Ir&tx zus*9+40T^dewHNq02_|B4TMkY`K>9V{;1aT#hpm;{@4sNw}dAK{2&r{rs26#u7AXX zEaZWG%KZ$-?ZCx}@bAq2b4zV!3NzLmi`%ghGGgIf8dPPY0dopBMLLzNC-;MnGbxLO z(TUhs*1nY>0QQ_i~KR0E*5IAX|yO6j-Ite=$NHE1*@uj&s(hwbhy81UKdY+a-X3P(Z#Y5SrP1zssv|~n($2V0_;z`+L zNuSTU@tlHXNHovrdG(wxm&Z5fiRlpJAQ5F0w2X8SvMnRoC?Y>yMa9n40K5w&x$`C4&q!D4xs?Yr9cc@^n(XT!2^ zTKX7IWS&?R`fOO_sTx2+F%ZRcl#(n$@JrMYCryl6Yz(NDa5*1OBRyHjq%rqJ4||M| zMg2@y*q0|fiO@WP$aNsn6q83v5D98ER~>Jsec+L+^-ibW@*LYG?G8bpz2$gLyM=$- zq~5wtcAeJN8sP$FB!r*p7&j%T5}Oh9;2<|Cz#N~5Zl-uJYcvrl85fWdOducHdBv8~nBP>Q8zIHzvHPp4Te!`KbBoPrACPJZgYb2FDHN^e| zYP2SO=~>}(wc2nvqzn!?;uO4w9wX8)#xrz)GGkPuey(8@98^=VS=(#>l^z}EtDf6g`Ff39 z!OjOBBe0RI>Rzx?7FxNuVbz|p^!{I9AOBN8@Bgt4&#ymYpZ`bTN5Bh!3w#8419HCs zZUR38bh)1Z<}wTz1`Gp+0mFb{z%XDKFbo(53 vX7s$MfGUOa`=nsP@uEG#ON{X^i`e0d^p4z@NA|~A7q2ZyAv-FxdeQn1&2=?# From 1613f3c252fa2a26c3bfd9e357a760c41555f8cd Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Wed, 30 Nov 2016 22:34:24 -0800 Subject: [PATCH 105/151] update tag to 0.157 to avoid unnecessary conflicts --- pom.xml | 4 ++-- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- 40 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pom.xml b/pom.xml index c31176988c59..a83b82f86e06 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.149-tw-0.27 + 0.157 diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 09465ba6c003..aba5eb525704 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 0339a4e61b2b..f8dbdf7d1b95 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 7e26a4b4544a..6fddcde5e32d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 1bce4669b733..ed9d3f7ad152 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8f5e60ab8f67..4bb0688136f1 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 147e19a1d0be..328239356ba4 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 8fca2daa6285..945ddbb8ddeb 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 4ce75a91f29e..0298db60e938 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index fc675ad793c1..04211daf674c 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index f1e97a471056..c2dd2d1c4615 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index e3a043887564..0e9b8fa8278c 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 56a91e857267..80db7ffef80e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 14232f3a3b5b..3dcb24fedbf5 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 636a011ee5e1..ec1d70ba482b 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 298456be757c..807618da781d 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f3b0ec562a81..2b9c0c165a6a 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 88e5f9bb334e..e9318e30acae 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 13f8dd9ec3f9..4eeaf29519af 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index b644db550652..77e40b94d371 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 8803cdbc9630..0ef69e5e32a6 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 02c1d2323f44..cf88c221ca21 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 0260655c0298..db2c2ce25228 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 28d973ed89c7..365fcc80b528 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 5e407459011c..26d655a500af 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 30cc3fa661f8..af785987e76f 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 4ba0e816b0f5..6d68e76e7beb 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-parser diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index de3ff795dbec..b534f0672eb3 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 9a9e22b52ad1..be29bb33055c 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 95c4461e9845..f7f7825a51af 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-raptor diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index c7439b37d553..80619687049e 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 1ccdab8fce4c..df7bba38b1d6 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-redis diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index b471f8b603f2..ad773d1d327a 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 6ceae8a36b78..bcea4da80b02 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index e26987fdaf37..c9913e9ea95a 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 6287071b0c71..38533a5d3c8e 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 46b9e6f1bcc4..b664281984a8 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 54849774f6c2..c6bbeb2d5d08 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 278763709289..10c39761f7b4 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 50e222cdc4fb..8c2cf5adabdc 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157 presto-verifier From 50b4884e4944bff8e18db773c1f5e7ab9bb9018d Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 1 Dec 2016 13:49:43 -0800 Subject: [PATCH 106/151] update tag --- pom.xml | 5 +++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 49 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 0701f6ac3b43..77381e4f9179 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157 + 0.157-tw-0.28 @@ -91,6 +91,7 @@ presto-hive-cdh5 presto-teradata-functions presto-example-http + twitter-eventlistener-plugin presto-local-file presto-tpch presto-raptor diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 3b26ff81741e..c626c3ad70b2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index 8ccf2972c75f..a500d24dd11f 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 75242cfb9a05..8761d3638bc6 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 1d806e5922c5..73936549aa05 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 6fddcde5e32d..4607b9c36eb8 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 38f595629841..d7b20168f9d9 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157 + 0.157-tw-0.28 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index c24904ad5f70..8ccb6e9fab7a 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 328239356ba4..e5c8a761868a 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 9ae5ae5a199b..f49684f0083f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 0298db60e938..e2e5a57835bf 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 04211daf674c..4f53264a9894 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 15d4926794b6..0cffa89f962b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index d1d735d3236a..533ffeda6853 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 3c60601e9c1c..2af0998f777e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index cf758f778024..03c62da1a4db 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 56c448a6db72..525324657f8d 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index af0312d7c184..7dfdfd71281c 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 9ee04fa53548..94c8f9f305e7 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index b49325fc1412..059491bbb6f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 927a48fafca0..68f97938c645 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 242720afd38b..e1d4ffd5735a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 319fcc74d9b2..9c387bfc64c5 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index f37725e8a11c..93da37fd6fc6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 69e9cf92500a..008c82fdbe20 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index f499be4b25cb..7087a7cdab75 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index dbda79095e31..874270eb3e39 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index af785987e76f..40c93e82bd6e 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 6d68e76e7beb..a73eefd181a2 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index 47396fe8eaf2..e3e52ccc9c94 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 4abebd8ebff1..7d6c08ae0f32 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 086ea98b80d0..699d996b9cf9 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157 + 0.157-tw-0.28 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 48885f10e414..fc953510fc6e 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index dcaa2b1a50f5..b550cf7cfe49 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 499cb961d1ac..caa9be15c786 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index c788b17af12c..33eabb69a5fa 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index cff6840b7dca..0dec7412e1e5 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 0417661668dd..dedf4daf4cb8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index bcea4da80b02..51b1da095a93 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8e4f18a6e8eb..8bb450c60706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 469d5c15dda9..2b9ec4feb53f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index b664281984a8..8ef0f28df850 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 594e2dc3ef03..0923fa884822 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157 + 0.157-tw-0.28 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 81fe59b71edc..9482bda29427 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index b17d6060f7f1..539a9fc3f03f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157 + 0.157-tw-0.28 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index 2dfb997a2475..a38d41663d1f 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.154-tw-0.24 + 0.157-tw-0.28 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.154-tw-0.24 + 0.157-tw-0.28 provided From 56fc85793630d262cddc97986bdb6dc410112b51 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 1 Dec 2016 13:59:49 -0800 Subject: [PATCH 107/151] remove current impl of twitter event scriber - upgrade to 0.157 part 1 of 3 --- pom.xml | 68 ------- presto-main/pom.xml | 18 -- .../facebook/presto/server/PrestoServer.java | 6 - .../presto/twitter/TwitterModuleLoader.java | 64 ------- .../presto/twitter/logging/QueryLogger.java | 78 -------- .../presto/twitter/logging/QueryScriber.java | 169 ------------------ 6 files changed, 403 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java diff --git a/pom.xml b/pom.xml index c31176988c59..8057e9ac0c7f 100644 --- a/pom.xml +++ b/pom.xml @@ -761,74 +761,6 @@ 2.1.7 - - - com.twitter - presto-thrift-java - 0.0.1 - - - com.twitter - util-core_2.11 - - - com.twitter - util-core-java - - - com.twitter - util-function_2.10 - - - com.twitter - util-function-java - - - commons-logging - commons-logging - - - org.scala-lang.modules - scala-parser-combinators_2.11 - - - com.twitter - scrooge-core - - - org.scala-lang - scala-library - - - org.scala-lang - scala-reflect - - - - - - com.twitter - util-logging_2.10 - 6.34.0 - - - commons-logging - commons-logging - - - - - org.scala-lang - scala-library - 2.10.6 - - - commons-logging - commons-logging - - - - diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 02c1d2323f44..7fa35421c891 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -252,24 +252,6 @@ jgrapht-core - - - com.twitter - presto-thrift-java - - - com.twitter - util-logging_2.10 - - - org.apache.thrift - libthrift - - - org.scala-lang - scala-library - - org.testng diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 7e1bbcd031c3..3aa94c7f9317 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -103,7 +103,6 @@ public void run() new ServerMainModule(sqlParserOptions), new GracefulShutdownModule()); - modules.addAll(getAdditionalModules()); Bootstrap app = new Bootstrap(modules.build()); try { @@ -132,11 +131,6 @@ public void run() } } - protected Iterable getAdditionalModules() - { - return com.facebook.presto.twitter.TwitterModuleLoader.getAdditionalModules(); - } - private static void updateDatasources(Announcer announcer, Metadata metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) { // get existing announcement diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java b/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java deleted file mode 100644 index eb23a7650a09..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/TwitterModuleLoader.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter; - -import com.facebook.presto.event.EventProcessor; -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryCreatedEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.facebook.presto.event.query.SplitCompletionEvent; -import com.facebook.presto.twitter.logging.QueryLogger; -import com.facebook.presto.twitter.logging.QueryScriber; -import com.google.common.collect.ImmutableList; -import com.google.inject.Module; -import com.google.inject.Scopes; -import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.Multibinder; -import io.airlift.event.client.EventClient; - -/** - * Loader that initializes custom Twitter code to inject into Presto. Whenever - * possible we should use this pattern to inject custom functionality, since it - * makes it easier to differentiate our patches from the core OS code. - * - * If the functionality we wish to add/override isn't currently possible to via - * overriding a guice module, we should contribute the necessary modules/interfaces - * into the OS Presto code base to make it possible. - */ -public class TwitterModuleLoader -{ - private TwitterModuleLoader() - { - } - - public static Iterable getAdditionalModules() - { - return ImmutableList.of( - binder -> Multibinder.newSetBinder(binder, EventClient.class) - .addBinding() - .to(EventProcessor.class) - .in(Scopes.SINGLETON), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>() {}), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) - .addBinding() - .to(new TypeLiteral(){}) - .in(Scopes.SINGLETON), - binder -> Multibinder.newSetBinder(binder, new TypeLiteral>(){}) - .addBinding() - .to(QueryScriber.class) - .in(Scopes.SINGLETON) - ); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java deleted file mode 100644 index 02edd8da2440..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryLogger.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter.logging; - -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.facebook.presto.spi.StandardErrorCode; -import io.airlift.log.Logger; -import io.airlift.units.Duration; - -import java.util.concurrent.TimeUnit; - -/** - * Class that logs query events to a file - */ -public class QueryLogger implements QueryEventHandler -{ - private static final int MAX_QUERY_LENGTH = 1000; - private static final String DASH = "-"; - private static final String COLON = ":"; - private static final String SPACE = " "; - private static final String ELIPSIS = "..."; - private static final String QUERY_COMPLETION = "QueryCompletion"; - - private static final Logger log = Logger.get(QueryLogger.class); - - @Override - public void handle(QueryCompletionEvent event) - { - String errorType = DASH; - String errorCode = DASH; - if (event.getErrorCode() != null) { - errorType = StandardErrorCode.toErrorType(event.getErrorCode()).toString(); - if (event.getErrorCodeName() != null) { - errorCode = event.getErrorCodeName() + COLON + event.getErrorCode(); - } - } - - Duration duration = (new Duration( - event.getQueryWallTimeMs(), TimeUnit.MILLISECONDS)) - .convertToMostSuccinctTimeUnit(); - - log.info(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", - QUERY_COMPLETION, event.getQueryId(), toLogValue(event.getRemoteClientAddress()), - event.getQueryState(), errorType, errorCode, event.getUser(), duration, - event.getSplits(), event.getTotalRows(), event.getTotalBytes(), - cleanseAndTrimQuery(event.getQuery()))); - } - - private static String toLogValue(Object object) - { - if (object == null) { - return DASH; - } - else { - return object.toString(); - } - } - - private static String cleanseAndTrimQuery(String query) - { - if (query.length() > MAX_QUERY_LENGTH) { - query = query.substring(0, MAX_QUERY_LENGTH) + ELIPSIS; - } - return query.replace(System.getProperty("line.separator"), SPACE); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java b/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java deleted file mode 100644 index 5e55aa1ac195..000000000000 --- a/presto-main/src/main/java/com/facebook/presto/twitter/logging/QueryScriber.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.twitter.logging; - -import com.facebook.presto.event.query.QueryCompletionEvent; -import com.facebook.presto.event.query.QueryEventHandler; -import com.google.common.base.Optional; -import com.twitter.logging.BareFormatter$; -import com.twitter.logging.Level; -import com.twitter.logging.QueueingHandler; -import com.twitter.logging.ScribeHandler; -import com.twitter.presto.thriftjava.QueryState; -import io.airlift.log.Logger; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; - -import java.util.Base64; -import java.util.logging.LogRecord; - -/** - * Class that scribes query completion events - */ -public class QueryScriber implements QueryEventHandler -{ - private static final String SCRIBE_CATEGORY = "presto_query_completion"; - private static final int MAX_QUEUE_SIZE = 1000; - - private static final Logger log = Logger.get(QueryScriber.class); - - private QueueingHandler queueingHandler; - - // TSerializer is not thread safe - private final ThreadLocal serializer = new ThreadLocal() - { - @Override protected TSerializer initialValue() - { - return new TSerializer(); - } - }; - - public QueryScriber() - { - ScribeHandler scribeHandler = new ScribeHandler( - ScribeHandler.DefaultHostname(), - ScribeHandler.DefaultPort(), - SCRIBE_CATEGORY, - ScribeHandler.DefaultBufferTime(), - ScribeHandler.DefaultConnectBackoff(), - ScribeHandler.DefaultMaxMessagesPerTransaction(), - ScribeHandler.DefaultMaxMessagesToBuffer(), - BareFormatter$.MODULE$, - scala.Option.apply((Level) null)); - queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); - } - - @Override - public void handle(QueryCompletionEvent event) - { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); - Optional message = serializeThriftToString(thriftEvent); - - if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); - } - else { - log.warn("Unable to serialize QueryCompletionEvent: " + event); - } - } - - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - */ - private Optional serializeThriftToString(TBase thriftMessage) - { - try { - return Optional.of( - Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); - } - catch (TException e) { - log.warn(e, "Could not serialize thrift object" + thriftMessage); - return Optional.absent(); - } - } - - private static com.twitter.presto.thriftjava.QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletionEvent event) - { - com.twitter.presto.thriftjava.QueryCompletionEvent thriftEvent = - new com.twitter.presto.thriftjava.QueryCompletionEvent(); - - thriftEvent.query_id = event.getQueryId(); - thriftEvent.transaction_id = event.getTransactionId(); - thriftEvent.user = event.getUser(); - thriftEvent.principal = event.getPrincipal(); - thriftEvent.source = event.getSource(); - thriftEvent.server_version = event.getServerVersion(); - thriftEvent.environment = event.getEnvironment(); - thriftEvent.catalog = event.getCatalog(); - thriftEvent.schema = event.getSchema(); - thriftEvent.remote_client_address = event.getRemoteClientAddress(); - thriftEvent.user_agent = event.getUserAgent(); - thriftEvent.query_state = QueryState.valueOf(event.getQueryState()); - thriftEvent.uri = event.getUri(); - thriftEvent.field_names = event.getFieldNames(); - thriftEvent.query = event.getQuery(); - thriftEvent.create_time_ms = event.getCreateTime().getMillis(); - thriftEvent.execution_start_time_ms = event.getExecutionStartTime().getMillis(); - thriftEvent.end_time_ms = event.getEndTime().getMillis(); - thriftEvent.queued_time_ms = event.getQueuedTimeMs(); - if (event.getAnalysisTimeMs() != null) { - thriftEvent.analysis_time_ms = event.getAnalysisTimeMs(); - } - if (event.getDistributedPlanningTimeMs() != null) { - thriftEvent.distributed_planning_time_ms = event.getDistributedPlanningTimeMs(); - } - if (event.getTotalSplitWallTimeMs() != null) { - thriftEvent.total_split_wall_time_ms = event.getTotalSplitWallTimeMs(); - } - if (event.getTotalSplitCpuTimeMs() != null) { - thriftEvent.total_split_cpu_time_ms = event.getTotalSplitCpuTimeMs(); - } - if (event.getTotalBytes() != null) { - thriftEvent.total_bytes = event.getTotalBytes(); - } - if (event.getTotalRows() != null) { - thriftEvent.total_rows = event.getTotalRows(); - } - thriftEvent.splits = event.getSplits(); - if (event.getErrorCode() != null) { - thriftEvent.error_code_id = event.getErrorCode(); - } - thriftEvent.error_code_name = event.getErrorCodeName(); - thriftEvent.failure_type = event.getFailureType(); - thriftEvent.failure_message = event.getFailureMessage(); - thriftEvent.failure_task = event.getFailureTask(); - thriftEvent.failure_host = event.getFailureHost(); - thriftEvent.output_stage_json = event.getOutputStageJson(); - thriftEvent.failures_json = event.getFailuresJson(); - thriftEvent.inputs_json = event.getInputsJson(); - thriftEvent.session_properties_json = event.getSessionPropertiesJson(); - thriftEvent.query_wall_time_ms = event.getQueryWallTimeMs(); - if (event.getBytesPerSec() != null) { - thriftEvent.bytes_per_sec = event.getBytesPerSec(); - } - if (event.getBytesPerCpuSec() != null) { - thriftEvent.bytes_per_cpu_sec = event.getBytesPerCpuSec(); - } - if (event.getRowsPerSec() != null) { - thriftEvent.rows_per_sec = event.getRowsPerSec(); - } - if (event.getRowsPerCpuSec() != null) { - thriftEvent.rows_per_cpu_sec = event.getRowsPerCpuSec(); - } - - return thriftEvent; - } -} From 77b5d27f2450c1d60b10741e89bbb09c6634820c Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 1 Dec 2016 14:43:10 -0800 Subject: [PATCH 108/151] upgrade to oss 0.157 without new twitter event scriber --- pom.xml | 5 +- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-server/src/main/provisio/presto.xml | 5 - presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 105 ---------- .../QueryCompletedEventScriber.java | 93 --------- .../plugin/eventlistener/Stringify.java | 183 ------------------ .../eventlistener/TwitterEventListener.java | 44 ----- .../TwitterEventListenerFactory.java | 34 ---- .../TwitterEventListenerPlugin.java | 28 --- .../plugin/eventlistener/TwitterScriber.java | 89 --------- 53 files changed, 46 insertions(+), 628 deletions(-) delete mode 100644 twitter-eventlistener-plugin/pom.xml delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java diff --git a/pom.xml b/pom.xml index 77381e4f9179..81e75c58d926 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157-tw-0.28 + 0.149-tw-0.27 @@ -91,7 +91,6 @@ presto-hive-cdh5 presto-teradata-functions presto-example-http - twitter-eventlistener-plugin presto-local-file presto-tpch presto-raptor diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index c626c3ad70b2..65ba4f1fb785 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index a500d24dd11f..c299b9aca579 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 8761d3638bc6..becb28e3392a 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 73936549aa05..d017a7eb34d9 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 4607b9c36eb8..7e26a4b4544a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index d7b20168f9d9..c561d30f9e3f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8ccb6e9fab7a..adf2c74a21e5 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index e5c8a761868a..147e19a1d0be 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index f49684f0083f..7c27715b1043 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e2e5a57835bf..4ce75a91f29e 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4f53264a9894..fc675ad793c1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0cffa89f962b..14ebeeb13e0c 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 533ffeda6853..7877315a96f6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2af0998f777e..5c60a1fe9932 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 03c62da1a4db..814bd89c1b11 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 525324657f8d..518124c6d008 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 7dfdfd71281c..5125111e3813 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 94c8f9f305e7..1ce9320300c8 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 059491bbb6f6..de39cf699043 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 68f97938c645..9ffea9cb6acd 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index e1d4ffd5735a..2094547348f0 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9c387bfc64c5..9effa7e37a10 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 93da37fd6fc6..094ca0e56546 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 008c82fdbe20..c0d1ebb11fbb 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 7087a7cdab75..41a39c831514 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 874270eb3e39..6fb9a73479e8 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 40c93e82bd6e..30cc3fa661f8 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a73eefd181a2..4ba0e816b0f5 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index e3e52ccc9c94..2e905d5fdd2c 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7d6c08ae0f32..75097c126502 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 699d996b9cf9..23952ce91df3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index fc953510fc6e..c7bf601cac62 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index b550cf7cfe49..90ab69762ffb 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index caa9be15c786..6d2f3a67c951 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 33eabb69a5fa..095928ee3535 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 0dec7412e1e5..c265d41f9d53 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index dedf4daf4cb8..202097b9ad4e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 51b1da095a93..6ceae8a36b78 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-server diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 584a241f24d7..1c1d17682dda 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -152,9 +152,4 @@ - - - - - diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8bb450c60706..3f61885c321f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2b9ec4feb53f..a66a95ad0657 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8ef0f28df850..46b9e6f1bcc4 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0923fa884822..5fb15b7fa9ef 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9482bda29427..761874475b86 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 539a9fc3f03f..7646cd0c933f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml deleted file mode 100644 index a38d41663d1f..000000000000 --- a/twitter-eventlistener-plugin/pom.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - 4.0.0 - - com.facebook.presto - presto-root - 0.157-tw-0.28 - - - twitter-eventlistener-plugin - Twitter Event Listener - scribes QueryCompletedEvent - presto-plugin - - - ${project.parent.basedir} - - - - - com.facebook.presto - presto-spi - 0.157-tw-0.28 - provided - - - io.airlift - log - - - com.google.guava - guava - - - - - com.twitter - presto-thrift-java - 0.0.1 - - - com.twitter - util-core_2.11 - - - com.twitter - util-core-java - - - com.twitter - util-function_2.10 - - - com.twitter - util-function-java - - - commons-logging - commons-logging - - - org.scala-lang.modules - scala-parser-combinators_2.11 - - - com.twitter - scrooge-core - - - org.scala-lang - scala-library - - - org.scala-lang - scala-reflect - - - - - com.twitter - util-logging_2.10 - 6.34.0 - - - commons-logging - commons-logging - - - - - org.apache.thrift - libthrift - - - org.scala-lang - scala-library - 2.10.6 - - - commons-logging - commons-logging - - - - - diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java deleted file mode 100644 index 9075bac1dd8b..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; -import com.facebook.presto.spi.eventlistener.QueryContext; -import com.facebook.presto.spi.eventlistener.QueryFailureInfo; -import com.facebook.presto.spi.eventlistener.QueryMetadata; -import com.facebook.presto.spi.eventlistener.QueryStatistics; - -import com.twitter.presto.thriftjava.QueryCompletionEvent; -import com.twitter.presto.thriftjava.QueryState; - -/** - * Class that scribes query completion events - */ -public class QueryCompletedEventScriber extends TwitterScriber -{ - public QueryCompletedEventScriber() - { - super("presto_query_completion"); - } - - public void handle(QueryCompletedEvent event) - { - QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); - if (!scribe(serializeThriftToString(thriftEvent))) { - log.warn("Unable to serialize QueryCompletedEvent: " + event); - } - } - - private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedEvent event) - { - QueryMetadata eventMetadata = event.getMetadata(); - QueryContext eventContext = event.getContext(); - QueryStatistics eventStat = event.getStatistics(); - - QueryCompletionEvent thriftEvent = - new com.twitter.presto.thriftjava.QueryCompletionEvent(); - - thriftEvent.query_id = eventMetadata.getQueryId(); - thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(DASH); - thriftEvent.user = eventContext.getUser(); - thriftEvent.principal = eventContext.getPrincipal().orElse(DASH); - thriftEvent.source = eventContext.getSource().orElse(DASH); - thriftEvent.server_version = eventContext.getServerVersion(); - thriftEvent.environment = eventContext.getEnvironment(); - thriftEvent.catalog = eventContext.getCatalog().orElse(DASH); - thriftEvent.schema = eventContext.getSchema().orElse(DASH); - thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(DASH); - thriftEvent.user_agent = eventContext.getUserAgent().orElse(DASH); - thriftEvent.query_state = QueryState.valueOf(eventMetadata.getQueryState()); - thriftEvent.uri = eventMetadata.getUri().toString(); - thriftEvent.query = eventMetadata.getQuery(); - thriftEvent.create_time_ms = event.getCreateTime().toEpochMilli(); - thriftEvent.execution_start_time_ms = event.getExecutionStartTime().toEpochMilli(); - thriftEvent.end_time_ms = event.getEndTime().toEpochMilli(); - thriftEvent.queued_time_ms = eventStat.getQueuedTime().toMillis(); - thriftEvent.query_wall_time_ms = eventStat.getWallTime().toMillis(); - if (eventStat.getAnalysisTime().isPresent()) { - thriftEvent.analysis_time_ms = eventStat.getAnalysisTime().get().toMillis(); - } - if (eventStat.getDistributedPlanningTime().isPresent()) { - thriftEvent.distributed_planning_time_ms = eventStat.getDistributedPlanningTime().get().toMillis(); - } - thriftEvent.total_bytes = eventStat.getTotalBytes(); - thriftEvent.total_rows = eventStat.getTotalRows(); - thriftEvent.splits = eventStat.getCompletedSplits(); - if (event.getFailureInfo().isPresent()) { - QueryFailureInfo eventFailureInfo = event.getFailureInfo().get(); - thriftEvent.error_code_id = eventFailureInfo.getErrorCode().getCode(); - thriftEvent.error_code_name = eventFailureInfo.getErrorCode().getName(); - thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(DASH); - thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(DASH); - thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(DASH); - thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(DASH); - thriftEvent.failures_json = eventFailureInfo.getFailuresJson(); - } - - return thriftEvent; - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java deleted file mode 100644 index 861511ad49ee..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; -import com.facebook.presto.spi.eventlistener.QueryContext; -import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; -import com.facebook.presto.spi.eventlistener.QueryFailureInfo; -import com.facebook.presto.spi.eventlistener.QueryMetadata; -import com.facebook.presto.spi.eventlistener.QueryStatistics; -import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; -import com.facebook.presto.spi.eventlistener.SplitFailureInfo; -import com.facebook.presto.spi.eventlistener.SplitStatistics; - -import java.time.Duration; -import java.util.Optional; - -class Stringify -{ - private static final String DASH = "-"; - private static final String FIELD_DELIMITER = " ,"; - private static final String OBJECT_DELIMITER = "\n\t"; - - private static String getMillisOrElse(Optional d, String defaultVal) - { - return d.isPresent() ? String.valueOf(d.get().toMillis()) : defaultVal; - } - - private Stringify() - {} - - public static String toString(QueryContext o) - { - String[] content = {"environment(%s)", - "user(%s)", - "userAgent(%s)", - "source(%s)", - "catalog(%s)", - "schema(%s)", - "principal(%s)", - "remoteClientAddress(%s)"}; - return String.format( - String.format("QueryContext: %s", String.join(FIELD_DELIMITER, content)), - o.getEnvironment(), - o.getUser(), - o.getUserAgent().orElse(DASH), - o.getSource().orElse(DASH), - o.getCatalog().orElse(DASH), - o.getSchema().orElse(DASH), - o.getPrincipal().orElse(DASH), - o.getRemoteClientAddress().orElse(DASH)); - } - - public static String toString(QueryFailureInfo o) - { - String[] content = {"errorCode(%s)", - "failureType(%s)", - "failureMessage(%s)", - "failureTask(%s)", - "failureHost(%s)"}; - return String.format( - String.format("QueryFailureInfo: %s", String.join(FIELD_DELIMITER, content)), - o.getErrorCode().toString(), - o.getFailureType().orElse(DASH), - o.getFailureMessage().orElse(DASH), - o.getFailureTask().orElse(DASH), - o.getFailureHost().orElse(DASH)); - } - - public static String toString(QueryMetadata o) - { - String[] content = {"queryId(%s)", - "transactionId(%s)", - "query(%s)", - "queryState(%s)", - "uri(%s)", - "payload(%s)"}; - return String.format( - String.format("QueryMetadata: %s", String.join(FIELD_DELIMITER, content)), - o.getQueryId(), - o.getTransactionId().orElse(DASH), - o.getQuery(), - o.getQueryState(), - o.getUri().toString(), - o.getPayload().orElse(DASH)); - } - - public static String toString(QueryStatistics o) - { - String[] content = {"cpuTime(%d ms)", - "wallTime(%d ms)", - "queuedTime(%d ms)", - "analysisTime(%s ms)", - "distributedPlanningTime(%s ms)", - "peakMemoryBytes(%d)", - "totalBytes(%d)", - "totalRows(%d)", - "completedSplits(%d)", - "complete(%b)"}; - return String.format( - String.format("QueryStatistics: %s", String.join(FIELD_DELIMITER, content)), - o.getCpuTime().toMillis(), - o.getWallTime().toMillis(), - o.getQueuedTime().toMillis(), - getMillisOrElse(o.getAnalysisTime(), DASH), - getMillisOrElse(o.getDistributedPlanningTime(), DASH), - o.getPeakMemoryBytes(), - o.getTotalBytes(), - o.getTotalRows(), - o.getCompletedSplits(), - o.isComplete()); - } - - public static String toString(SplitFailureInfo o) - { - String[] content = {"failureType(%s)", - "failureMessage(%s)"}; - return String.format( - String.format("SplitFailureInfo: %s", String.join(FIELD_DELIMITER, content)), - o.getFailureType(), - o.getFailureMessage()); - } - - public static String toString(SplitStatistics o) - { - String[] content = {"cpuTime(%d ms)", - "wallTime(%d ms)", - "queuedTime(%d ms)", - "userTime(%d ms)", - "completedReadTime(%d ms)", - "completedPositions(%d)", - "completedDataSizeBytes(%d)", - "timeToFirstByte(%s ms)", - "timeToLastByte(%s ms)"}; - return String.format( - String.format("SplitStatistics: %s", String.join(FIELD_DELIMITER, content)), - o.getCpuTime().toMillis(), - o.getWallTime().toMillis(), - o.getQueuedTime().toMillis(), - o.getUserTime().toMillis(), - o.getCompletedReadTime().toMillis(), - o.getCompletedPositions(), - o.getCompletedDataSizeBytes(), - getMillisOrElse(o.getTimeToFirstByte(), DASH), - getMillisOrElse(o.getTimeToLastByte(), DASH)); - } - - public static String toString(QueryCompletedEvent o) - { - String[] content = {toString(o.getMetadata()), - toString(o.getStatistics()), - toString(o.getContext()), - o.getFailureInfo().isPresent() ? toString(o.getFailureInfo().get()) : DASH, - String.format("creationTime: %s", o.getCreateTime().toString()), - String.format("executionStartTime: %s", o.getExecutionStartTime().toString()), - String.format("endTime: %s", o.getEndTime().toString())}; - return String.format("\nQueryCompletedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); - } - - public static String toString(QueryCreatedEvent o) - { - String[] content = {String.format("createTime: %s", o.getCreateTime().toString()), - toString(o.getContext()), - toString(o.getMetadata())}; - return String.format("\nQueryCreatedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); - } - - public static String toString(SplitCompletedEvent o) - { - return ""; - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java deleted file mode 100644 index 57bb6d812e22..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.eventlistener.EventListener; -import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; -import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; -import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; - -import io.airlift.log.Logger; - -public class TwitterEventListener implements EventListener -{ - private static final Logger log = Logger.get(TwitterEventListener.class); - private final QueryCompletedEventScriber scriber = new QueryCompletedEventScriber(); - - @Override - public void queryCreated(QueryCreatedEvent queryCreatedEvent) - { - } - - @Override - public void queryCompleted(QueryCompletedEvent queryCompletedEvent) - { - log.info(Stringify.toString(queryCompletedEvent)); - scriber.handle(queryCompletedEvent); - } - - @Override - public void splitCompleted(SplitCompletedEvent splitCompletedEvent) - { - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java deleted file mode 100644 index 6fedc2d780f1..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.eventlistener.EventListener; -import com.facebook.presto.spi.eventlistener.EventListenerFactory; - -import java.util.Map; - -public class TwitterEventListenerFactory implements EventListenerFactory -{ - @Override - public String getName() - { - return "twitter-event-listener"; - } - - @Override - public EventListener create(Map config) - { - return new TwitterEventListener(); - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java deleted file mode 100644 index 557256058490..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.Plugin; -import com.facebook.presto.spi.eventlistener.EventListenerFactory; - -import com.google.common.collect.ImmutableList; - -public class TwitterEventListenerPlugin implements Plugin -{ - @Override - public Iterable getEventListenerFactories() - { - return ImmutableList.of(new TwitterEventListenerFactory()); - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java deleted file mode 100644 index 1b7e6714e084..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.twitter.logging.BareFormatter$; -import com.twitter.logging.Level; -import com.twitter.logging.QueueingHandler; -import com.twitter.logging.ScribeHandler; - -import io.airlift.log.Logger; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; - -import java.util.Base64; -import java.util.Optional; -import java.util.logging.LogRecord; - -public class TwitterScriber -{ - protected static final String DASH = "-"; - protected static final int MAX_QUEUE_SIZE = 1000; - - protected static final Logger log = Logger.get(TwitterScriber.class); - - private QueueingHandler queueingHandler; - - // TSerializer is not thread safe - private final ThreadLocal serializer = new ThreadLocal() - { - @Override protected TSerializer initialValue() - { - return new TSerializer(); - } - }; - - public TwitterScriber(String scribeCategory) - { - ScribeHandler scribeHandler = new ScribeHandler( - ScribeHandler.DefaultHostname(), - ScribeHandler.DefaultPort(), - scribeCategory, - ScribeHandler.DefaultBufferTime(), - ScribeHandler.DefaultConnectBackoff(), - ScribeHandler.DefaultMaxMessagesPerTransaction(), - ScribeHandler.DefaultMaxMessagesToBuffer(), - BareFormatter$.MODULE$, - scala.Option.apply((Level) null)); - queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); - } - - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - */ - protected Optional serializeThriftToString(TBase thriftMessage) - { - try { - return Optional.of( - Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); - } - catch (TException e) { - log.warn(e, "Could not serialize thrift object" + thriftMessage); - return Optional.empty(); - } - } - - protected boolean scribe(Optional message) - { - if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); - return true; - } - else { - return false; - } - } -} From c6fe499465417d2fcafa0674caffdc2e61d0619a Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 1 Dec 2016 18:06:06 -0800 Subject: [PATCH 109/151] add new twitter event scriber impl as plugin --- presto-server/src/main/provisio/presto.xml | 5 + twitter-eventlistener-plugin/pom.xml | 105 ++++++++++ .../QueryCompletedEventScriber.java | 93 +++++++++ .../plugin/eventlistener/Stringify.java | 183 ++++++++++++++++++ .../eventlistener/TwitterEventListener.java | 44 +++++ .../TwitterEventListenerFactory.java | 34 ++++ .../TwitterEventListenerPlugin.java | 28 +++ .../plugin/eventlistener/TwitterScriber.java | 89 +++++++++ 8 files changed, 581 insertions(+) create mode 100644 twitter-eventlistener-plugin/pom.xml create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 1c1d17682dda..584a241f24d7 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -152,4 +152,9 @@ + + + + + diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml new file mode 100644 index 000000000000..a38d41663d1f --- /dev/null +++ b/twitter-eventlistener-plugin/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + com.facebook.presto + presto-root + 0.157-tw-0.28 + + + twitter-eventlistener-plugin + Twitter Event Listener - scribes QueryCompletedEvent + presto-plugin + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + 0.157-tw-0.28 + provided + + + io.airlift + log + + + com.google.guava + guava + + + + + com.twitter + presto-thrift-java + 0.0.1 + + + com.twitter + util-core_2.11 + + + com.twitter + util-core-java + + + com.twitter + util-function_2.10 + + + com.twitter + util-function-java + + + commons-logging + commons-logging + + + org.scala-lang.modules + scala-parser-combinators_2.11 + + + com.twitter + scrooge-core + + + org.scala-lang + scala-library + + + org.scala-lang + scala-reflect + + + + + com.twitter + util-logging_2.10 + 6.34.0 + + + commons-logging + commons-logging + + + + + org.apache.thrift + libthrift + + + org.scala-lang + scala-library + 2.10.6 + + + commons-logging + commons-logging + + + + + diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java new file mode 100644 index 000000000000..9075bac1dd8b --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryFailureInfo; +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; + +import com.twitter.presto.thriftjava.QueryCompletionEvent; +import com.twitter.presto.thriftjava.QueryState; + +/** + * Class that scribes query completion events + */ +public class QueryCompletedEventScriber extends TwitterScriber +{ + public QueryCompletedEventScriber() + { + super("presto_query_completion"); + } + + public void handle(QueryCompletedEvent event) + { + QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); + if (!scribe(serializeThriftToString(thriftEvent))) { + log.warn("Unable to serialize QueryCompletedEvent: " + event); + } + } + + private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedEvent event) + { + QueryMetadata eventMetadata = event.getMetadata(); + QueryContext eventContext = event.getContext(); + QueryStatistics eventStat = event.getStatistics(); + + QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); + + thriftEvent.query_id = eventMetadata.getQueryId(); + thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(DASH); + thriftEvent.user = eventContext.getUser(); + thriftEvent.principal = eventContext.getPrincipal().orElse(DASH); + thriftEvent.source = eventContext.getSource().orElse(DASH); + thriftEvent.server_version = eventContext.getServerVersion(); + thriftEvent.environment = eventContext.getEnvironment(); + thriftEvent.catalog = eventContext.getCatalog().orElse(DASH); + thriftEvent.schema = eventContext.getSchema().orElse(DASH); + thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(DASH); + thriftEvent.user_agent = eventContext.getUserAgent().orElse(DASH); + thriftEvent.query_state = QueryState.valueOf(eventMetadata.getQueryState()); + thriftEvent.uri = eventMetadata.getUri().toString(); + thriftEvent.query = eventMetadata.getQuery(); + thriftEvent.create_time_ms = event.getCreateTime().toEpochMilli(); + thriftEvent.execution_start_time_ms = event.getExecutionStartTime().toEpochMilli(); + thriftEvent.end_time_ms = event.getEndTime().toEpochMilli(); + thriftEvent.queued_time_ms = eventStat.getQueuedTime().toMillis(); + thriftEvent.query_wall_time_ms = eventStat.getWallTime().toMillis(); + if (eventStat.getAnalysisTime().isPresent()) { + thriftEvent.analysis_time_ms = eventStat.getAnalysisTime().get().toMillis(); + } + if (eventStat.getDistributedPlanningTime().isPresent()) { + thriftEvent.distributed_planning_time_ms = eventStat.getDistributedPlanningTime().get().toMillis(); + } + thriftEvent.total_bytes = eventStat.getTotalBytes(); + thriftEvent.total_rows = eventStat.getTotalRows(); + thriftEvent.splits = eventStat.getCompletedSplits(); + if (event.getFailureInfo().isPresent()) { + QueryFailureInfo eventFailureInfo = event.getFailureInfo().get(); + thriftEvent.error_code_id = eventFailureInfo.getErrorCode().getCode(); + thriftEvent.error_code_name = eventFailureInfo.getErrorCode().getName(); + thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(DASH); + thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(DASH); + thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(DASH); + thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(DASH); + thriftEvent.failures_json = eventFailureInfo.getFailuresJson(); + } + + return thriftEvent; + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java new file mode 100644 index 000000000000..861511ad49ee --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; +import com.facebook.presto.spi.eventlistener.QueryFailureInfo; +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; +import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; +import com.facebook.presto.spi.eventlistener.SplitFailureInfo; +import com.facebook.presto.spi.eventlistener.SplitStatistics; + +import java.time.Duration; +import java.util.Optional; + +class Stringify +{ + private static final String DASH = "-"; + private static final String FIELD_DELIMITER = " ,"; + private static final String OBJECT_DELIMITER = "\n\t"; + + private static String getMillisOrElse(Optional d, String defaultVal) + { + return d.isPresent() ? String.valueOf(d.get().toMillis()) : defaultVal; + } + + private Stringify() + {} + + public static String toString(QueryContext o) + { + String[] content = {"environment(%s)", + "user(%s)", + "userAgent(%s)", + "source(%s)", + "catalog(%s)", + "schema(%s)", + "principal(%s)", + "remoteClientAddress(%s)"}; + return String.format( + String.format("QueryContext: %s", String.join(FIELD_DELIMITER, content)), + o.getEnvironment(), + o.getUser(), + o.getUserAgent().orElse(DASH), + o.getSource().orElse(DASH), + o.getCatalog().orElse(DASH), + o.getSchema().orElse(DASH), + o.getPrincipal().orElse(DASH), + o.getRemoteClientAddress().orElse(DASH)); + } + + public static String toString(QueryFailureInfo o) + { + String[] content = {"errorCode(%s)", + "failureType(%s)", + "failureMessage(%s)", + "failureTask(%s)", + "failureHost(%s)"}; + return String.format( + String.format("QueryFailureInfo: %s", String.join(FIELD_DELIMITER, content)), + o.getErrorCode().toString(), + o.getFailureType().orElse(DASH), + o.getFailureMessage().orElse(DASH), + o.getFailureTask().orElse(DASH), + o.getFailureHost().orElse(DASH)); + } + + public static String toString(QueryMetadata o) + { + String[] content = {"queryId(%s)", + "transactionId(%s)", + "query(%s)", + "queryState(%s)", + "uri(%s)", + "payload(%s)"}; + return String.format( + String.format("QueryMetadata: %s", String.join(FIELD_DELIMITER, content)), + o.getQueryId(), + o.getTransactionId().orElse(DASH), + o.getQuery(), + o.getQueryState(), + o.getUri().toString(), + o.getPayload().orElse(DASH)); + } + + public static String toString(QueryStatistics o) + { + String[] content = {"cpuTime(%d ms)", + "wallTime(%d ms)", + "queuedTime(%d ms)", + "analysisTime(%s ms)", + "distributedPlanningTime(%s ms)", + "peakMemoryBytes(%d)", + "totalBytes(%d)", + "totalRows(%d)", + "completedSplits(%d)", + "complete(%b)"}; + return String.format( + String.format("QueryStatistics: %s", String.join(FIELD_DELIMITER, content)), + o.getCpuTime().toMillis(), + o.getWallTime().toMillis(), + o.getQueuedTime().toMillis(), + getMillisOrElse(o.getAnalysisTime(), DASH), + getMillisOrElse(o.getDistributedPlanningTime(), DASH), + o.getPeakMemoryBytes(), + o.getTotalBytes(), + o.getTotalRows(), + o.getCompletedSplits(), + o.isComplete()); + } + + public static String toString(SplitFailureInfo o) + { + String[] content = {"failureType(%s)", + "failureMessage(%s)"}; + return String.format( + String.format("SplitFailureInfo: %s", String.join(FIELD_DELIMITER, content)), + o.getFailureType(), + o.getFailureMessage()); + } + + public static String toString(SplitStatistics o) + { + String[] content = {"cpuTime(%d ms)", + "wallTime(%d ms)", + "queuedTime(%d ms)", + "userTime(%d ms)", + "completedReadTime(%d ms)", + "completedPositions(%d)", + "completedDataSizeBytes(%d)", + "timeToFirstByte(%s ms)", + "timeToLastByte(%s ms)"}; + return String.format( + String.format("SplitStatistics: %s", String.join(FIELD_DELIMITER, content)), + o.getCpuTime().toMillis(), + o.getWallTime().toMillis(), + o.getQueuedTime().toMillis(), + o.getUserTime().toMillis(), + o.getCompletedReadTime().toMillis(), + o.getCompletedPositions(), + o.getCompletedDataSizeBytes(), + getMillisOrElse(o.getTimeToFirstByte(), DASH), + getMillisOrElse(o.getTimeToLastByte(), DASH)); + } + + public static String toString(QueryCompletedEvent o) + { + String[] content = {toString(o.getMetadata()), + toString(o.getStatistics()), + toString(o.getContext()), + o.getFailureInfo().isPresent() ? toString(o.getFailureInfo().get()) : DASH, + String.format("creationTime: %s", o.getCreateTime().toString()), + String.format("executionStartTime: %s", o.getExecutionStartTime().toString()), + String.format("endTime: %s", o.getEndTime().toString())}; + return String.format("\nQueryCompletedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); + } + + public static String toString(QueryCreatedEvent o) + { + String[] content = {String.format("createTime: %s", o.getCreateTime().toString()), + toString(o.getContext()), + toString(o.getMetadata())}; + return String.format("\nQueryCreatedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); + } + + public static String toString(SplitCompletedEvent o) + { + return ""; + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java new file mode 100644 index 000000000000..57bb6d812e22 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; +import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; + +import io.airlift.log.Logger; + +public class TwitterEventListener implements EventListener +{ + private static final Logger log = Logger.get(TwitterEventListener.class); + private final QueryCompletedEventScriber scriber = new QueryCompletedEventScriber(); + + @Override + public void queryCreated(QueryCreatedEvent queryCreatedEvent) + { + } + + @Override + public void queryCompleted(QueryCompletedEvent queryCompletedEvent) + { + log.info(Stringify.toString(queryCompletedEvent)); + scriber.handle(queryCompletedEvent); + } + + @Override + public void splitCompleted(SplitCompletedEvent splitCompletedEvent) + { + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java new file mode 100644 index 000000000000..6fedc2d780f1 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; + +import java.util.Map; + +public class TwitterEventListenerFactory implements EventListenerFactory +{ + @Override + public String getName() + { + return "twitter-event-listener"; + } + + @Override + public EventListener create(Map config) + { + return new TwitterEventListener(); + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java new file mode 100644 index 000000000000..557256058490 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; + +import com.google.common.collect.ImmutableList; + +public class TwitterEventListenerPlugin implements Plugin +{ + @Override + public Iterable getEventListenerFactories() + { + return ImmutableList.of(new TwitterEventListenerFactory()); + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java new file mode 100644 index 000000000000..1b7e6714e084 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; + +import io.airlift.log.Logger; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.Optional; +import java.util.logging.LogRecord; + +public class TwitterScriber +{ + protected static final String DASH = "-"; + protected static final int MAX_QUEUE_SIZE = 1000; + + protected static final Logger log = Logger.get(TwitterScriber.class); + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() + { + @Override protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public TwitterScriber(String scribeCategory) + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + scribeCategory, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + */ + protected Optional serializeThriftToString(TBase thriftMessage) + { + try { + return Optional.of( + Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); + } + catch (TException e) { + log.warn(e, "Could not serialize thrift object" + thriftMessage); + return Optional.empty(); + } + } + + protected boolean scribe(Optional message) + { + if (message.isPresent()) { + LogRecord logRecord = new LogRecord(Level.ALL, message.get()); + queueingHandler.publish(logRecord); + return true; + } + else { + return false; + } + } +} From 978c18637cb8f9c570cde096c7e2ccb150843e49 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 1 Dec 2016 18:08:09 -0800 Subject: [PATCH 110/151] add twitter-eventlistener-plugin as module to top level pom --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 81e75c58d926..5212a89f95dd 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ presto-hive-cdh5 presto-teradata-functions presto-example-http + twitter-eventlistener-plugin presto-local-file presto-tpch presto-raptor From 3df6fed5bd9076a60731ddeab57009cd34b8a034 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Sat, 3 Dec 2016 14:26:59 -0800 Subject: [PATCH 111/151] address Bill's comments --- pom.xml | 4 +- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- .../QueryCompletedEventScriber.java | 41 ++-- .../plugin/eventlistener/Stringify.java | 183 ------------------ .../eventlistener/TwitterEventListener.java | 1 - .../plugin/eventlistener/TwitterScriber.java | 28 ++- 49 files changed, 80 insertions(+), 265 deletions(-) delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java diff --git a/pom.xml b/pom.xml index 5212a89f95dd..77381e4f9179 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.149-tw-0.27 + 0.157-tw-0.28 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 65ba4f1fb785..c626c3ad70b2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index c299b9aca579..a500d24dd11f 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index becb28e3392a..8761d3638bc6 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d017a7eb34d9..73936549aa05 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 7e26a4b4544a..4607b9c36eb8 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index c561d30f9e3f..d7b20168f9d9 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index adf2c74a21e5..8ccb6e9fab7a 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 147e19a1d0be..e5c8a761868a 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 7c27715b1043..f49684f0083f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 4ce75a91f29e..e2e5a57835bf 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index fc675ad793c1..4f53264a9894 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 14ebeeb13e0c..0cffa89f962b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 7877315a96f6..533ffeda6853 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 5c60a1fe9932..2af0998f777e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 814bd89c1b11..03c62da1a4db 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 518124c6d008..525324657f8d 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 5125111e3813..7dfdfd71281c 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 1ce9320300c8..94c8f9f305e7 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index de39cf699043..059491bbb6f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 9ffea9cb6acd..68f97938c645 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2094547348f0..e1d4ffd5735a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9effa7e37a10..9c387bfc64c5 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 094ca0e56546..93da37fd6fc6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c0d1ebb11fbb..008c82fdbe20 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 41a39c831514..7087a7cdab75 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 6fb9a73479e8..874270eb3e39 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 30cc3fa661f8..40c93e82bd6e 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 4ba0e816b0f5..a73eefd181a2 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index 2e905d5fdd2c..e3e52ccc9c94 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 75097c126502..7d6c08ae0f32 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 23952ce91df3..699d996b9cf9 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index c7bf601cac62..fc953510fc6e 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 90ab69762ffb..b550cf7cfe49 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 6d2f3a67c951..caa9be15c786 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 095928ee3535..33eabb69a5fa 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index c265d41f9d53..0dec7412e1e5 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 202097b9ad4e..dedf4daf4cb8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 6ceae8a36b78..51b1da095a93 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 3f61885c321f..8bb450c60706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a66a95ad0657..2b9ec4feb53f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 46b9e6f1bcc4..8ef0f28df850 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 5fb15b7fa9ef..0923fa884822 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 761874475b86..9482bda29427 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7646cd0c933f..539a9fc3f03f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-verifier diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java index 9075bac1dd8b..97d8decce936 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java @@ -25,19 +25,20 @@ /** * Class that scribes query completion events */ -public class QueryCompletedEventScriber extends TwitterScriber +public class QueryCompletedEventScriber { - public QueryCompletedEventScriber() - { - super("presto_query_completion"); - } + private static String dash = "-"; + private TwitterScriber scriber = new TwitterScriber("presto_query_completion"); public void handle(QueryCompletedEvent event) { - QueryCompletionEvent thriftEvent = toThriftQueryCompletionEvent(event); - if (!scribe(serializeThriftToString(thriftEvent))) { - log.warn("Unable to serialize QueryCompletedEvent: " + event); - } + scriber.scribe(toThriftQueryCompletionEvent(event), + String.format("Query(id=%s, user=%s, env=%s, schema=%s.%s)", + event.getMetadata().getQueryId(), + event.getContext().getUser(), + event.getContext().getEnvironment(), + event.getContext().getCatalog().orElse(dash), + event.getContext().getSchema().orElse(dash))); } private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedEvent event) @@ -50,16 +51,16 @@ private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedE new com.twitter.presto.thriftjava.QueryCompletionEvent(); thriftEvent.query_id = eventMetadata.getQueryId(); - thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(DASH); + thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(dash); thriftEvent.user = eventContext.getUser(); - thriftEvent.principal = eventContext.getPrincipal().orElse(DASH); - thriftEvent.source = eventContext.getSource().orElse(DASH); + thriftEvent.principal = eventContext.getPrincipal().orElse(dash); + thriftEvent.source = eventContext.getSource().orElse(dash); thriftEvent.server_version = eventContext.getServerVersion(); thriftEvent.environment = eventContext.getEnvironment(); - thriftEvent.catalog = eventContext.getCatalog().orElse(DASH); - thriftEvent.schema = eventContext.getSchema().orElse(DASH); - thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(DASH); - thriftEvent.user_agent = eventContext.getUserAgent().orElse(DASH); + thriftEvent.catalog = eventContext.getCatalog().orElse(dash); + thriftEvent.schema = eventContext.getSchema().orElse(dash); + thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(dash); + thriftEvent.user_agent = eventContext.getUserAgent().orElse(dash); thriftEvent.query_state = QueryState.valueOf(eventMetadata.getQueryState()); thriftEvent.uri = eventMetadata.getUri().toString(); thriftEvent.query = eventMetadata.getQuery(); @@ -81,10 +82,10 @@ private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedE QueryFailureInfo eventFailureInfo = event.getFailureInfo().get(); thriftEvent.error_code_id = eventFailureInfo.getErrorCode().getCode(); thriftEvent.error_code_name = eventFailureInfo.getErrorCode().getName(); - thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(DASH); - thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(DASH); - thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(DASH); - thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(DASH); + thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(dash); + thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(dash); + thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(dash); + thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(dash); thriftEvent.failures_json = eventFailureInfo.getFailuresJson(); } diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java deleted file mode 100644 index 861511ad49ee..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/Stringify.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; -import com.facebook.presto.spi.eventlistener.QueryContext; -import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; -import com.facebook.presto.spi.eventlistener.QueryFailureInfo; -import com.facebook.presto.spi.eventlistener.QueryMetadata; -import com.facebook.presto.spi.eventlistener.QueryStatistics; -import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; -import com.facebook.presto.spi.eventlistener.SplitFailureInfo; -import com.facebook.presto.spi.eventlistener.SplitStatistics; - -import java.time.Duration; -import java.util.Optional; - -class Stringify -{ - private static final String DASH = "-"; - private static final String FIELD_DELIMITER = " ,"; - private static final String OBJECT_DELIMITER = "\n\t"; - - private static String getMillisOrElse(Optional d, String defaultVal) - { - return d.isPresent() ? String.valueOf(d.get().toMillis()) : defaultVal; - } - - private Stringify() - {} - - public static String toString(QueryContext o) - { - String[] content = {"environment(%s)", - "user(%s)", - "userAgent(%s)", - "source(%s)", - "catalog(%s)", - "schema(%s)", - "principal(%s)", - "remoteClientAddress(%s)"}; - return String.format( - String.format("QueryContext: %s", String.join(FIELD_DELIMITER, content)), - o.getEnvironment(), - o.getUser(), - o.getUserAgent().orElse(DASH), - o.getSource().orElse(DASH), - o.getCatalog().orElse(DASH), - o.getSchema().orElse(DASH), - o.getPrincipal().orElse(DASH), - o.getRemoteClientAddress().orElse(DASH)); - } - - public static String toString(QueryFailureInfo o) - { - String[] content = {"errorCode(%s)", - "failureType(%s)", - "failureMessage(%s)", - "failureTask(%s)", - "failureHost(%s)"}; - return String.format( - String.format("QueryFailureInfo: %s", String.join(FIELD_DELIMITER, content)), - o.getErrorCode().toString(), - o.getFailureType().orElse(DASH), - o.getFailureMessage().orElse(DASH), - o.getFailureTask().orElse(DASH), - o.getFailureHost().orElse(DASH)); - } - - public static String toString(QueryMetadata o) - { - String[] content = {"queryId(%s)", - "transactionId(%s)", - "query(%s)", - "queryState(%s)", - "uri(%s)", - "payload(%s)"}; - return String.format( - String.format("QueryMetadata: %s", String.join(FIELD_DELIMITER, content)), - o.getQueryId(), - o.getTransactionId().orElse(DASH), - o.getQuery(), - o.getQueryState(), - o.getUri().toString(), - o.getPayload().orElse(DASH)); - } - - public static String toString(QueryStatistics o) - { - String[] content = {"cpuTime(%d ms)", - "wallTime(%d ms)", - "queuedTime(%d ms)", - "analysisTime(%s ms)", - "distributedPlanningTime(%s ms)", - "peakMemoryBytes(%d)", - "totalBytes(%d)", - "totalRows(%d)", - "completedSplits(%d)", - "complete(%b)"}; - return String.format( - String.format("QueryStatistics: %s", String.join(FIELD_DELIMITER, content)), - o.getCpuTime().toMillis(), - o.getWallTime().toMillis(), - o.getQueuedTime().toMillis(), - getMillisOrElse(o.getAnalysisTime(), DASH), - getMillisOrElse(o.getDistributedPlanningTime(), DASH), - o.getPeakMemoryBytes(), - o.getTotalBytes(), - o.getTotalRows(), - o.getCompletedSplits(), - o.isComplete()); - } - - public static String toString(SplitFailureInfo o) - { - String[] content = {"failureType(%s)", - "failureMessage(%s)"}; - return String.format( - String.format("SplitFailureInfo: %s", String.join(FIELD_DELIMITER, content)), - o.getFailureType(), - o.getFailureMessage()); - } - - public static String toString(SplitStatistics o) - { - String[] content = {"cpuTime(%d ms)", - "wallTime(%d ms)", - "queuedTime(%d ms)", - "userTime(%d ms)", - "completedReadTime(%d ms)", - "completedPositions(%d)", - "completedDataSizeBytes(%d)", - "timeToFirstByte(%s ms)", - "timeToLastByte(%s ms)"}; - return String.format( - String.format("SplitStatistics: %s", String.join(FIELD_DELIMITER, content)), - o.getCpuTime().toMillis(), - o.getWallTime().toMillis(), - o.getQueuedTime().toMillis(), - o.getUserTime().toMillis(), - o.getCompletedReadTime().toMillis(), - o.getCompletedPositions(), - o.getCompletedDataSizeBytes(), - getMillisOrElse(o.getTimeToFirstByte(), DASH), - getMillisOrElse(o.getTimeToLastByte(), DASH)); - } - - public static String toString(QueryCompletedEvent o) - { - String[] content = {toString(o.getMetadata()), - toString(o.getStatistics()), - toString(o.getContext()), - o.getFailureInfo().isPresent() ? toString(o.getFailureInfo().get()) : DASH, - String.format("creationTime: %s", o.getCreateTime().toString()), - String.format("executionStartTime: %s", o.getExecutionStartTime().toString()), - String.format("endTime: %s", o.getEndTime().toString())}; - return String.format("\nQueryCompletedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); - } - - public static String toString(QueryCreatedEvent o) - { - String[] content = {String.format("createTime: %s", o.getCreateTime().toString()), - toString(o.getContext()), - toString(o.getMetadata())}; - return String.format("\nQueryCreatedEvent:\n\t%s\n", String.join(OBJECT_DELIMITER, content)); - } - - public static String toString(SplitCompletedEvent o) - { - return ""; - } -} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java index 57bb6d812e22..0c622b0ba34c 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java @@ -33,7 +33,6 @@ public void queryCreated(QueryCreatedEvent queryCreatedEvent) @Override public void queryCompleted(QueryCompletedEvent queryCompletedEvent) { - log.info(Stringify.toString(queryCompletedEvent)); scriber.handle(queryCompletedEvent); } diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java index 1b7e6714e084..11be68af43d2 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java @@ -24,7 +24,6 @@ import org.apache.thrift.TSerializer; import java.util.Base64; -import java.util.Optional; import java.util.logging.LogRecord; public class TwitterScriber @@ -62,28 +61,27 @@ public TwitterScriber(String scribeCategory) /** * Serialize a thrift object to bytes, compress, then encode as a base64 string. + * Throws TException */ - protected Optional serializeThriftToString(TBase thriftMessage) + public String serializeThriftToString(TBase thriftMessage) throws TException + { + return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); + } + + public void scribe(TBase thriftMessage, String origEventIdentifier) { try { - return Optional.of( - Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage))); + String encodedStr = serializeThriftToString(thriftMessage); + scribe(encodedStr); } catch (TException e) { - log.warn(e, "Could not serialize thrift object" + thriftMessage); - return Optional.empty(); + log.warn(e, String.format("Could not serialize thrift object of %s", origEventIdentifier)); } } - protected boolean scribe(Optional message) + public void scribe(String message) { - if (message.isPresent()) { - LogRecord logRecord = new LogRecord(Level.ALL, message.get()); - queueingHandler.publish(logRecord); - return true; - } - else { - return false; - } + LogRecord logRecord = new LogRecord(Level.ALL, message); + queueingHandler.publish(logRecord); } } From 86917eaa24ad4d7867460d24b1db5a813c7045b8 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Mon, 5 Dec 2016 12:16:25 -0800 Subject: [PATCH 112/151] back out version change, address comments --- pom.xml | 4 +- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 +- .../QueryCompletedEventScriber.java | 48 +++++++++++-------- .../plugin/eventlistener/TwitterScriber.java | 48 ++++++++----------- 48 files changed, 97 insertions(+), 95 deletions(-) diff --git a/pom.xml b/pom.xml index 77381e4f9179..5212a89f95dd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157-tw-0.28 + 0.149-tw-0.27 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index c626c3ad70b2..65ba4f1fb785 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index a500d24dd11f..c299b9aca579 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 8761d3638bc6..becb28e3392a 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 73936549aa05..d017a7eb34d9 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 4607b9c36eb8..7e26a4b4544a 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index d7b20168f9d9..c561d30f9e3f 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8ccb6e9fab7a..adf2c74a21e5 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index e5c8a761868a..147e19a1d0be 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index f49684f0083f..7c27715b1043 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e2e5a57835bf..4ce75a91f29e 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4f53264a9894..fc675ad793c1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0cffa89f962b..14ebeeb13e0c 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 533ffeda6853..7877315a96f6 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2af0998f777e..5c60a1fe9932 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 03c62da1a4db..814bd89c1b11 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 525324657f8d..518124c6d008 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 7dfdfd71281c..5125111e3813 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 94c8f9f305e7..1ce9320300c8 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 059491bbb6f6..de39cf699043 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 68f97938c645..9ffea9cb6acd 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index e1d4ffd5735a..2094547348f0 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9c387bfc64c5..9effa7e37a10 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 93da37fd6fc6..094ca0e56546 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 008c82fdbe20..c0d1ebb11fbb 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 7087a7cdab75..41a39c831514 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 874270eb3e39..6fb9a73479e8 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 40c93e82bd6e..30cc3fa661f8 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a73eefd181a2..4ba0e816b0f5 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index e3e52ccc9c94..2e905d5fdd2c 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7d6c08ae0f32..75097c126502 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 699d996b9cf9..23952ce91df3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index fc953510fc6e..c7bf601cac62 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index b550cf7cfe49..90ab69762ffb 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index caa9be15c786..6d2f3a67c951 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 33eabb69a5fa..095928ee3535 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 0dec7412e1e5..c265d41f9d53 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index dedf4daf4cb8..202097b9ad4e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 51b1da095a93..6ceae8a36b78 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8bb450c60706..3f61885c321f 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2b9ec4feb53f..a66a95ad0657 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8ef0f28df850..46b9e6f1bcc4 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0923fa884822..5fb15b7fa9ef 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.149-tw-0.27 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9482bda29427..761874475b86 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 539a9fc3f03f..7646cd0c933f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index a38d41663d1f..4ad84ca03757 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.149-tw-0.27 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.157-tw-0.28 + 0.149-tw-0.27 provided diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java index 97d8decce936..e9c2faf3eda0 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java @@ -22,23 +22,33 @@ import com.twitter.presto.thriftjava.QueryCompletionEvent; import com.twitter.presto.thriftjava.QueryState; +import io.airlift.log.Logger; +import org.apache.thrift.TException; + /** * Class that scribes query completion events */ public class QueryCompletedEventScriber { - private static String dash = "-"; + private static final String DASH = "-"; + private static final Logger log = Logger.get(QueryCompletedEventScriber.class); + private TwitterScriber scriber = new TwitterScriber("presto_query_completion"); public void handle(QueryCompletedEvent event) { - scriber.scribe(toThriftQueryCompletionEvent(event), - String.format("Query(id=%s, user=%s, env=%s, schema=%s.%s)", - event.getMetadata().getQueryId(), - event.getContext().getUser(), - event.getContext().getEnvironment(), - event.getContext().getCatalog().orElse(dash), - event.getContext().getSchema().orElse(dash))); + try { + scriber.scribe(toThriftQueryCompletionEvent(event)); + } + catch (TException e) { + log.warn(e, + String.format("Could not serialize thrift object of Query(id=%s, user=%s, env=%s, schema=%s.%s)", + event.getMetadata().getQueryId(), + event.getContext().getUser(), + event.getContext().getEnvironment(), + event.getContext().getCatalog().orElse(DASH), + event.getContext().getSchema().orElse(DASH))); + } } private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedEvent event) @@ -51,16 +61,16 @@ private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedE new com.twitter.presto.thriftjava.QueryCompletionEvent(); thriftEvent.query_id = eventMetadata.getQueryId(); - thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(dash); + thriftEvent.transaction_id = eventMetadata.getTransactionId().orElse(DASH); thriftEvent.user = eventContext.getUser(); - thriftEvent.principal = eventContext.getPrincipal().orElse(dash); - thriftEvent.source = eventContext.getSource().orElse(dash); + thriftEvent.principal = eventContext.getPrincipal().orElse(DASH); + thriftEvent.source = eventContext.getSource().orElse(DASH); thriftEvent.server_version = eventContext.getServerVersion(); thriftEvent.environment = eventContext.getEnvironment(); - thriftEvent.catalog = eventContext.getCatalog().orElse(dash); - thriftEvent.schema = eventContext.getSchema().orElse(dash); - thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(dash); - thriftEvent.user_agent = eventContext.getUserAgent().orElse(dash); + thriftEvent.catalog = eventContext.getCatalog().orElse(DASH); + thriftEvent.schema = eventContext.getSchema().orElse(DASH); + thriftEvent.remote_client_address = eventContext.getRemoteClientAddress().orElse(DASH); + thriftEvent.user_agent = eventContext.getUserAgent().orElse(DASH); thriftEvent.query_state = QueryState.valueOf(eventMetadata.getQueryState()); thriftEvent.uri = eventMetadata.getUri().toString(); thriftEvent.query = eventMetadata.getQuery(); @@ -82,10 +92,10 @@ private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedE QueryFailureInfo eventFailureInfo = event.getFailureInfo().get(); thriftEvent.error_code_id = eventFailureInfo.getErrorCode().getCode(); thriftEvent.error_code_name = eventFailureInfo.getErrorCode().getName(); - thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(dash); - thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(dash); - thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(dash); - thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(dash); + thriftEvent.failure_type = eventFailureInfo.getFailureType().orElse(DASH); + thriftEvent.failure_message = eventFailureInfo.getFailureMessage().orElse(DASH); + thriftEvent.failure_task = eventFailureInfo.getFailureTask().orElse(DASH); + thriftEvent.failure_host = eventFailureInfo.getFailureHost().orElse(DASH); thriftEvent.failures_json = eventFailureInfo.getFailuresJson(); } diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java index 11be68af43d2..604ebad694e7 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java @@ -18,7 +18,6 @@ import com.twitter.logging.QueueingHandler; import com.twitter.logging.ScribeHandler; -import io.airlift.log.Logger; import org.apache.thrift.TBase; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; @@ -28,10 +27,8 @@ public class TwitterScriber { - protected static final String DASH = "-"; - protected static final int MAX_QUEUE_SIZE = 1000; - - protected static final Logger log = Logger.get(TwitterScriber.class); + private static final String DASH = "-"; + private static final int MAX_QUEUE_SIZE = 1000; private QueueingHandler queueingHandler; @@ -44,6 +41,21 @@ public class TwitterScriber } }; + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + * Throws TException + */ + private String serializeThriftToString(TBase thriftMessage) throws TException + { + return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); + } + + private void scribe(String message) + { + LogRecord logRecord = new LogRecord(Level.ALL, message); + queueingHandler.publish(logRecord); + } + public TwitterScriber(String scribeCategory) { ScribeHandler scribeHandler = new ScribeHandler( @@ -59,29 +71,9 @@ public TwitterScriber(String scribeCategory) queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); } - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - * Throws TException - */ - public String serializeThriftToString(TBase thriftMessage) throws TException - { - return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); - } - - public void scribe(TBase thriftMessage, String origEventIdentifier) + public void scribe(TBase thriftMessage) throws TException { - try { - String encodedStr = serializeThriftToString(thriftMessage); - scribe(encodedStr); - } - catch (TException e) { - log.warn(e, String.format("Could not serialize thrift object of %s", origEventIdentifier)); - } - } - - public void scribe(String message) - { - LogRecord logRecord = new LogRecord(Level.ALL, message); - queueingHandler.publish(logRecord); + String encodedStr = serializeThriftToString(thriftMessage); + scribe(encodedStr); } } From 1d36d6492205047127464905110f97a408887a62 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Mon, 5 Dec 2016 13:11:56 -0800 Subject: [PATCH 113/151] constructor first then public, private methods --- .../twitter/presto/plugin/eventlistener/:q | 78 +++++++++++++++++++ .../plugin/eventlistener/TwitterScriber.java | 33 ++++---- 2 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q new file mode 100644 index 000000000000..9e8524602255 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; + +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.logging.LogRecord; + +public class TwitterScriber +{ + private static final String DASH = "-"; + private static final int MAX_QUEUE_SIZE = 1000; + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() + { + @Override protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public TwitterScriber(String scribeCategory) + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + scribeCategory, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + public void scribe(TBase thriftMessage) throws TException + { + scribe(serializeThriftToString(thriftMessage)); + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + * Throws TException + */ + private String serializeThriftToString(TBase thriftMessage) throws TException + { + return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); + } + + private void scribe(String message) + { + LogRecord logRecord = new LogRecord(Level.ALL, message); + queueingHandler.publish(logRecord); + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java index 604ebad694e7..9e8524602255 100644 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java @@ -41,21 +41,6 @@ public class TwitterScriber } }; - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - * Throws TException - */ - private String serializeThriftToString(TBase thriftMessage) throws TException - { - return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); - } - - private void scribe(String message) - { - LogRecord logRecord = new LogRecord(Level.ALL, message); - queueingHandler.publish(logRecord); - } - public TwitterScriber(String scribeCategory) { ScribeHandler scribeHandler = new ScribeHandler( @@ -73,7 +58,21 @@ public TwitterScriber(String scribeCategory) public void scribe(TBase thriftMessage) throws TException { - String encodedStr = serializeThriftToString(thriftMessage); - scribe(encodedStr); + scribe(serializeThriftToString(thriftMessage)); + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + * Throws TException + */ + private String serializeThriftToString(TBase thriftMessage) throws TException + { + return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); + } + + private void scribe(String message) + { + LogRecord logRecord = new LogRecord(Level.ALL, message); + queueingHandler.publish(logRecord); } } From 41a0ab334fcbe81ca4f258d2ec27dd5aef988df3 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Mon, 5 Dec 2016 13:26:12 -0800 Subject: [PATCH 114/151] remove accidental commit --- .../twitter/presto/plugin/eventlistener/:q | 78 ------------------- 1 file changed, 78 deletions(-) delete mode 100644 twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q deleted file mode 100644 index 9e8524602255..000000000000 --- a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/:q +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.twitter.presto.plugin.eventlistener; - -import com.twitter.logging.BareFormatter$; -import com.twitter.logging.Level; -import com.twitter.logging.QueueingHandler; -import com.twitter.logging.ScribeHandler; - -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; - -import java.util.Base64; -import java.util.logging.LogRecord; - -public class TwitterScriber -{ - private static final String DASH = "-"; - private static final int MAX_QUEUE_SIZE = 1000; - - private QueueingHandler queueingHandler; - - // TSerializer is not thread safe - private final ThreadLocal serializer = new ThreadLocal() - { - @Override protected TSerializer initialValue() - { - return new TSerializer(); - } - }; - - public TwitterScriber(String scribeCategory) - { - ScribeHandler scribeHandler = new ScribeHandler( - ScribeHandler.DefaultHostname(), - ScribeHandler.DefaultPort(), - scribeCategory, - ScribeHandler.DefaultBufferTime(), - ScribeHandler.DefaultConnectBackoff(), - ScribeHandler.DefaultMaxMessagesPerTransaction(), - ScribeHandler.DefaultMaxMessagesToBuffer(), - BareFormatter$.MODULE$, - scala.Option.apply((Level) null)); - queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); - } - - public void scribe(TBase thriftMessage) throws TException - { - scribe(serializeThriftToString(thriftMessage)); - } - - /** - * Serialize a thrift object to bytes, compress, then encode as a base64 string. - * Throws TException - */ - private String serializeThriftToString(TBase thriftMessage) throws TException - { - return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); - } - - private void scribe(String message) - { - LogRecord logRecord = new LogRecord(Level.ALL, message); - queueingHandler.publish(logRecord); - } -} From 0ed477adad4f12910b37edefc985b9f6afcab188 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Mon, 5 Dec 2016 16:48:43 -0800 Subject: [PATCH 115/151] Prepare for 0.157-tw-0.28 internal release --- pom.xml | 4 ++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 5212a89f95dd..77381e4f9179 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.149-tw-0.27 + 0.157-tw-0.28 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 65ba4f1fb785..c626c3ad70b2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index c299b9aca579..a500d24dd11f 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index becb28e3392a..8761d3638bc6 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d017a7eb34d9..73936549aa05 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 7e26a4b4544a..4607b9c36eb8 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index c561d30f9e3f..d7b20168f9d9 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index adf2c74a21e5..8ccb6e9fab7a 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 147e19a1d0be..e5c8a761868a 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 7c27715b1043..f49684f0083f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 4ce75a91f29e..e2e5a57835bf 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index fc675ad793c1..4f53264a9894 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 14ebeeb13e0c..0cffa89f962b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 7877315a96f6..533ffeda6853 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 5c60a1fe9932..2af0998f777e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 814bd89c1b11..03c62da1a4db 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 518124c6d008..525324657f8d 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 5125111e3813..7dfdfd71281c 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 1ce9320300c8..94c8f9f305e7 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index de39cf699043..059491bbb6f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 9ffea9cb6acd..68f97938c645 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 2094547348f0..e1d4ffd5735a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9effa7e37a10..9c387bfc64c5 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 094ca0e56546..93da37fd6fc6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index c0d1ebb11fbb..008c82fdbe20 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 41a39c831514..7087a7cdab75 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 6fb9a73479e8..874270eb3e39 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 30cc3fa661f8..40c93e82bd6e 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 4ba0e816b0f5..a73eefd181a2 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index 2e905d5fdd2c..e3e52ccc9c94 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 75097c126502..7d6c08ae0f32 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 23952ce91df3..699d996b9cf9 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index c7bf601cac62..fc953510fc6e 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 90ab69762ffb..b550cf7cfe49 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 6d2f3a67c951..caa9be15c786 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 095928ee3535..33eabb69a5fa 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index c265d41f9d53..0dec7412e1e5 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 202097b9ad4e..dedf4daf4cb8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 6ceae8a36b78..51b1da095a93 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 3f61885c321f..8bb450c60706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index a66a95ad0657..2b9ec4feb53f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 46b9e6f1bcc4..8ef0f28df850 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 5fb15b7fa9ef..0923fa884822 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.149-tw-0.27 + 0.157-tw-0.28 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 761874475b86..9482bda29427 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7646cd0c933f..539a9fc3f03f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index 4ad84ca03757..a38d41663d1f 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.149-tw-0.27 + 0.157-tw-0.28 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.149-tw-0.27 + 0.157-tw-0.28 provided From c0ada2b05adf2c3c363f6a1b983527863b9be07b Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Wed, 7 Dec 2016 17:31:52 -0800 Subject: [PATCH 116/151] Added TTransportPool --- .../PooledHiveMetastoreClientFactory.java | 67 +++++++++++ .../hive/util/PooledTTransportFactory.java | 38 ++++++ .../twitter/hive/util/TTransportPool.java | 108 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java new file mode 100644 index 000000000000..cea665466ff1 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.ThriftHiveMetastoreClient; +import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; +import com.facebook.presto.hive.metastore.HiveMetastoreClient; +import com.facebook.presto.twitter.hive.util.PooledTTransportFactory; +import com.facebook.presto.twitter.hive.util.TTransportPool; +import com.google.common.net.HostAndPort; +import com.google.common.primitives.Ints; +import io.airlift.units.Duration; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import javax.annotation.Nullable; +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + +public class PooledHiveMetastoreClientFactory +{ + private final HostAndPort socksProxy; + private final int timeoutMillis; + private final HiveMetastoreAuthentication metastoreAuthentication; + + public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, HiveMetastoreAuthentication metastoreAuthentication) + { + this.socksProxy = socksProxy; + this.timeoutMillis = Ints.checkedCast(timeout.toMillis()); + this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); + this.transportPool = new TTransportPool(); + } + + @Inject + public PooledHiveMetastoreClientFactory(HiveClientConfig config, HiveMetastoreAuthentication metastoreAuthentication) + { + this(config.getMetastoreSocksProxy(), config.getMetastoreTimeout(), metastoreAuthentication); + } + + public HiveMetastoreClient create(String host, int port) + throws TTransportException + { + try { + TTransport transport= transportPool.borrowObject(host, port); + if (transport == null) { + transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(host, port, socksProxy, timeoutMillis, metastoreAuthentication)); + } + } + catch (Exception e) { + throw new TTransportException(e.getType(), String.format("%s: %s", host, e.getMessage()), e.getCause()) + } + return new ThriftHiveMetastoreClient(transport); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java new file mode 100644 index 000000000000..5e04ed9f64a6 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.util; + +import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.net.InetSocketAddress +import java.net.SocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +/** + * Utility class to handle creating and caching the UserGroupInformation object. + */ +public class PooledTTransportFactory +{ + +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java new file mode 100644 index 000000000000..b148404a7ff6 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.util; + +import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.net.InetSocketAddress +import java.net.SocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +/** + * Utility class to handle creating and caching the UserGroupInformation object. + */ +public class TTransportPool +{ + private final ConcurrentMap> pools = new ConcurrentHashMap(); + + public TTransportPool(){} + + private void add(SocketAddress remote, PooledObjectFactory transportFactory) + { + pools.put(remote, new GenericObjectPool(transportFactory)); + } + + protected TTransport get(SocketAddress remote, PooledObjectFactory transportFactory) + { + ObjectPool pool = pools.get(remote) + if (pool == null) + { + add(remote, transportFactory); + pool = pools.get(remote); + } + + return pool.borrowObject(); + } + + protected TTransport get(SocketAddress remote) + { + ObjectPool pool = pools.get(remote) + if (pool == null) + { + return null; + } + + return pool.borrowObject(); + } + + public TTransport borrowObject(String host, int port, PooledObjectFactory transportFactory) + { + return get(InetSocketAddress.createUnresolved(host, port), transportFactory); + } + + public TTransport borrowObject(String host, int port) + { + return get(InetSocketAddress.createUnresolved(host, port)); + } + + private static void closeQuietly(Closeable closeable) + { + try { + closeable.close(); + } + catch (IOException e) { + // ignored + } + } + + public void returnObject(TSocket socket) + { + SocketAddress remote = socket.getSocket().getRemoteSocketAddress() + if (remote == null) + { + return closeQuietly(socket); + } + ObjectPool pool = pools.get(remote); + if (pool == null) + { + return closeQuietly(socket); + } + pool.returnObject(socket); + } + + public void returnObject(TTransport transport) + { + return closeQuietly(transport); + } +} From 46031f8e841341e609813b7ddb97f810e54f5fbf Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 8 Dec 2016 18:19:21 -0800 Subject: [PATCH 117/151] Added PooledTTransportFactory --- .../PooledHiveMetastoreClientFactory.java | 2 +- .../hive/util/PooledTTransportFactory.java | 174 ++++++++++++++++-- .../twitter/hive/util/TTransportPool.java | 2 +- 3 files changed, 164 insertions(+), 14 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index cea665466ff1..5823d257017f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -56,7 +56,7 @@ public HiveMetastoreClient create(String host, int port) try { TTransport transport= transportPool.borrowObject(host, port); if (transport == null) { - transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(host, port, socksProxy, timeoutMillis, metastoreAuthentication)); + transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(transportPool, host, port, socksProxy, timeoutMillis, metastoreAuthentication)); } } catch (Exception e) { diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 5e04ed9f64a6..9cda24470d1d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -14,25 +14,175 @@ package com.facebook.presto.twitter.hive.util; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; - +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; import javax.annotation.Nullable; - -import java.io.IOException; -import java.net.InetSocketAddress -import java.net.SocketAddress; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import org.apache.commons.pool2.ObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; /** * Utility class to handle creating and caching the UserGroupInformation object. */ public class PooledTTransportFactory + extends BasePooledObjectFactory { - + private final TTransportPool pool; + private final String host; + private final int port; + private final HostAndPort socksProxy; + private final Duration timeoutMillis; + private final HiveMetastoreAuthentication metastoreAuthentication; + + public PooledTTransportFactory(TTransportPool pool, String host, int port, @Nullable HostAndPort socksProxy, Duration timeoutMillis, HiveMetastoreAuthentication metastoreAuthentication) + { + this.pool = requireNonNull(pool, "pool is null"); + this.host = requireNonNull(host, "host is null"); + this.port = port; + this.socksProxy = socksProxy; + this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); + } + + @Override + public TTransport create() + { + return new PooledTTransport(Transport.create(host, port, socksProxy, timeoutMillis, metastoreAuthentication), pool); + } + + @Override + public void destroyObject(PooledObject pooledTransport) + { + try { + ((PooledTTransport) pooledTransport.getObject()).getTTransport().close(); + } + catch (TTransportException e) { + // ignored + } + pooledTransport.invalidate(); + } + + @Override + public PooledObject wrap(TTransport transport) + throws TTransportException + { + return new DefaultPooledObject(transport); + } + + @Override + public void passivateObject(PooledObject pooledTransport) + { + try { + pooledTransport.getObject().flush(); + } + catch (TTransportException e) { + destroyObject(pooledTransport); + } + } + + private static class PooledTTransport + extends TTransport + { + private final TTransportPool pool; + private final TTransport transport; + + public PooledTTransport(TTransport transport, TTransportPool, pool) + { + this.transport = transport; + this.pool = pool; + } + + @Override + public void close() + { + pool.returnObject((TSocket) transport); + } + + @Override + public TTransport.getTTransport() + { + return transport; + } + + @Override + public boolean isOpen() + { + return transport.isOpen(); + } + + @Override + public boolean peek() + { + return transport.peek(); + } + + @Override + public byte[] getBuffer() + { + return transport.getBuffer(); + } + + @Override + public int getBufferPosition() + { + return transport.getBufferPosition(); + } + + @Override + public int getBytesRemainingInBuffer() + { + return transport.getBytesRemainingInBuffer(); + } + + @Override + public void consumeBuffer(int len) + { + transport.consumeBuffer(len); + } + + @Override + public void open() + throws TTransportException + { + transport.open(); + } + + @Override + public int readAll(byte[] bytes, int off, int len) + throws TTransportException + { + return transport.readAll(bytes, off, len); + } + + @Override + public int read(byte[] bytes, int off, int len) + throws TTransportException + { + return transport.read(bytes, off, len); + } + + @Override + public void write(byte[] bytes) + throws TTransportException + { + transport.write(bytes); + } + + @Override + public void write(byte[] bytes, int off, int len) + throws TTransportException + { + transport.write(bytes, off, len); + } + + @Override + public void flush() + throws TTransportException + { + transport.flush(); + } + } + } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index b148404a7ff6..ad189d95e7a8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -18,7 +18,7 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.net.InetSocketAddress +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; From 94991f204aff8efb8d84e5fa61fa47f0dfdfeb5d Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 8 Dec 2016 23:05:24 -0800 Subject: [PATCH 118/151] Correct errors --- pom.xml | 6 ++ presto-hive/pom.xml | 5 ++ .../presto/hive/HiveClientModule.java | 2 + .../PooledHiveMetastoreClientFactory.java | 7 ++- .../hive/ZookeeperServersetHiveCluster.java | 5 +- .../hive/util/PooledTTransportFactory.java | 44 ++++++------- .../twitter/hive/util/TTransportPool.java | 62 ++++++++----------- .../presto/hive/TestingHiveCluster.java | 3 +- 8 files changed, 71 insertions(+), 63 deletions(-) diff --git a/pom.xml b/pom.xml index 77381e4f9179..6458b946f779 100644 --- a/pom.xml +++ b/pom.xml @@ -644,6 +644,12 @@ 3.2 + + org.apache.commons + commons-pool2 + 2.4.2 + + commons-codec commons-codec diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 94c8f9f305e7..7eef88e3c083 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -96,6 +96,11 @@ libthrift + + org.apache.commons + commons-pool2 + + io.airlift aircompressor diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index 25de3499bccc..7a187f48015c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.twitter.hive.PooledHiveMetastoreClientFactory; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; @@ -99,6 +100,7 @@ public void configure(Binder binder) newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class)); binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); + binder.bind(PooledHiveMetastoreClientFactory.class).in(Scopes.SINGLETON); binder.bind(NodeManager.class).toInstance(nodeManager); binder.bind(TypeManager.class).toInstance(typeManager); diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index 5823d257017f..41fad29fa09b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -35,6 +35,7 @@ public class PooledHiveMetastoreClientFactory private final HostAndPort socksProxy; private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; + private final TTransportPool transportPool; public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, HiveMetastoreAuthentication metastoreAuthentication) { @@ -54,14 +55,14 @@ public HiveMetastoreClient create(String host, int port) throws TTransportException { try { - TTransport transport= transportPool.borrowObject(host, port); + TTransport transport = transportPool.borrowObject(host, port); if (transport == null) { transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(transportPool, host, port, socksProxy, timeoutMillis, metastoreAuthentication)); } + return new ThriftHiveMetastoreClient(transport); } catch (Exception e) { - throw new TTransportException(e.getType(), String.format("%s: %s", host, e.getMessage()), e.getCause()) + throw new TTransportException(String.format("%s: %s", host, e.getMessage()), e.getCause()); } - return new ThriftHiveMetastoreClient(transport); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java index 379352d4100e..c4f803311b80 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -14,7 +14,6 @@ package com.facebook.presto.twitter.hive; import com.facebook.presto.hive.HiveCluster; -import com.facebook.presto.hive.HiveMetastoreClientFactory; import com.facebook.presto.hive.metastore.HiveMetastoreClient; import com.google.common.net.HostAndPort; import io.airlift.log.Logger; @@ -31,11 +30,11 @@ public class ZookeeperServersetHiveCluster implements HiveCluster { private static final Logger log = Logger.get(ZookeeperServersetHiveCluster.class); - private final HiveMetastoreClientFactory clientFactory; + private final PooledHiveMetastoreClientFactory clientFactory; private ZookeeperMetastoreMonitor zkMetastoreMonitor; @Inject - public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, HiveMetastoreClientFactory clientFactory) + public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, PooledHiveMetastoreClientFactory clientFactory) throws Exception { String zkServerHostAndPort = requireNonNull(config.getZookeeperServerHostAndPort(), "zkServerHostAndPort is null"); diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 9cda24470d1d..ed465d89503f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -14,16 +14,19 @@ package com.facebook.presto.twitter.hive.util; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; +import com.facebook.presto.hive.thrift.Transport; import com.google.common.net.HostAndPort; -import io.airlift.units.Duration; -import javax.annotation.Nullable; -import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import javax.annotation.Nullable; + +import static java.util.Objects.requireNonNull; + /** * Utility class to handle creating and caching the UserGroupInformation object. */ @@ -34,51 +37,52 @@ public class PooledTTransportFactory private final String host; private final int port; private final HostAndPort socksProxy; - private final Duration timeoutMillis; + private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; - public PooledTTransportFactory(TTransportPool pool, String host, int port, @Nullable HostAndPort socksProxy, Duration timeoutMillis, HiveMetastoreAuthentication metastoreAuthentication) + public PooledTTransportFactory(TTransportPool pool, String host, int port, @Nullable HostAndPort socksProxy, int timeoutMillis, HiveMetastoreAuthentication metastoreAuthentication) { this.pool = requireNonNull(pool, "pool is null"); this.host = requireNonNull(host, "host is null"); this.port = port; this.socksProxy = socksProxy; + this.timeoutMillis = timeoutMillis; this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); } @Override public TTransport create() + throws Exception { return new PooledTTransport(Transport.create(host, port, socksProxy, timeoutMillis, metastoreAuthentication), pool); } @Override - public void destroyObject(PooledObject pooledTransport) + public void destroyObject(PooledObject pooledObject) { try { - ((PooledTTransport) pooledTransport.getObject()).getTTransport().close(); + ((PooledTTransport) pooledObject.getObject()).getTTransport().close(); } - catch (TTransportException e) { - // ignored + catch (ClassCastException e) { + // ignore } - pooledTransport.invalidate(); + pooledObject.invalidate(); } @Override public PooledObject wrap(TTransport transport) - throws TTransportException { return new DefaultPooledObject(transport); } @Override - public void passivateObject(PooledObject pooledTransport) + public void passivateObject(PooledObject pooledObject) { try { - pooledTransport.getObject().flush(); + pooledObject.getObject().flush(); } catch (TTransportException e) { - destroyObject(pooledTransport); + destroyObject(pooledObject); } } @@ -88,22 +92,21 @@ private static class PooledTTransport private final TTransportPool pool; private final TTransport transport; - public PooledTTransport(TTransport transport, TTransportPool, pool) + public PooledTTransport(TTransport transport, TTransportPool pool) { this.transport = transport; this.pool = pool; } - @Override - public void close() + public TTransport getTTransport() { - pool.returnObject((TSocket) transport); + return transport; } @Override - public TTransport.getTTransport() + public void close() { - return transport; + pool.returnObject((TSocket) transport); } @Override @@ -184,5 +187,4 @@ public void flush() transport.flush(); } } - } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index ad189d95e7a8..cf1b4a432592 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -13,22 +13,17 @@ */ package com.facebook.presto.twitter.hive.util; -import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; - -import javax.annotation.Nullable; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; -import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import org.apache.commons.pool2.ObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; - /** * Utility class to handle creating and caching the UserGroupInformation object. */ @@ -44,10 +39,10 @@ private void add(SocketAddress remote, PooledObjectFactory transportFactory) } protected TTransport get(SocketAddress remote, PooledObjectFactory transportFactory) + throws Exception { - ObjectPool pool = pools.get(remote) - if (pool == null) - { + ObjectPool pool = pools.get(remote); + if (pool == null) { add(remote, transportFactory); pool = pools.get(remote); } @@ -56,10 +51,10 @@ protected TTransport get(SocketAddress remote, PooledObjectFactory transportFact } protected TTransport get(SocketAddress remote) + throws Exception { - ObjectPool pool = pools.get(remote) - if (pool == null) - { + ObjectPool pool = pools.get(remote); + if (pool == null) { return null; } @@ -67,42 +62,39 @@ protected TTransport get(SocketAddress remote) } public TTransport borrowObject(String host, int port, PooledObjectFactory transportFactory) + throws Exception { return get(InetSocketAddress.createUnresolved(host, port), transportFactory); } public TTransport borrowObject(String host, int port) + throws Exception { return get(InetSocketAddress.createUnresolved(host, port)); } - private static void closeQuietly(Closeable closeable) - { - try { - closeable.close(); - } - catch (IOException e) { - // ignored - } - } - public void returnObject(TSocket socket) { - SocketAddress remote = socket.getSocket().getRemoteSocketAddress() - if (remote == null) - { - return closeQuietly(socket); + SocketAddress remote = socket.getSocket().getRemoteSocketAddress(); + if (remote == null) { + socket.close(); + return; } ObjectPool pool = pools.get(remote); - if (pool == null) - { - return closeQuietly(socket); + if (pool == null) { + socket.close(); + return; + } + try { + pool.returnObject(socket); + } + catch (Exception e) { + // ignored } - pool.returnObject(socket); } public void returnObject(TTransport transport) { - return closeQuietly(transport); + transport.close(); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java index c35f682a03c8..6b8ac2370e90 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.authentication.NoHiveMetastoreAuthentication; import com.facebook.presto.hive.metastore.HiveMetastoreClient; +import com.facebook.presto.twitter.hive.PooledHiveMetastoreClientFactory; import com.google.common.base.Throwables; import org.apache.thrift.transport.TTransportException; @@ -40,7 +41,7 @@ public TestingHiveCluster(HiveClientConfig config, String host, int port) public HiveMetastoreClient createMetastoreClient() { try { - return new HiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(host, port); + return new PooledHiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(host, port); } catch (TTransportException e) { throw Throwables.propagate(e); From 0f1a82669a640d9b22bd58fb56b15c06515f91df Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 9 Dec 2016 02:17:25 -0800 Subject: [PATCH 119/151] Remove Transport in PooledObjectFactory --- .../hive/util/PooledTTransportFactory.java | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index ed465d89503f..2a87a19e7f2d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -14,7 +14,6 @@ package com.facebook.presto.twitter.hive.util; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; -import com.facebook.presto.hive.thrift.Transport; import com.google.common.net.HostAndPort; import org.apache.commons.pool2.BasePooledObjectFactory; import org.apache.commons.pool2.PooledObject; @@ -25,6 +24,13 @@ import javax.annotation.Nullable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; + import static java.util.Objects.requireNonNull; /** @@ -54,7 +60,36 @@ public PooledTTransportFactory(TTransportPool pool, String host, int port, @Null public TTransport create() throws Exception { - return new PooledTTransport(Transport.create(host, port, socksProxy, timeoutMillis, metastoreAuthentication), pool); + TTransport transport; + if (socksProxy == null) { + transport = new TSocket(host, port, timeoutMillis); + } + else { + SocketAddress address = InetSocketAddress.createUnresolved(socksProxy.getHostText(), socksProxy.getPort()); + Socket socket = new Socket(new Proxy(Proxy.Type.SOCKS, address)); + try { + socket.connect(InetSocketAddress.createUnresolved(host, port), timeoutMillis); + socket.setSoTimeout(timeoutMillis); + transport = new TSocket(socket); + } + catch (SocketException e) { + if (socket.isConnected()) { + try { + socket.close(); + } + catch (IOException ioEexception) { + // ignored + } + } + throw e; + } + } + TTransport authenticatedTransport = metastoreAuthentication.authenticate(transport, host); + if (!authenticatedTransport.isOpen()) { + authenticatedTransport.open(); + } + + return new PooledTTransport(authenticatedTransport, pool); } @Override @@ -106,7 +141,7 @@ public TTransport getTTransport() @Override public void close() { - pool.returnObject((TSocket) transport); + pool.returnObject(transport); } @Override From 65e2bea910311f4921dd830be29250faf53cb6d9 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 9 Dec 2016 11:20:00 -0800 Subject: [PATCH 120/151] Added logs --- .../hive/PooledHiveMetastoreClientFactory.java | 3 +++ .../twitter/hive/util/PooledTTransportFactory.java | 10 ++++++++++ .../presto/twitter/hive/util/TTransportPool.java | 13 +++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index 41fad29fa09b..87efde6a62fb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -21,6 +21,7 @@ import com.facebook.presto.twitter.hive.util.TTransportPool; import com.google.common.net.HostAndPort; import com.google.common.primitives.Ints; +import io.airlift.log.Logger; import io.airlift.units.Duration; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -32,6 +33,7 @@ public class PooledHiveMetastoreClientFactory { + private static final Logger log = Logger.get(PooledHiveMetastoreClientFactory.class); private final HostAndPort socksProxy; private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; @@ -59,6 +61,7 @@ public HiveMetastoreClient create(String host, int port) if (transport == null) { transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(transportPool, host, port, socksProxy, timeoutMillis, metastoreAuthentication)); } + log.debug("borrowed a transport for host: %s", host); return new ThriftHiveMetastoreClient(transport); } catch (Exception e) { diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 2a87a19e7f2d..ecd0c9b7b176 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; import org.apache.commons.pool2.BasePooledObjectFactory; import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.impl.DefaultPooledObject; @@ -39,6 +40,7 @@ public class PooledTTransportFactory extends BasePooledObjectFactory { + private static final Logger log = Logger.get(PooledTTransportFactory.class); private final TTransportPool pool; private final String host; private final int port; @@ -60,6 +62,7 @@ public PooledTTransportFactory(TTransportPool pool, String host, int port, @Null public TTransport create() throws Exception { + log.debug("creating a transport to: %s", host); TTransport transport; if (socksProxy == null) { transport = new TSocket(host, port, timeoutMillis); @@ -89,16 +92,19 @@ public TTransport create() authenticatedTransport.open(); } + log.debug("created a transport to: %s", host); return new PooledTTransport(authenticatedTransport, pool); } @Override public void destroyObject(PooledObject pooledObject) { + log.debug("destroy a transport to: %s", host); try { ((PooledTTransport) pooledObject.getObject()).getTTransport().close(); } catch (ClassCastException e) { + log.debug("cannot cast to PooledTTransport"); // ignore } pooledObject.invalidate(); @@ -107,16 +113,19 @@ public void destroyObject(PooledObject pooledObject) @Override public PooledObject wrap(TTransport transport) { + log.debug("wrapping a transport to %s", host); return new DefaultPooledObject(transport); } @Override public void passivateObject(PooledObject pooledObject) { + log.debug("passivate a transport to %s", host); try { pooledObject.getObject().flush(); } catch (TTransportException e) { + log.debug("Failed to flush transport, destroy it"); destroyObject(pooledObject); } } @@ -141,6 +150,7 @@ public TTransport getTTransport() @Override public void close() { + log.debug("attempt to close a PooledTTransport, returning it to pool."); pool.returnObject(transport); } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index cf1b4a432592..001f7fe03bbc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.twitter.hive.util; +import io.airlift.log.Logger; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; import org.apache.commons.pool2.impl.GenericObjectPool; @@ -29,12 +30,14 @@ */ public class TTransportPool { + private static final Logger log = Logger.get(TTransportPool.class); private final ConcurrentMap> pools = new ConcurrentHashMap(); public TTransportPool(){} private void add(SocketAddress remote, PooledObjectFactory transportFactory) { + log.debug("Added new pool for destination: %s", remote); pools.put(remote, new GenericObjectPool(transportFactory)); } @@ -46,7 +49,7 @@ protected TTransport get(SocketAddress remote, PooledObjectFactory transportFact add(remote, transportFactory); pool = pools.get(remote); } - + log.debug("Fetched transport pool for : %s", remote); return pool.borrowObject(); } @@ -55,9 +58,11 @@ protected TTransport get(SocketAddress remote) { ObjectPool pool = pools.get(remote); if (pool == null) { + log.debug("Doesn't have transport for : %s", remote); return null; } + log.debug("Fetched transport pool for : %s", remote); return pool.borrowObject(); } @@ -76,25 +81,29 @@ public TTransport borrowObject(String host, int port) public void returnObject(TSocket socket) { SocketAddress remote = socket.getSocket().getRemoteSocketAddress(); + log.debug("Return a socket to: %s", remote); if (remote == null) { socket.close(); + log.debug("Remote is null"); return; } ObjectPool pool = pools.get(remote); if (pool == null) { socket.close(); + log.debug("Cannot find pool"); return; } try { pool.returnObject(socket); } catch (Exception e) { - // ignored + log.debug("Got an error when return to pool: %s", e.getMessage()); } } public void returnObject(TTransport transport) { + log.debug("Return a transport, close directly"); transport.close(); } } From 090cd4a253d0fc30735b8585d5d00419a4bf29f9 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 9 Dec 2016 11:55:29 -0800 Subject: [PATCH 121/151] Try to cast class before return --- .../presto/twitter/hive/util/PooledTTransportFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index ecd0c9b7b176..08252220a84a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -151,7 +151,12 @@ public TTransport getTTransport() public void close() { log.debug("attempt to close a PooledTTransport, returning it to pool."); - pool.returnObject(transport); + try { + pool.returnObject((TSocket) transport); + } + catch (ClassCastException e) { + pool.returnObject(transport); + } } @Override From 30b6d592b77bfabeba00ca6e18f7cece1b5056ec Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 9 Dec 2016 15:18:15 -0800 Subject: [PATCH 122/151] Changed mapping key as String --- .../hive/util/PooledTTransportFactory.java | 8 +++--- .../twitter/hive/util/TTransportPool.java | 25 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 08252220a84a..3f91cca813c8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -93,7 +93,7 @@ public TTransport create() } log.debug("created a transport to: %s", host); - return new PooledTTransport(authenticatedTransport, pool); + return new PooledTTransport(authenticatedTransport, pool, HostAndPort.fromParts(host, port).toString()); } @Override @@ -133,13 +133,15 @@ public void passivateObject(PooledObject pooledObject) private static class PooledTTransport extends TTransport { + private final String remote; private final TTransportPool pool; private final TTransport transport; - public PooledTTransport(TTransport transport, TTransportPool pool) + public PooledTTransport(TTransport transport, TTransportPool pool, String remote) { this.transport = transport; this.pool = pool; + this.remote = remote; } public TTransport getTTransport() @@ -152,7 +154,7 @@ public void close() { log.debug("attempt to close a PooledTTransport, returning it to pool."); try { - pool.returnObject((TSocket) transport); + pool.returnObject(remote, (TSocket) transport); } catch (ClassCastException e) { pool.returnObject(transport); diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 001f7fe03bbc..6b76b591eb3c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.twitter.hive.util; +import com.google.common.net.HostAndPort; import io.airlift.log.Logger; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; @@ -20,8 +21,6 @@ import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; -import java.net.InetSocketAddress; -import java.net.SocketAddress; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -31,17 +30,17 @@ public class TTransportPool { private static final Logger log = Logger.get(TTransportPool.class); - private final ConcurrentMap> pools = new ConcurrentHashMap(); + private final ConcurrentMap> pools = new ConcurrentHashMap(); public TTransportPool(){} - private void add(SocketAddress remote, PooledObjectFactory transportFactory) + private void add(String remote, PooledObjectFactory transportFactory) { log.debug("Added new pool for destination: %s", remote); pools.put(remote, new GenericObjectPool(transportFactory)); } - protected TTransport get(SocketAddress remote, PooledObjectFactory transportFactory) + protected TTransport get(String remote, PooledObjectFactory transportFactory) throws Exception { ObjectPool pool = pools.get(remote); @@ -53,7 +52,7 @@ protected TTransport get(SocketAddress remote, PooledObjectFactory transportFact return pool.borrowObject(); } - protected TTransport get(SocketAddress remote) + protected TTransport get(String remote) throws Exception { ObjectPool pool = pools.get(remote); @@ -69,18 +68,17 @@ protected TTransport get(SocketAddress remote) public TTransport borrowObject(String host, int port, PooledObjectFactory transportFactory) throws Exception { - return get(InetSocketAddress.createUnresolved(host, port), transportFactory); + return get(HostAndPort.fromParts(host, port).toString(), transportFactory); } public TTransport borrowObject(String host, int port) throws Exception { - return get(InetSocketAddress.createUnresolved(host, port)); + return get(HostAndPort.fromParts(host, port).toString()); } - public void returnObject(TSocket socket) + public void returnObject(String remote, TSocket socket) { - SocketAddress remote = socket.getSocket().getRemoteSocketAddress(); log.debug("Return a socket to: %s", remote); if (remote == null) { socket.close(); @@ -101,9 +99,14 @@ public void returnObject(TSocket socket) } } + public void returnObject(String remote, TTransport transport) + { + returnObject(transport); + } + public void returnObject(TTransport transport) { - log.debug("Return a transport, close directly"); + log.debug("Return a ttransport, close directly"); transport.close(); } } From 646106a717cce9e61da3459b279b76eb2f00956e Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 9 Dec 2016 16:28:08 -0800 Subject: [PATCH 123/151] Removed logging --- .../PooledHiveMetastoreClientFactory.java | 3 --- .../hive/util/PooledTTransportFactory.java | 18 +++---------- .../twitter/hive/util/TTransportPool.java | 25 +++---------------- 3 files changed, 8 insertions(+), 38 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index 87efde6a62fb..41fad29fa09b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -21,7 +21,6 @@ import com.facebook.presto.twitter.hive.util.TTransportPool; import com.google.common.net.HostAndPort; import com.google.common.primitives.Ints; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -33,7 +32,6 @@ public class PooledHiveMetastoreClientFactory { - private static final Logger log = Logger.get(PooledHiveMetastoreClientFactory.class); private final HostAndPort socksProxy; private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; @@ -61,7 +59,6 @@ public HiveMetastoreClient create(String host, int port) if (transport == null) { transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(transportPool, host, port, socksProxy, timeoutMillis, metastoreAuthentication)); } - log.debug("borrowed a transport for host: %s", host); return new ThriftHiveMetastoreClient(transport); } catch (Exception e) { diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 3f91cca813c8..17b26479583e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -15,7 +15,6 @@ import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; import com.google.common.net.HostAndPort; -import io.airlift.log.Logger; import org.apache.commons.pool2.BasePooledObjectFactory; import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.impl.DefaultPooledObject; @@ -40,7 +39,6 @@ public class PooledTTransportFactory extends BasePooledObjectFactory { - private static final Logger log = Logger.get(PooledTTransportFactory.class); private final TTransportPool pool; private final String host; private final int port; @@ -62,7 +60,6 @@ public PooledTTransportFactory(TTransportPool pool, String host, int port, @Null public TTransport create() throws Exception { - log.debug("creating a transport to: %s", host); TTransport transport; if (socksProxy == null) { transport = new TSocket(host, port, timeoutMillis); @@ -80,7 +77,7 @@ public TTransport create() try { socket.close(); } - catch (IOException ioEexception) { + catch (IOException ioException) { // ignored } } @@ -92,19 +89,16 @@ public TTransport create() authenticatedTransport.open(); } - log.debug("created a transport to: %s", host); return new PooledTTransport(authenticatedTransport, pool, HostAndPort.fromParts(host, port).toString()); } @Override public void destroyObject(PooledObject pooledObject) { - log.debug("destroy a transport to: %s", host); try { ((PooledTTransport) pooledObject.getObject()).getTTransport().close(); } catch (ClassCastException e) { - log.debug("cannot cast to PooledTTransport"); // ignore } pooledObject.invalidate(); @@ -113,19 +107,16 @@ public void destroyObject(PooledObject pooledObject) @Override public PooledObject wrap(TTransport transport) { - log.debug("wrapping a transport to %s", host); return new DefaultPooledObject(transport); } @Override public void passivateObject(PooledObject pooledObject) { - log.debug("passivate a transport to %s", host); try { pooledObject.getObject().flush(); } catch (TTransportException e) { - log.debug("Failed to flush transport, destroy it"); destroyObject(pooledObject); } } @@ -152,12 +143,11 @@ public TTransport getTTransport() @Override public void close() { - log.debug("attempt to close a PooledTTransport, returning it to pool."); try { - pool.returnObject(remote, (TSocket) transport); + pool.returnObject(remote, this, transport); } - catch (ClassCastException e) { - pool.returnObject(transport); + catch (Exception e) { + transport.close(); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 6b76b591eb3c..20effdf46967 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -14,11 +14,9 @@ package com.facebook.presto.twitter.hive.util; import com.google.common.net.HostAndPort; -import io.airlift.log.Logger; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import java.util.concurrent.ConcurrentHashMap; @@ -29,14 +27,12 @@ */ public class TTransportPool { - private static final Logger log = Logger.get(TTransportPool.class); private final ConcurrentMap> pools = new ConcurrentHashMap(); public TTransportPool(){} private void add(String remote, PooledObjectFactory transportFactory) { - log.debug("Added new pool for destination: %s", remote); pools.put(remote, new GenericObjectPool(transportFactory)); } @@ -48,7 +44,6 @@ protected TTransport get(String remote, PooledObjectFactory transportFactory) add(remote, transportFactory); pool = pools.get(remote); } - log.debug("Fetched transport pool for : %s", remote); return pool.borrowObject(); } @@ -57,11 +52,9 @@ protected TTransport get(String remote) { ObjectPool pool = pools.get(remote); if (pool == null) { - log.debug("Doesn't have transport for : %s", remote); return null; } - log.debug("Fetched transport pool for : %s", remote); return pool.borrowObject(); } @@ -77,36 +70,26 @@ public TTransport borrowObject(String host, int port) return get(HostAndPort.fromParts(host, port).toString()); } - public void returnObject(String remote, TSocket socket) + public void returnObject(String remote, TTransport pooledTransport, TTransport transport) { - log.debug("Return a socket to: %s", remote); if (remote == null) { - socket.close(); - log.debug("Remote is null"); + transport.close(); return; } ObjectPool pool = pools.get(remote); if (pool == null) { - socket.close(); - log.debug("Cannot find pool"); + transport.close(); return; } try { - pool.returnObject(socket); + pool.returnObject(pooledTransport); } catch (Exception e) { - log.debug("Got an error when return to pool: %s", e.getMessage()); } } - public void returnObject(String remote, TTransport transport) - { - returnObject(transport); - } - public void returnObject(TTransport transport) { - log.debug("Return a ttransport, close directly"); transport.close(); } } From eca3b40f1a7cef02c8c3e23f7dcf2924730feecf Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 12 Dec 2016 12:33:43 -0800 Subject: [PATCH 124/151] Added validation, added log to print mean wait time for each pool --- .../hive/util/PooledTTransportFactory.java | 9 ++++--- .../twitter/hive/util/TTransportPool.java | 27 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 17b26479583e..b7896754351a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -33,9 +33,6 @@ import static java.util.Objects.requireNonNull; -/** - * Utility class to handle creating and caching the UserGroupInformation object. - */ public class PooledTTransportFactory extends BasePooledObjectFactory { @@ -56,6 +53,12 @@ public PooledTTransportFactory(TTransportPool pool, String host, int port, @Null this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); } + @Override + public boolean validateObject(PooledObject pooledObject) + { + return pooledObject.getObject().isOpen(); + } + @Override public TTransport create() throws Exception diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 20effdf46967..7836f92b5c09 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -14,26 +14,41 @@ package com.facebook.presto.twitter.hive.util; import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.thrift.transport.TTransport; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -/** - * Utility class to handle creating and caching the UserGroupInformation object. - */ public class TTransportPool { + public static final Logger log = Logger.get(TTransportPool.class); + private static final int MAX_IDLE = 8; + private static final int MIN_IDLE = 0; + private static final int MAX_TOTAL = 100; private final ConcurrentMap> pools = new ConcurrentHashMap(); + private GenericObjectPoolConfig poolConfig; - public TTransportPool(){} + public TTransportPool() + { + poolConfig = new GenericObjectPoolConfig(); + poolConfig.setMaxIdle(MAX_IDLE); + poolConfig.setMinIdle(MIN_IDLE); + poolConfig.setMaxTotal(MAX_TOTAL); + } + + public TTransportPool(GenericObjectPoolConfig poolConfig) + { + this.poolConfig = poolConfig; + } private void add(String remote, PooledObjectFactory transportFactory) { - pools.put(remote, new GenericObjectPool(transportFactory)); + pools.put(remote, new GenericObjectPool(transportFactory, poolConfig)); } protected TTransport get(String remote, PooledObjectFactory transportFactory) @@ -54,7 +69,7 @@ protected TTransport get(String remote) if (pool == null) { return null; } - + log.debug("The mean borrow wait time for %s is %d millis", remote, ((GenericObjectPool) pool).getMeanBorrowWaitTimeMillis()); return pool.borrowObject(); } From 9c0e9f2714aa5d0f15dae662f5f23a14d39e7f2a Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 12 Dec 2016 12:55:28 -0800 Subject: [PATCH 125/151] Added more values for logging --- .../facebook/presto/twitter/hive/util/TTransportPool.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 7836f92b5c09..d06635e70de5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -29,7 +29,7 @@ public class TTransportPool public static final Logger log = Logger.get(TTransportPool.class); private static final int MAX_IDLE = 8; private static final int MIN_IDLE = 0; - private static final int MAX_TOTAL = 100; + private static final int MAX_TOTAL = 128; private final ConcurrentMap> pools = new ConcurrentHashMap(); private GenericObjectPoolConfig poolConfig; @@ -69,7 +69,11 @@ protected TTransport get(String remote) if (pool == null) { return null; } - log.debug("The mean borrow wait time for %s is %d millis", remote, ((GenericObjectPool) pool).getMeanBorrowWaitTimeMillis()); + log.debug("Pool %s: mean wait time: %d millis, borrowed %d, idle %d.", + remote, + ((GenericObjectPool) pool).getMeanBorrowWaitTimeMillis(), + ((GenericObjectPool) pool).getNumActive(), + ((GenericObjectPool) pool).getNumIdle()); return pool.borrowObject(); } From 6123869f83d488dee22c22ca089dd4f4dc77be4d Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 12 Dec 2016 16:36:00 -0800 Subject: [PATCH 126/151] Set Eviction Configs --- .../twitter/hive/util/TTransportPool.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index d06635e70de5..7a435dd438cd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -14,7 +14,7 @@ package com.facebook.presto.twitter.hive.util; import com.google.common.net.HostAndPort; -import io.airlift.log.Logger; +import io.airlift.units.Duration; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; import org.apache.commons.pool2.impl.GenericObjectPool; @@ -23,22 +23,27 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; public class TTransportPool { - public static final Logger log = Logger.get(TTransportPool.class); - private static final int MAX_IDLE = 8; private static final int MIN_IDLE = 0; private static final int MAX_TOTAL = 128; + private static final int NUM_TESTS = 3; + private static final Duration IDLE_TIMEOUT = new Duration(300, TimeUnit.SECONDS); + private static final Duration EVICTION_INTERVEL = new Duration(10, TimeUnit.SECONDS); private final ConcurrentMap> pools = new ConcurrentHashMap(); private GenericObjectPoolConfig poolConfig; public TTransportPool() { poolConfig = new GenericObjectPoolConfig(); - poolConfig.setMaxIdle(MAX_IDLE); + poolConfig.setMaxIdle(MAX_TOTAL); poolConfig.setMinIdle(MIN_IDLE); poolConfig.setMaxTotal(MAX_TOTAL); + poolConfig.setMinEvictableIdleTimeMillis(IDLE_TIMEOUT.toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(EVICTION_INTERVEL.toMillis()); + poolConfig.setNumTestsPerEvictionRun(NUM_TESTS); } public TTransportPool(GenericObjectPoolConfig poolConfig) @@ -69,11 +74,6 @@ protected TTransport get(String remote) if (pool == null) { return null; } - log.debug("Pool %s: mean wait time: %d millis, borrowed %d, idle %d.", - remote, - ((GenericObjectPool) pool).getMeanBorrowWaitTimeMillis(), - ((GenericObjectPool) pool).getNumActive(), - ((GenericObjectPool) pool).getNumIdle()); return pool.borrowObject(); } @@ -104,6 +104,7 @@ public void returnObject(String remote, TTransport pooledTransport, TTransport t pool.returnObject(pooledTransport); } catch (Exception e) { + transport.close(); } } From cdc7433184b1139fde6fb18e7a3d37449d4de35b Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 13 Dec 2016 16:11:56 -0800 Subject: [PATCH 127/151] Added support to pass transport pool config --- .../PooledHiveMetastoreClientFactory.java | 17 ++++-- .../ZookeeperServersetMetastoreConfig.java | 54 +++++++++++++++++++ .../twitter/hive/util/TTransportPool.java | 4 +- .../presto/hive/TestingHiveCluster.java | 3 +- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index 41fad29fa09b..e411ffa879fe 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -22,6 +22,7 @@ import com.google.common.net.HostAndPort; import com.google.common.primitives.Ints; import io.airlift.units.Duration; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -37,18 +38,26 @@ public class PooledHiveMetastoreClientFactory private final HiveMetastoreAuthentication metastoreAuthentication; private final TTransportPool transportPool; - public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, HiveMetastoreAuthentication metastoreAuthentication) + public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, HiveMetastoreAuthentication metastoreAuthentication, + int maxTransport, long idleTimeout, long transportEvictInterval, int evictNumTests) { this.socksProxy = socksProxy; this.timeoutMillis = Ints.checkedCast(timeout.toMillis()); this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); - this.transportPool = new TTransportPool(); + GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); + poolConfig.setMaxIdle(maxTransport); + poolConfig.setMaxTotal(maxTransport); + poolConfig.setMinEvictableIdleTimeMillis(idleTimeout); + poolConfig.setTimeBetweenEvictionRunsMillis(transportEvictInterval); + poolConfig.setNumTestsPerEvictionRun(evictNumTests); + this.transportPool = new TTransportPool(poolConfig); } @Inject - public PooledHiveMetastoreClientFactory(HiveClientConfig config, HiveMetastoreAuthentication metastoreAuthentication) + public PooledHiveMetastoreClientFactory(HiveClientConfig config, ZookeeperServersetMetastoreConfig zkConfig, HiveMetastoreAuthentication metastoreAuthentication) { - this(config.getMetastoreSocksProxy(), config.getMetastoreTimeout(), metastoreAuthentication); + this(config.getMetastoreSocksProxy(), config.getMetastoreTimeout(), metastoreAuthentication, + zkConfig.getMaxTransport(), zkConfig.getTransportIdleTimeout(), zkConfig.getTransportEvictInterval(), zkConfig.getTransportEvictNumTests()); } public HiveMetastoreClient create(String host, int port) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java index 26e36b469d0c..65b424b6c437 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java @@ -25,6 +25,10 @@ public class ZookeeperServersetMetastoreConfig private String zookeeperMetastorePath; private int zookeeperRetrySleepTime = 500; // ms private int zookeeperMaxRetries = 3; + private int maxTransport = 128; + private long transportIdleTimeout = 300_000L; + private long transportEvictInterval = 10_000L; + private int transportEvictNumTests = 3; public String getZookeeperServerHostAndPort() { @@ -79,4 +83,54 @@ public ZookeeperServersetMetastoreConfig setZookeeperMaxRetries(int zookeeperMax this.zookeeperMaxRetries = zookeeperMaxRetries; return this; } + + @Min(1) + public int getMaxTransport() + { + return maxTransport; + } + + @Config("hive.metastore.max-transport-num") + public ZookeeperServersetMetastoreConfig setMaxTransport(int maxTransport) + { + this.maxTransport = maxTransport; + return this; + } + + public long getTransportIdleTimeout() + { + return transportIdleTimeout; + } + + @Config("hive.metastore.transport-idle-timeout") + public ZookeeperServersetMetastoreConfig setTransportIdleTimeout(long transportIdleTimeout) + { + this.transportIdleTimeout = transportIdleTimeout; + return this; + } + + public long getTransportEvictInterval() + { + return transportEvictInterval; + } + + @Config("hive.metastore.transport-eviction-interval") + public ZookeeperServersetMetastoreConfig setTransportEvictInterval(long transportEvictInterval) + { + this.transportEvictInterval = transportEvictInterval; + return this; + } + + @Min(0) + public int getTransportEvictNumTests() + { + return transportEvictNumTests; + } + + @Config("hive.metastore.transport-eviction-num-tests") + public ZookeeperServersetMetastoreConfig setTransportEvictNumTests(int transportEvictNumTests) + { + this.transportEvictNumTests = transportEvictNumTests; + return this; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 7a435dd438cd..cbb5c58333f1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -31,7 +31,7 @@ public class TTransportPool private static final int MAX_TOTAL = 128; private static final int NUM_TESTS = 3; private static final Duration IDLE_TIMEOUT = new Duration(300, TimeUnit.SECONDS); - private static final Duration EVICTION_INTERVEL = new Duration(10, TimeUnit.SECONDS); + private static final Duration EVICTION_INTERVAL = new Duration(10, TimeUnit.SECONDS); private final ConcurrentMap> pools = new ConcurrentHashMap(); private GenericObjectPoolConfig poolConfig; @@ -42,7 +42,7 @@ public TTransportPool() poolConfig.setMinIdle(MIN_IDLE); poolConfig.setMaxTotal(MAX_TOTAL); poolConfig.setMinEvictableIdleTimeMillis(IDLE_TIMEOUT.toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(EVICTION_INTERVEL.toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(EVICTION_INTERVAL.toMillis()); poolConfig.setNumTestsPerEvictionRun(NUM_TESTS); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java index 6b8ac2370e90..c35f682a03c8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingHiveCluster.java @@ -15,7 +15,6 @@ import com.facebook.presto.hive.authentication.NoHiveMetastoreAuthentication; import com.facebook.presto.hive.metastore.HiveMetastoreClient; -import com.facebook.presto.twitter.hive.PooledHiveMetastoreClientFactory; import com.google.common.base.Throwables; import org.apache.thrift.transport.TTransportException; @@ -41,7 +40,7 @@ public TestingHiveCluster(HiveClientConfig config, String host, int port) public HiveMetastoreClient createMetastoreClient() { try { - return new PooledHiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(host, port); + return new HiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(host, port); } catch (TTransportException e) { throw Throwables.propagate(e); From 604d9d2d65aa5b5358a79789da7ae42d5365ffea Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 13 Dec 2016 17:57:43 -0800 Subject: [PATCH 128/151] Update ZKMConfig test set --- .../TestZookeeperServersetMetastoreConfig.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java index 6992a752dea1..b839a8d15404 100644 --- a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java @@ -31,7 +31,11 @@ public void testDefaults() .setZookeeperMaxRetries(3) .setZookeeperRetrySleepTime(500) .setZookeeperMetastorePath(null) - .setZookeeperServerHostAndPort(null)); + .setZookeeperServerHostAndPort(null) + .setMaxTransport(128) + .setTransportIdleTimeout(300_000L) + .setTransportEvictInterval(10_000L) + .setTransportEvictNumTests(3)); } @Test @@ -42,13 +46,21 @@ public void testExplicitPropertyMappingsSingleMetastore() .put("hive.metastore.zookeeper.path", "/zookeeper/path/") .put("hive.metastore.zookeeper.retry.sleeptime", "200") .put("hive.metastore.zookeeper.max.retries", "2") + .put("hive.metastore.max-transport-num", "64") + .put("hive.metastore.transport-idle-timeout", "100000") + .put("hive.metastore.transport-eviction-interval", "1000") + .put("hive.metastore.transport-eviction-num-tests", "10") .build(); ZookeeperServersetMetastoreConfig expected = new ZookeeperServersetMetastoreConfig() .setZookeeperServerHostAndPort("localhost:2181") .setZookeeperMetastorePath("/zookeeper/path/") .setZookeeperRetrySleepTime(200) - .setZookeeperMaxRetries(2); + .setZookeeperMaxRetries(2) + .setMaxTransport(64) + .setTransportIdleTimeout(100_000L) + .setTransportEvictInterval(1_000L) + .setTransportEvictNumTests(10); assertFullMapping(properties, expected); } From 602acb029d7251d79bfff1bf61e0487bf3107b96 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 13 Dec 2016 21:42:17 -0800 Subject: [PATCH 129/151] Enforce thread safety --- .../presto/twitter/hive/util/TTransportPool.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index cbb5c58333f1..6042a484bfe5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -51,20 +51,16 @@ public TTransportPool(GenericObjectPoolConfig poolConfig) this.poolConfig = poolConfig; } - private void add(String remote, PooledObjectFactory transportFactory) + protected synchronized void add(String remote, PooledObjectFactory transportFactory) { - pools.put(remote, new GenericObjectPool(transportFactory, poolConfig)); + pools.putIfAbsent(remote, new GenericObjectPool(transportFactory, poolConfig)); } protected TTransport get(String remote, PooledObjectFactory transportFactory) throws Exception { - ObjectPool pool = pools.get(remote); - if (pool == null) { - add(remote, transportFactory); - pool = pools.get(remote); - } - return pool.borrowObject(); + add(remote, transportFactory); + return get(remote); } protected TTransport get(String remote) From 518c27edf249d60e569a033460649382cf278571 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Wed, 14 Dec 2016 13:42:34 -0800 Subject: [PATCH 130/151] Remove default constructor --- .../twitter/hive/util/TTransportPool.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java index 6042a484bfe5..0e34f6756f1c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -14,7 +14,6 @@ package com.facebook.presto.twitter.hive.util; import com.google.common.net.HostAndPort; -import io.airlift.units.Duration; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.PooledObjectFactory; import org.apache.commons.pool2.impl.GenericObjectPool; @@ -23,29 +22,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; public class TTransportPool { - private static final int MIN_IDLE = 0; - private static final int MAX_TOTAL = 128; - private static final int NUM_TESTS = 3; - private static final Duration IDLE_TIMEOUT = new Duration(300, TimeUnit.SECONDS); - private static final Duration EVICTION_INTERVAL = new Duration(10, TimeUnit.SECONDS); private final ConcurrentMap> pools = new ConcurrentHashMap(); private GenericObjectPoolConfig poolConfig; - public TTransportPool() - { - poolConfig = new GenericObjectPoolConfig(); - poolConfig.setMaxIdle(MAX_TOTAL); - poolConfig.setMinIdle(MIN_IDLE); - poolConfig.setMaxTotal(MAX_TOTAL); - poolConfig.setMinEvictableIdleTimeMillis(IDLE_TIMEOUT.toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(EVICTION_INTERVAL.toMillis()); - poolConfig.setNumTestsPerEvictionRun(NUM_TESTS); - } - public TTransportPool(GenericObjectPoolConfig poolConfig) { this.poolConfig = poolConfig; From e2bc69238e5fda34e713051d41f076918bcf6fba Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 15 Dec 2016 11:38:16 -0800 Subject: [PATCH 131/151] Apply 100 ruler --- .../PooledHiveMetastoreClientFactory.java | 21 ++++++++++++++----- .../hive/util/PooledTTransportFactory.java | 10 ++++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java index e411ffa879fe..483a736759b0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -38,7 +38,8 @@ public class PooledHiveMetastoreClientFactory private final HiveMetastoreAuthentication metastoreAuthentication; private final TTransportPool transportPool; - public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, HiveMetastoreAuthentication metastoreAuthentication, + public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Duration timeout, + HiveMetastoreAuthentication metastoreAuthentication, int maxTransport, long idleTimeout, long transportEvictInterval, int evictNumTests) { this.socksProxy = socksProxy; @@ -54,10 +55,17 @@ public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, Durati } @Inject - public PooledHiveMetastoreClientFactory(HiveClientConfig config, ZookeeperServersetMetastoreConfig zkConfig, HiveMetastoreAuthentication metastoreAuthentication) + public PooledHiveMetastoreClientFactory(HiveClientConfig config, + ZookeeperServersetMetastoreConfig zkConfig, + HiveMetastoreAuthentication metastoreAuthentication) { - this(config.getMetastoreSocksProxy(), config.getMetastoreTimeout(), metastoreAuthentication, - zkConfig.getMaxTransport(), zkConfig.getTransportIdleTimeout(), zkConfig.getTransportEvictInterval(), zkConfig.getTransportEvictNumTests()); + this(config.getMetastoreSocksProxy(), + config.getMetastoreTimeout(), + metastoreAuthentication, + zkConfig.getMaxTransport(), + zkConfig.getTransportIdleTimeout(), + zkConfig.getTransportEvictInterval(), + zkConfig.getTransportEvictNumTests()); } public HiveMetastoreClient create(String host, int port) @@ -66,7 +74,10 @@ public HiveMetastoreClient create(String host, int port) try { TTransport transport = transportPool.borrowObject(host, port); if (transport == null) { - transport = transportPool.borrowObject(host, port, new PooledTTransportFactory(transportPool, host, port, socksProxy, timeoutMillis, metastoreAuthentication)); + transport = transportPool.borrowObject(host, port, + new PooledTTransportFactory(transportPool, + host, port, socksProxy, + timeoutMillis, metastoreAuthentication)); } return new ThriftHiveMetastoreClient(transport); } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index b7896754351a..02ac8955c2cd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -43,7 +43,9 @@ public class PooledTTransportFactory private final int timeoutMillis; private final HiveMetastoreAuthentication metastoreAuthentication; - public PooledTTransportFactory(TTransportPool pool, String host, int port, @Nullable HostAndPort socksProxy, int timeoutMillis, HiveMetastoreAuthentication metastoreAuthentication) + public PooledTTransportFactory(TTransportPool pool, String host, int port, + @Nullable HostAndPort socksProxy, int timeoutMillis, + HiveMetastoreAuthentication metastoreAuthentication) { this.pool = requireNonNull(pool, "pool is null"); this.host = requireNonNull(host, "host is null"); @@ -68,7 +70,8 @@ public TTransport create() transport = new TSocket(host, port, timeoutMillis); } else { - SocketAddress address = InetSocketAddress.createUnresolved(socksProxy.getHostText(), socksProxy.getPort()); + SocketAddress address = InetSocketAddress.createUnresolved(socksProxy.getHostText(), + socksProxy.getPort()); Socket socket = new Socket(new Proxy(Proxy.Type.SOCKS, address)); try { socket.connect(InetSocketAddress.createUnresolved(host, port), timeoutMillis); @@ -92,7 +95,8 @@ public TTransport create() authenticatedTransport.open(); } - return new PooledTTransport(authenticatedTransport, pool, HostAndPort.fromParts(host, port).toString()); + return new PooledTTransport(authenticatedTransport, pool, + HostAndPort.fromParts(host, port).toString()); } @Override From ae66897529a2765ff154c32d98bcb323108d01b5 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 16 Dec 2016 13:06:31 -0800 Subject: [PATCH 132/151] Update release version to 0.157-tw-0.29 --- pom.xml | 4 ++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 6458b946f779..4544307a43fa 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157-tw-0.28 + 0.157-tw-0.29 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index c626c3ad70b2..25b2dfa6873b 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index a500d24dd11f..9464d398187b 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 8761d3638bc6..351ea0688410 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 73936549aa05..d13a25aad0bd 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 4607b9c36eb8..45160e8845c6 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index d7b20168f9d9..4ca2d25d5552 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8ccb6e9fab7a..4660227dace9 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index e5c8a761868a..92241b60b780 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index f49684f0083f..70ce2e31fb2c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e2e5a57835bf..784141533e83 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4f53264a9894..96d2efbf15f1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0cffa89f962b..33c7c59d1464 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 533ffeda6853..9d91549fea3f 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2af0998f777e..2daad579d330 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 03c62da1a4db..d911b3d72990 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 525324657f8d..17f770566608 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 7dfdfd71281c..c0fe52ce0229 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 7eef88e3c083..fb87edef3156 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 059491bbb6f6..d1cb20840cec 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 68f97938c645..8514c25365b5 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index e1d4ffd5735a..8781a46e9ce7 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9c387bfc64c5..c0fbc160d1c2 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 93da37fd6fc6..97ddd14fac53 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 008c82fdbe20..367ab2fc95d8 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 7087a7cdab75..ff06402a5013 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 874270eb3e39..8519ca166be6 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 40c93e82bd6e..a552f39fabbe 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a73eefd181a2..578271f897bc 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index e3e52ccc9c94..b54d6eb8a7b6 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7d6c08ae0f32..646e8ebeb0a9 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 699d996b9cf9..6095c86862f3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index fc953510fc6e..5bd2ec84bd73 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index b550cf7cfe49..62aba5012cb2 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index caa9be15c786..aa4683471ce7 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 33eabb69a5fa..cb0adca83ada 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 0dec7412e1e5..4f975bd1e7e3 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index dedf4daf4cb8..48c5116da66e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 51b1da095a93..1b0743a1c432 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8bb450c60706..59eb0ee34a0b 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2b9ec4feb53f..cca9f667d352 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8ef0f28df850..0d5b8b0e0a69 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 0923fa884822..1dd7f0988428 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9482bda29427..38b538532789 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 539a9fc3f03f..c6733e132dbe 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index a38d41663d1f..b06141de197f 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.157-tw-0.28 + 0.157-tw-0.29 provided From aba8854b43249d63f19057022fd5021f620d2128 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 20 Dec 2016 11:20:15 -0800 Subject: [PATCH 133/151] invoke isReachable to test connection --- .../hive/util/PooledTTransportFactory.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java index 02ac8955c2cd..68efa43131cb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -55,10 +55,23 @@ public PooledTTransportFactory(TTransportPool pool, String host, int port, this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); } + @Override + public void activateObject(PooledObject pooledObject) + throws Exception + { + pooledObject.getObject().flush(); + } + @Override public boolean validateObject(PooledObject pooledObject) { - return pooledObject.getObject().isOpen(); + try { + return (pooledObject.getObject().isOpen() && + ((PooledTTransport) pooledObject.getObject()).isReachable(timeoutMillis)); + } + catch (Exception e) { + return false; + } } @Override @@ -147,6 +160,12 @@ public TTransport getTTransport() return transport; } + public boolean isReachable(int timeoutMillis) + throws ClassCastException, IOException + { + return ((TSocket) transport).getSocket().getInetAddress().isReachable(timeoutMillis); + } + @Override public void close() { @@ -196,42 +215,42 @@ public void consumeBuffer(int len) @Override public void open() - throws TTransportException + throws TTransportException { transport.open(); } @Override public int readAll(byte[] bytes, int off, int len) - throws TTransportException + throws TTransportException { return transport.readAll(bytes, off, len); } @Override public int read(byte[] bytes, int off, int len) - throws TTransportException + throws TTransportException { return transport.read(bytes, off, len); } @Override public void write(byte[] bytes) - throws TTransportException + throws TTransportException { transport.write(bytes); } @Override public void write(byte[] bytes, int off, int len) - throws TTransportException + throws TTransportException { transport.write(bytes, off, len); } @Override public void flush() - throws TTransportException + throws TTransportException { transport.flush(); } From b1c48542f3340ae32381722a7c0a6b65da7b7c19 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Wed, 4 Jan 2017 13:27:36 -0800 Subject: [PATCH 134/151] switch to 0.161 tag first --- pom.xml | 4 ++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 4544307a43fa..195c7db584fe 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157-tw-0.29 + 0.161 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 25b2dfa6873b..2544cc5f1525 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index 9464d398187b..2282780a9128 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 351ea0688410..fad7d54f0588 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index d13a25aad0bd..fc8e320103e4 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 45160e8845c6..14777b2334bc 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 4ca2d25d5552..e8fc18894388 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.29 + 0.161 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 4660227dace9..5fd7c56c789d 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 92241b60b780..d997e07f4444 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 70ce2e31fb2c..8f38bb6e8cc2 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 784141533e83..82b574340661 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 96d2efbf15f1..04e58e8133d5 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 33c7c59d1464..793ae8f7cab2 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 9d91549fea3f..710a09b4c3f1 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2daad579d330..001eb372d4a1 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index d911b3d72990..f0fc34b1e084 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 17f770566608..2244deeae111 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index c0fe52ce0229..ffbda20228fa 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index fb87edef3156..f52e6f7418d1 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d1cb20840cec..e3ff898e6354 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 8514c25365b5..d3208026e086 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 8781a46e9ce7..69f7007f1706 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index c0fbc160d1c2..7cf3ed1802fa 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 97ddd14fac53..4953177ca686 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 367ab2fc95d8..7defae7dcd90 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index ff06402a5013..cd7f0a2f6558 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 8519ca166be6..fedfd06bb31e 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index a552f39fabbe..eed944e1ea55 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 578271f897bc..870d38cc3bc6 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index b54d6eb8a7b6..e4c62d424c2e 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 646e8ebeb0a9..26bc7d494653 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 6095c86862f3..3578bdedfb46 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.29 + 0.161 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 5bd2ec84bd73..556815dba842 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 62aba5012cb2..3bc78b50778c 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index aa4683471ce7..95de42244f7b 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index cb0adca83ada..45b8de7318c9 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 4f975bd1e7e3..8b2922cbfa0c 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 48c5116da66e..8e1642fff6bd 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 1b0743a1c432..c502c46cb324 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 59eb0ee34a0b..92fe25ab6793 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index cca9f667d352..0f1a27d58293 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 0d5b8b0e0a69..34263e6d85b3 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 1dd7f0988428..c9c564b034ed 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.29 + 0.161 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 38b538532789..685fc6942f52 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index c6733e132dbe..307c17576842 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index b06141de197f..c7bbeb7d0d39 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.29 + 0.161 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.157-tw-0.29 + 0.161 provided From dcb3a3948bd96d84a0244d1a7208db78c6b5ffdb Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Wed, 4 Jan 2017 13:31:03 -0800 Subject: [PATCH 135/151] revert back to old twitter tag --- pom.xml | 4 ++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 09f571764e7d..1a5bb015822c 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.161 + 0.157-tw-0.28 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 2544cc5f1525..c626c3ad70b2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index 2282780a9128..a500d24dd11f 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index fad7d54f0588..8761d3638bc6 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index fc8e320103e4..73936549aa05 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 14777b2334bc..4607b9c36eb8 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index e8fc18894388..d7b20168f9d9 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.161 + 0.157-tw-0.28 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 5fd7c56c789d..8ccb6e9fab7a 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index d997e07f4444..e5c8a761868a 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 8f38bb6e8cc2..f49684f0083f 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index 82b574340661..e2e5a57835bf 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 04e58e8133d5..4f53264a9894 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index d431d1265028..087f040d63ae 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 710a09b4c3f1..533ffeda6853 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 001eb372d4a1..2af0998f777e 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index f0fc34b1e084..03c62da1a4db 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 2244deeae111..525324657f8d 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index ffbda20228fa..7dfdfd71281c 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index f52e6f7418d1..7eef88e3c083 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index e3ff898e6354..059491bbb6f6 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index d3208026e086..68f97938c645 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 69f7007f1706..e1d4ffd5735a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 7cf3ed1802fa..9c387bfc64c5 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 4953177ca686..93da37fd6fc6 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 7defae7dcd90..008c82fdbe20 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index cd7f0a2f6558..7087a7cdab75 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index fedfd06bb31e..874270eb3e39 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index eed944e1ea55..40c93e82bd6e 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 870d38cc3bc6..a73eefd181a2 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index e4c62d424c2e..e3e52ccc9c94 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 26bc7d494653..7d6c08ae0f32 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 3ad29be3abec..4cedd7b332d3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.161 + 0.157-tw-0.28 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 556815dba842..fc953510fc6e 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 3bc78b50778c..b550cf7cfe49 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 95de42244f7b..caa9be15c786 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 45b8de7318c9..33eabb69a5fa 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 8b2922cbfa0c..0dec7412e1e5 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 8e1642fff6bd..dedf4daf4cb8 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index c502c46cb324..51b1da095a93 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 92fe25ab6793..8bb450c60706 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 0f1a27d58293..2b9ec4feb53f 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 34263e6d85b3..8ef0f28df850 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 170e4291d73b..d0ea923390cf 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.161 + 0.157-tw-0.28 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 685fc6942f52..9482bda29427 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 307c17576842..539a9fc3f03f 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index c7bbeb7d0d39..a38d41663d1f 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.161 + 0.157-tw-0.28 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.161 + 0.157-tw-0.28 provided From 0b0da52b96c4fc961b575346f8a4e17c2f86cc59 Mon Sep 17 00:00:00 2001 From: Thomas Sun Date: Thu, 5 Jan 2017 09:22:31 -0800 Subject: [PATCH 136/151] correct revert version --- pom.xml | 4 ++-- presto-accumulo/pom.xml | 2 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-blackhole/pom.xml | 2 +- presto-bytecode/pom.xml | 2 +- presto-cassandra/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-docs/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-hive-cdh4/pom.xml | 2 +- presto-hive-cdh5/pom.xml | 2 +- presto-hive-hadoop1/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mongodb/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 2 +- presto-parser/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 2 +- presto-raptor/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-redis/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 2 +- presto-spi/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- twitter-eventlistener-plugin/pom.xml | 4 ++-- 46 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pom.xml b/pom.xml index 1a5bb015822c..a321d331717c 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 pom presto-root @@ -30,7 +30,7 @@ scm:git:git://github.com/twitter-forks/presto.git https://github.com/twitter-forks/presto - 0.157-tw-0.28 + 0.157-tw-0.29 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index c626c3ad70b2..25b2dfa6873b 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index a500d24dd11f..9464d398187b 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 8761d3638bc6..351ea0688410 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 73936549aa05..d13a25aad0bd 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-base-jdbc diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 4607b9c36eb8..45160e8845c6 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index d7b20168f9d9..4ca2d25d5552 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-benchmark diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 8ccb6e9fab7a..4660227dace9 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-blackhole diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index e5c8a761868a..92241b60b780 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-bytecode diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index f49684f0083f..70ce2e31fb2c 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index e2e5a57835bf..784141533e83 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 4f53264a9894..96d2efbf15f1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 087f040d63ae..39d843f4c136 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-docs diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index 533ffeda6853..9d91549fea3f 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-example-http diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml index 2af0998f777e..2daad579d330 100644 --- a/presto-hive-cdh4/pom.xml +++ b/presto-hive-cdh4/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-cdh4 diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml index 03c62da1a4db..d911b3d72990 100644 --- a/presto-hive-cdh5/pom.xml +++ b/presto-hive-cdh5/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-cdh5 diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml index 525324657f8d..17f770566608 100644 --- a/presto-hive-hadoop1/pom.xml +++ b/presto-hive-hadoop1/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-hadoop1 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 7dfdfd71281c..c0fe52ce0229 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 7eef88e3c083..fb87edef3156 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-hive diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 059491bbb6f6..d1cb20840cec 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 68f97938c645..8514c25365b5 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index e1d4ffd5735a..8781a46e9ce7 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-kafka diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 9c387bfc64c5..c0fbc160d1c2 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-local-file diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 93da37fd6fc6..97ddd14fac53 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-main diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 008c82fdbe20..367ab2fc95d8 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 7087a7cdab75..ff06402a5013 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 874270eb3e39..8519ca166be6 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 40c93e82bd6e..a552f39fabbe 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index a73eefd181a2..578271f897bc 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-parser diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index e3e52ccc9c94..b54d6eb8a7b6 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7d6c08ae0f32..646e8ebeb0a9 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-postgresql diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 4cedd7b332d3..0225fb57e04e 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-product-tests diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index fc953510fc6e..5bd2ec84bd73 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index b550cf7cfe49..62aba5012cb2 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index caa9be15c786..aa4683471ce7 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 33eabb69a5fa..cb0adca83ada 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-redis diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 0dec7412e1e5..4f975bd1e7e3 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index dedf4daf4cb8..48c5116da66e 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 51b1da095a93..1b0743a1c432 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 8bb450c60706..59eb0ee34a0b 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-spi diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 2b9ec4feb53f..cca9f667d352 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 8ef0f28df850..0d5b8b0e0a69 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index d0ea923390cf..2d8d6e714669 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.157-tw-0.28 + 0.157-tw-0.29 presto-tests diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 9482bda29427..38b538532789 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 539a9fc3f03f..c6733e132dbe 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml index a38d41663d1f..b06141de197f 100644 --- a/twitter-eventlistener-plugin/pom.xml +++ b/twitter-eventlistener-plugin/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.157-tw-0.28 + 0.157-tw-0.29 twitter-eventlistener-plugin @@ -19,7 +19,7 @@ com.facebook.presto presto-spi - 0.157-tw-0.28 + 0.157-tw-0.29 provided From dad17c2211b44c0278661d1a65423a03c0134fbd Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Thu, 5 Jan 2017 17:16:42 -0800 Subject: [PATCH 137/151] disable ui in coordinator --- .../main/java/com/facebook/presto/server/CoordinatorModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index b5abdaa641ca..01b4e7ac15ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -130,7 +130,7 @@ public class CoordinatorModule @Override protected void setup(Binder binder) { - httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + // httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); // presto coordinator announcement discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); From 96681ee8e605f8b4381cc6c814ff0a46836ac3a5 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 6 Jan 2017 14:12:36 -0800 Subject: [PATCH 138/151] Disable ui and added ui module --- .../facebook/presto/server/CoordinatorModule.java | 2 +- .../com/facebook/presto/server/PrestoServer.java | 15 +++++++++++++++ .../com/facebook/presto/server/ServerConfig.java | 13 +++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index 01b4e7ac15ee..d6198e2ecef2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -116,7 +116,7 @@ import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -import static io.airlift.http.server.HttpServerBinder.httpServerBinder; +// import static io.airlift.http.server.HttpServerBinder.httpServerBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 7143ce5ee588..de5e0a36438d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -136,6 +136,21 @@ public void run() log.error(e); System.exit(1); } + + ImmutableList.Builder uIModules = ImmutableList.builder(); + uIModules.add(new CoordinatorUIHttpServerModule()); + + Bootstrap uIApp = new Bootstrap(uIModules.build()); + + try { + Injector injector = uIApp.strictConfig().initialize(); + + log.info("======== UI STARTED ========"); + } + catch (Throwable e) { + log.error(e); + System.exit(1); + } } private static void updateConnectorIds(Announcer announcer, CatalogManager metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java index 8c2d600893fe..eb6a9c2d4cd8 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java @@ -25,6 +25,7 @@ public class ServerConfig private String dataSources; private boolean includeExceptionInResponse = true; private Duration gracePeriod = new Duration(2, MINUTES); + private int uIHttpPort = 0; public boolean isCoordinator() { @@ -38,6 +39,18 @@ public ServerConfig setCoordinator(boolean coordinator) return this; } + public int getUIHttpPort() + { + return uIHttpPort; + } + + @Config("http-server.ui.http.port") + public ServerConfig setUIHttpPort(int uIHttpPort) + { + this.uIHttpPort = uIHttpPort; + return this; + } + public String getPrestoVersion() { return prestoVersion; From a2dbd99764f5efdec7019f15ec4c90943f6c130e Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 6 Jan 2017 14:13:02 -0800 Subject: [PATCH 139/151] Added ui module --- .../server/CoordinatorUIHttpServerModule.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java new file mode 100644 index 000000000000..999e20f5e741 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.server; + +import com.google.inject.Binder; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.discovery.client.AnnouncementHttpServerInfo; +import io.airlift.http.server.HttpRequestEvent; +import io.airlift.http.server.HttpServer; +import io.airlift.http.server.HttpServerBinder.HttpResourceBinding; +import io.airlift.http.server.HttpServerConfig; +import io.airlift.http.server.HttpServerInfo; +import io.airlift.http.server.HttpServerProvider; +import io.airlift.http.server.LocalAnnouncementHttpServerInfo; +import io.airlift.http.server.RequestStats; +import io.airlift.http.server.TheServlet; + +import javax.servlet.Filter; + +import static io.airlift.event.client.EventBinder.eventBinder; +import static io.airlift.http.server.HttpServerBinder.httpServerBinder; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class CoordinatorUIHttpServerModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + binder.disableCircularProxies(); + + binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); + newExporter(binder).export(HttpServer.class).withGeneratedName(); + binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(RequestStats.class).in(Scopes.SINGLETON); + Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); + Multibinder.newSetBinder(binder, HttpResourceBinding.class, TheServlet.class); + + newExporter(binder).export(RequestStats.class).withGeneratedName(); + + eventBinder(binder).bindEventClient(HttpRequestEvent.class); + + binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); + + ServerConfig serverConfig = buildConfigObject(ServerConfig.class); + + if (serverConfig.isCoordinator()) { + HttpServerConfig config = new HttpServerConfig().setHttpPort(serverConfig.getUIHttpPort()); + binder.bind(HttpServerConfig.class).toInstance(config); + + httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + } + } +} From bcc8c189c794d9da592a9ff5386263f9954cefda Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 6 Jan 2017 22:19:03 -0800 Subject: [PATCH 140/151] bind port --- .../server/CoordinatorUIHttpServerModule.java | 38 +++++++++++-------- .../facebook/presto/server/PrestoServer.java | 10 ++++- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 999e20f5e741..6ff92d0a86c9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -20,17 +20,19 @@ import io.airlift.discovery.client.AnnouncementHttpServerInfo; import io.airlift.http.server.HttpRequestEvent; import io.airlift.http.server.HttpServer; -import io.airlift.http.server.HttpServerBinder.HttpResourceBinding; import io.airlift.http.server.HttpServerConfig; import io.airlift.http.server.HttpServerInfo; import io.airlift.http.server.HttpServerProvider; import io.airlift.http.server.LocalAnnouncementHttpServerInfo; import io.airlift.http.server.RequestStats; +import io.airlift.http.server.TheAdminServlet; import io.airlift.http.server.TheServlet; import javax.servlet.Filter; +import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; import static io.airlift.event.client.EventBinder.eventBinder; +import static io.airlift.http.server.HttpServerBinder.HttpResourceBinding; import static io.airlift.http.server.HttpServerBinder.httpServerBinder; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -40,28 +42,32 @@ public class CoordinatorUIHttpServerModule @Override protected void setup(Binder binder) { - binder.disableCircularProxies(); - - binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); - newExporter(binder).export(HttpServer.class).withGeneratedName(); - binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); - binder.bind(RequestStats.class).in(Scopes.SINGLETON); - Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); - Multibinder.newSetBinder(binder, HttpResourceBinding.class, TheServlet.class); - - newExporter(binder).export(RequestStats.class).withGeneratedName(); - - eventBinder(binder).bindEventClient(HttpRequestEvent.class); - - binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); - ServerConfig serverConfig = buildConfigObject(ServerConfig.class); if (serverConfig.isCoordinator()) { HttpServerConfig config = new HttpServerConfig().setHttpPort(serverConfig.getUIHttpPort()); binder.bind(HttpServerConfig.class).toInstance(config); + // HttpServerConfig config = buildConfigObject(HttpServerConfig.class); + // config.setHttpPort(serverConfig.getUIHttpPort()); + + binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); + binder.disableCircularProxies(); + + Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); + Multibinder.newSetBinder(binder, Filter.class, TheAdminServlet.class); + Multibinder.newSetBinder(binder, HttpResourceBinding.class, TheServlet.class); + newExporter(binder).export(HttpServer.class).withGeneratedName(); + binder.bind(RequestStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(RequestStats.class).withGeneratedName(); + eventBinder(binder).bindEventClient(HttpRequestEvent.class); + + binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + + // presto coordinator announcement + discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator-ui"); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index de5e0a36438d..24fdbdb5a7f3 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -138,12 +138,18 @@ public void run() } ImmutableList.Builder uIModules = ImmutableList.builder(); - uIModules.add(new CoordinatorUIHttpServerModule()); + uIModules.add( + new NodeModule(), + new DiscoveryModule(), + new HttpEventModule(), + new JsonModule(), + new JaxrsModule(true), + new CoordinatorUIHttpServerModule()); Bootstrap uIApp = new Bootstrap(uIModules.build()); try { - Injector injector = uIApp.strictConfig().initialize(); + Injector injector = uIApp.doNotInitializeLogging().initialize(); log.info("======== UI STARTED ========"); } From 4dcee6a8982288405b8e0b459a2de134caf03c24 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Sat, 7 Jan 2017 18:07:45 -0800 Subject: [PATCH 141/151] Fixed json encoder --- .../server/CoordinatorUIHttpServerModule.java | 79 ++++++++++++++++++- .../facebook/presto/server/PrestoServer.java | 35 ++++---- 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 6ff92d0a86c9..1d701cdcf108 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -13,7 +13,29 @@ */ package com.facebook.presto.server; +import com.facebook.presto.block.BlockJsonSerde; +import com.facebook.presto.client.QueryResults; +import com.facebook.presto.execution.QueryInfo; +import com.facebook.presto.execution.TaskInfo; +import com.facebook.presto.execution.TaskStatus; +import com.facebook.presto.memory.MemoryInfo; +import com.facebook.presto.memory.MemoryPoolAssignmentsRequest; +import com.facebook.presto.metadata.ViewDefinition; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.Serialization.ExpressionDeserializer; +import com.facebook.presto.sql.Serialization.ExpressionSerializer; +import com.facebook.presto.sql.Serialization.FunctionCallDeserializer; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.FunctionCall; +import com.facebook.presto.transaction.TransactionManager; +import com.facebook.presto.type.TypeDeserializer; import com.google.inject.Binder; +import com.google.inject.Injector; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; @@ -27,6 +49,7 @@ import io.airlift.http.server.RequestStats; import io.airlift.http.server.TheAdminServlet; import io.airlift.http.server.TheServlet; +import io.airlift.slice.Slice; import javax.servlet.Filter; @@ -34,11 +57,22 @@ import static io.airlift.event.client.EventBinder.eventBinder; import static io.airlift.http.server.HttpServerBinder.HttpResourceBinding; import static io.airlift.http.server.HttpServerBinder.httpServerBinder; +import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class CoordinatorUIHttpServerModule extends AbstractConfigurationAwareModule { + private final Injector injector; + + public CoordinatorUIHttpServerModule(Injector injector) + { + this.injector = requireNonNull(injector, "injector is null"); + } + @Override protected void setup(Binder binder) { @@ -47,8 +81,6 @@ protected void setup(Binder binder) if (serverConfig.isCoordinator()) { HttpServerConfig config = new HttpServerConfig().setHttpPort(serverConfig.getUIHttpPort()); binder.bind(HttpServerConfig.class).toInstance(config); - // HttpServerConfig config = buildConfigObject(HttpServerConfig.class); - // config.setHttpPort(serverConfig.getUIHttpPort()); binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); @@ -66,8 +98,49 @@ protected void setup(Binder binder) httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); + binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); + binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); + binder.bind(TransactionManager.class).toInstance(injector.getInstance(TransactionManager.class)); + + jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class); + jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); + jsonCodecBinder(binder).bindJsonCodec(QueryResults.class); + jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class); + jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class); + jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class); + jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class); + jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class); + jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class); + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); + jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); + jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); + jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); + jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); + jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); + jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); + // presto coordinator announcement - discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator-ui"); + discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); + // query execution visualizer + jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); + // query manager + jaxrsBinder(binder).bindInstance(injector.getInstance(QueryResource.class)); + jaxrsBinder(binder).bindInstance(injector.getInstance(StageResource.class)); + // cluster statistics + jaxrsBinder(binder).bindInstance(injector.getInstance(ClusterStatsResource.class)); + // server info resource + jaxrsBinder(binder).bindInstance(injector.getInstance(ServerInfoResource.class)); + // server node resource + jaxrsBinder(binder).bindInstance(injector.getInstance(NodeResource.class)); + // task execution + // jaxrsBinder(binder).bindInstance(injector.getInstance(TaskResource.class)); + // jaxrsBinder(binder).bindInstance(injector.getInstance(PagesResponseWriter.class)); + // thread visualizer + // jaxrsBinder(binder).bindInstance(injector.getInstance(ThreadResource.class)); + // memory manager + // jaxrsBinder(binder).bindInstance(injector.getInstance(MemoryResource.class)); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 24fdbdb5a7f3..fa0787516053 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -131,27 +131,22 @@ public void run() injector.getInstance(Announcer.class).start(); log.info("======== SERVER STARTED ========"); - } - catch (Throwable e) { - log.error(e); - System.exit(1); - } - ImmutableList.Builder uIModules = ImmutableList.builder(); - uIModules.add( - new NodeModule(), - new DiscoveryModule(), - new HttpEventModule(), - new JsonModule(), - new JaxrsModule(true), - new CoordinatorUIHttpServerModule()); - - Bootstrap uIApp = new Bootstrap(uIModules.build()); - - try { - Injector injector = uIApp.doNotInitializeLogging().initialize(); - - log.info("======== UI STARTED ========"); + if (injector.getInstance(ServerConfig.class).isCoordinator()) { + Bootstrap uIApp = new Bootstrap(new NodeModule(), + new DiscoveryModule(), + new JsonModule(), + new JaxrsModule(true), + new JmxModule(), + new JmxHttpModule(), + new JsonEventModule(), + new HttpEventModule(), + new ServerSecurityModule(), + new CoordinatorUIHttpServerModule(injector)); + Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); + + log.info("======== UI STARTED ========"); + } } catch (Throwable e) { log.error(e); From 97bdf3021bde94eff58d5de2cee385ea39720fa0 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Sat, 7 Jan 2017 18:44:18 -0800 Subject: [PATCH 142/151] Add ui port enable config, clean code --- .../presto/server/CoordinatorModule.java | 4 +- .../server/CoordinatorUIHttpServerModule.java | 110 +++++++++--------- .../facebook/presto/server/PrestoServer.java | 4 +- .../facebook/presto/server/ServerConfig.java | 13 +++ 4 files changed, 70 insertions(+), 61 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index ae98b456ceac..fa56b5aeb713 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -116,7 +116,7 @@ import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -// import static io.airlift.http.server.HttpServerBinder.httpServerBinder; +import static io.airlift.http.server.HttpServerBinder.httpServerBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -130,7 +130,7 @@ public class CoordinatorModule @Override protected void setup(Binder binder) { - // httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); // presto coordinator announcement discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 1d701cdcf108..806e85a0cbb8 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -76,71 +76,65 @@ public CoordinatorUIHttpServerModule(Injector injector) @Override protected void setup(Binder binder) { - ServerConfig serverConfig = buildConfigObject(ServerConfig.class); + binder.disableCircularProxies(); - if (serverConfig.isCoordinator()) { - HttpServerConfig config = new HttpServerConfig().setHttpPort(serverConfig.getUIHttpPort()); - binder.bind(HttpServerConfig.class).toInstance(config); + ServerConfig serverConfig = injector.getInstance(ServerConfig.class); + HttpServerConfig httpServerConfig = injector.getInstance(HttpServerConfig.class) + .setHttpPort(serverConfig.getUIHttpPort()); + binder.bind(HttpServerConfig.class).toInstance(httpServerConfig); - binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); - binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); - binder.disableCircularProxies(); + binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); - Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); - Multibinder.newSetBinder(binder, Filter.class, TheAdminServlet.class); - Multibinder.newSetBinder(binder, HttpResourceBinding.class, TheServlet.class); - newExporter(binder).export(HttpServer.class).withGeneratedName(); - binder.bind(RequestStats.class).in(Scopes.SINGLETON); - newExporter(binder).export(RequestStats.class).withGeneratedName(); - eventBinder(binder).bindEventClient(HttpRequestEvent.class); + Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); + Multibinder.newSetBinder(binder, Filter.class, TheAdminServlet.class); + Multibinder.newSetBinder(binder, HttpResourceBinding.class, TheServlet.class); + newExporter(binder).export(HttpServer.class).withGeneratedName(); + binder.bind(RequestStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(RequestStats.class).withGeneratedName(); + eventBinder(binder).bindEventClient(HttpRequestEvent.class); - binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); - httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + - binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); - binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); - binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); - binder.bind(TransactionManager.class).toInstance(injector.getInstance(TransactionManager.class)); + binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); + binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); + binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); + binder.bind(TransactionManager.class).toInstance(injector.getInstance(TransactionManager.class)); - jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class); - jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); - jsonCodecBinder(binder).bindJsonCodec(QueryResults.class); - jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class); - jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class); - jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class); - jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class); - jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class); - jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class); - jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); - jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); - jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); - jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); - jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); - jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); - jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); - jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); + jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class); + jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); + jsonCodecBinder(binder).bindJsonCodec(QueryResults.class); + jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class); + jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class); + jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class); + jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class); + jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class); + jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class); + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); + jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); + jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); + jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); + jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); + jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); + jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); - // presto coordinator announcement - discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); - // query execution visualizer - jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); - // query manager - jaxrsBinder(binder).bindInstance(injector.getInstance(QueryResource.class)); - jaxrsBinder(binder).bindInstance(injector.getInstance(StageResource.class)); - // cluster statistics - jaxrsBinder(binder).bindInstance(injector.getInstance(ClusterStatsResource.class)); - // server info resource - jaxrsBinder(binder).bindInstance(injector.getInstance(ServerInfoResource.class)); - // server node resource - jaxrsBinder(binder).bindInstance(injector.getInstance(NodeResource.class)); - // task execution - // jaxrsBinder(binder).bindInstance(injector.getInstance(TaskResource.class)); - // jaxrsBinder(binder).bindInstance(injector.getInstance(PagesResponseWriter.class)); - // thread visualizer - // jaxrsBinder(binder).bindInstance(injector.getInstance(ThreadResource.class)); - // memory manager - // jaxrsBinder(binder).bindInstance(injector.getInstance(MemoryResource.class)); - } + // bind webapp + httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); + // presto coordinator announcement + discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); + // query execution visualizer + jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); + // query manager + jaxrsBinder(binder).bindInstance(injector.getInstance(QueryResource.class)); + jaxrsBinder(binder).bindInstance(injector.getInstance(StageResource.class)); + // cluster statistics + jaxrsBinder(binder).bindInstance(injector.getInstance(ClusterStatsResource.class)); + // server info resource + jaxrsBinder(binder).bindInstance(injector.getInstance(ServerInfoResource.class)); + // server node resource + jaxrsBinder(binder).bindInstance(injector.getInstance(NodeResource.class)); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index fa0787516053..17f0998f7599 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -132,7 +132,9 @@ public void run() log.info("======== SERVER STARTED ========"); - if (injector.getInstance(ServerConfig.class).isCoordinator()) { + ServerConfig serverConfig = injector.getInstance(ServerConfig.class); + + if (serverConfig.isCoordinator() && serverConfig.isBindUIonSecondaryPort()) { Bootstrap uIApp = new Bootstrap(new NodeModule(), new DiscoveryModule(), new JsonModule(), diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java index eb6a9c2d4cd8..7452fc26e41a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java @@ -25,6 +25,7 @@ public class ServerConfig private String dataSources; private boolean includeExceptionInResponse = true; private Duration gracePeriod = new Duration(2, MINUTES); + private boolean bindUIonSecondaryPort = false; private int uIHttpPort = 0; public boolean isCoordinator() @@ -39,6 +40,18 @@ public ServerConfig setCoordinator(boolean coordinator) return this; } + public boolean isBindUIonSecondaryPort() + { + return bindUIonSecondaryPort; + } + + @Config("http-server.ui.secondary.port.enable") + public ServerConfig setBindUIonSecondaryPort(boolean bindUIonSecondaryPort) + { + this.bindUIonSecondaryPort = bindUIonSecondaryPort; + return this; + } + public int getUIHttpPort() { return uIHttpPort; From 59530467b558ee72416197489a75816dd40e4362 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Sat, 7 Jan 2017 19:30:06 -0800 Subject: [PATCH 143/151] clearup --- .../facebook/presto/server/CoordinatorUIHttpServerModule.java | 3 --- .../src/main/java/com/facebook/presto/server/PrestoServer.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 806e85a0cbb8..6fd82ef822aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -95,9 +95,6 @@ protected void setup(Binder binder) eventBinder(binder).bindEventClient(HttpRequestEvent.class); binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); - - - binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 17f0998f7599..d6f67e5a7a11 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -147,7 +147,7 @@ public void run() new CoordinatorUIHttpServerModule(injector)); Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); - log.info("======== UI STARTED ========"); + log.info("======== SECONDARY UI STARTED ========"); } } catch (Throwable e) { From 36f14d7a7bfff7e0c4b0ccfb3a420398ced1ea61 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Sat, 7 Jan 2017 21:45:30 -0800 Subject: [PATCH 144/151] Changed config property name --- .../com/facebook/presto/server/PrestoServer.java | 2 +- .../com/facebook/presto/server/ServerConfig.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index d6f67e5a7a11..58bab15329c7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -134,7 +134,7 @@ public void run() ServerConfig serverConfig = injector.getInstance(ServerConfig.class); - if (serverConfig.isCoordinator() && serverConfig.isBindUIonSecondaryPort()) { + if (serverConfig.isCoordinator() && serverConfig.isEnabledUIonSecondaryPort()) { Bootstrap uIApp = new Bootstrap(new NodeModule(), new DiscoveryModule(), new JsonModule(), diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java index 7452fc26e41a..f623b43f4ba9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java @@ -25,7 +25,7 @@ public class ServerConfig private String dataSources; private boolean includeExceptionInResponse = true; private Duration gracePeriod = new Duration(2, MINUTES); - private boolean bindUIonSecondaryPort = false; + private boolean enabledUIonSecondaryPort = false; private int uIHttpPort = 0; public boolean isCoordinator() @@ -40,15 +40,15 @@ public ServerConfig setCoordinator(boolean coordinator) return this; } - public boolean isBindUIonSecondaryPort() + public boolean isEnabledUIonSecondaryPort() { - return bindUIonSecondaryPort; + return enabledUIonSecondaryPort; } - @Config("http-server.ui.secondary.port.enable") - public ServerConfig setBindUIonSecondaryPort(boolean bindUIonSecondaryPort) + @Config("http-server.ui.secondary.port.enabled") + public ServerConfig setEnabledUIonSecondaryPort(boolean enabledUIonSecondaryPort) { - this.bindUIonSecondaryPort = bindUIonSecondaryPort; + this.enabledUIonSecondaryPort = enabledUIonSecondaryPort; return this; } From 495a22f4f86d976e4904dce2b5fb7285590a2292 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 9 Jan 2017 17:30:15 -0800 Subject: [PATCH 145/151] remove unused modules --- .../presto/server/CoordinatorUIHttpServerModule.java | 6 ++++-- .../com/facebook/presto/server/PrestoServer.java | 12 +++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 6fd82ef822aa..3368e186558f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -40,6 +40,7 @@ import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.discovery.client.AnnouncementHttpServerInfo; +import io.airlift.event.client.EventClient; import io.airlift.http.server.HttpRequestEvent; import io.airlift.http.server.HttpServer; import io.airlift.http.server.HttpServerConfig; @@ -84,6 +85,7 @@ protected void setup(Binder binder) binder.bind(HttpServerConfig.class).toInstance(httpServerConfig); binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(EventClient.class).toInstance(injector.getInstance(EventClient.class)); binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); Multibinder.newSetBinder(binder, Filter.class, TheServlet.class); @@ -120,8 +122,8 @@ protected void setup(Binder binder) // bind webapp httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); - // presto coordinator announcement - discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator"); + // presto coordinator ui announcement + discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator-ui"); // query execution visualizer jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); // query manager diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 58bab15329c7..7576e2de419f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -130,8 +130,6 @@ public void run() injector.getInstance(Announcer.class).start(); - log.info("======== SERVER STARTED ========"); - ServerConfig serverConfig = injector.getInstance(ServerConfig.class); if (serverConfig.isCoordinator() && serverConfig.isEnabledUIonSecondaryPort()) { @@ -139,16 +137,12 @@ public void run() new DiscoveryModule(), new JsonModule(), new JaxrsModule(true), - new JmxModule(), - new JmxHttpModule(), - new JsonEventModule(), - new HttpEventModule(), - new ServerSecurityModule(), new CoordinatorUIHttpServerModule(injector)); Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); - - log.info("======== SECONDARY UI STARTED ========"); + log.info("UI runs on a seperate port: %d", serverConfig.getUIHttpPort()); } + + log.info("======== SERVER STARTED ========"); } catch (Throwable e) { log.error(e); From 228dbd73e9abbad5066dfbc4aba3e75d88227177 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 9 Jan 2017 18:28:31 -0800 Subject: [PATCH 146/151] remove more modules --- .../facebook/presto/server/CoordinatorUIHttpServerModule.java | 2 ++ .../main/java/com/facebook/presto/server/PrestoServer.java | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 3368e186558f..b76e160ed13d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -50,6 +50,7 @@ import io.airlift.http.server.RequestStats; import io.airlift.http.server.TheAdminServlet; import io.airlift.http.server.TheServlet; +import io.airlift.node.NodeInfo; import io.airlift.slice.Slice; import javax.servlet.Filter; @@ -85,6 +86,7 @@ protected void setup(Binder binder) binder.bind(HttpServerConfig.class).toInstance(httpServerConfig); binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(NodeInfo.class).toInstance(injector.getInstance(NodeInfo.class)); binder.bind(EventClient.class).toInstance(injector.getInstance(EventClient.class)); binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 7576e2de419f..afda2c3941c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -133,9 +133,7 @@ public void run() ServerConfig serverConfig = injector.getInstance(ServerConfig.class); if (serverConfig.isCoordinator() && serverConfig.isEnabledUIonSecondaryPort()) { - Bootstrap uIApp = new Bootstrap(new NodeModule(), - new DiscoveryModule(), - new JsonModule(), + Bootstrap uIApp = new Bootstrap(new JsonModule(), new JaxrsModule(true), new CoordinatorUIHttpServerModule(injector)); Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); From 432dcfb14bdc4711d1c7b2884ae2215da32836a8 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Mon, 9 Jan 2017 19:14:00 -0800 Subject: [PATCH 147/151] reuse initialized json mapper --- .../server/CoordinatorUIHttpServerModule.java | 41 ++----------------- .../facebook/presto/server/PrestoServer.java | 3 +- 2 files changed, 5 insertions(+), 39 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index b76e160ed13d..5ebfd1d5dd67 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -13,27 +13,11 @@ */ package com.facebook.presto.server; -import com.facebook.presto.block.BlockJsonSerde; -import com.facebook.presto.client.QueryResults; -import com.facebook.presto.execution.QueryInfo; -import com.facebook.presto.execution.TaskInfo; -import com.facebook.presto.execution.TaskStatus; -import com.facebook.presto.memory.MemoryInfo; -import com.facebook.presto.memory.MemoryPoolAssignmentsRequest; -import com.facebook.presto.metadata.ViewDefinition; -import com.facebook.presto.spi.ConnectorSplit; -import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncodingSerde; -import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; -import com.facebook.presto.sql.Serialization.ExpressionDeserializer; -import com.facebook.presto.sql.Serialization.ExpressionSerializer; -import com.facebook.presto.sql.Serialization.FunctionCallDeserializer; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.transaction.TransactionManager; -import com.facebook.presto.type.TypeDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Scopes; @@ -50,8 +34,8 @@ import io.airlift.http.server.RequestStats; import io.airlift.http.server.TheAdminServlet; import io.airlift.http.server.TheServlet; +import io.airlift.json.JsonCodecFactory; import io.airlift.node.NodeInfo; -import io.airlift.slice.Slice; import javax.servlet.Filter; @@ -60,8 +44,6 @@ import static io.airlift.http.server.HttpServerBinder.HttpResourceBinding; import static io.airlift.http.server.HttpServerBinder.httpServerBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -104,23 +86,8 @@ protected void setup(Binder binder) binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); binder.bind(TransactionManager.class).toInstance(injector.getInstance(TransactionManager.class)); - jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class); - jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); - jsonCodecBinder(binder).bindJsonCodec(QueryResults.class); - jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class); - jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class); - jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class); - jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class); - jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class); - jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class); - jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); - jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); - jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); - jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); - jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); - jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); - jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); - jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); + binder.bind(ObjectMapper.class).toInstance(injector.getInstance(ObjectMapper.class)); + binder.bind(JsonCodecFactory.class).toInstance(injector.getInstance(JsonCodecFactory.class)); // bind webapp httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index afda2c3941c1..36b65fff675e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -133,8 +133,7 @@ public void run() ServerConfig serverConfig = injector.getInstance(ServerConfig.class); if (serverConfig.isCoordinator() && serverConfig.isEnabledUIonSecondaryPort()) { - Bootstrap uIApp = new Bootstrap(new JsonModule(), - new JaxrsModule(true), + Bootstrap uIApp = new Bootstrap(new JaxrsModule(true), new CoordinatorUIHttpServerModule(injector)); Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); log.info("UI runs on a seperate port: %d", serverConfig.getUIHttpPort()); From fa6456c43bc805cd129b599dbcafadfcc649a480 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Wed, 11 Jan 2017 17:31:39 -0800 Subject: [PATCH 148/151] remove more resource --- .../facebook/presto/server/CoordinatorUIHttpServerModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 5ebfd1d5dd67..9e0f1fc35097 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -97,7 +97,6 @@ protected void setup(Binder binder) jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); // query manager jaxrsBinder(binder).bindInstance(injector.getInstance(QueryResource.class)); - jaxrsBinder(binder).bindInstance(injector.getInstance(StageResource.class)); // cluster statistics jaxrsBinder(binder).bindInstance(injector.getInstance(ClusterStatsResource.class)); // server info resource From 52f1a14b53dcc2ee17caf4c43ade5e3a81c81601 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 13 Jan 2017 15:39:18 -0800 Subject: [PATCH 149/151] Correct ui announcement --- .../presto/server/CoordinatorUIHttpServerModule.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 9e0f1fc35097..3e9789d7ad85 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -24,6 +24,8 @@ import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.discovery.client.AnnouncementHttpServerInfo; +import io.airlift.discovery.client.Announcer; +import io.airlift.discovery.server.DynamicAnnouncementResource; import io.airlift.event.client.EventClient; import io.airlift.http.server.HttpRequestEvent; import io.airlift.http.server.HttpServer; @@ -81,6 +83,7 @@ protected void setup(Binder binder) eventBinder(binder).bindEventClient(HttpRequestEvent.class); binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); + binder.bind(Announcer.class).toInstance(injector.getInstance(Announcer.class)); binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); @@ -93,6 +96,8 @@ protected void setup(Binder binder) httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); // presto coordinator ui announcement discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator-ui"); + // accept presto worker's announcement + jaxrsBinder(binder).bindInstance(injector.getInstance(DynamicAnnouncementResource.class)); // query execution visualizer jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); // query manager From 37fe86a23f4c80a4803b59ad69d32d7789c7e598 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Fri, 13 Jan 2017 18:45:34 -0800 Subject: [PATCH 150/151] use rerandomed node id to register ui service --- .../presto/server/CoordinatorUIHttpServerModule.java | 12 +++++++++--- .../com/facebook/presto/server/PrestoServer.java | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 3e9789d7ad85..13260f7e4d67 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -24,8 +24,8 @@ import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.discovery.client.AnnouncementHttpServerInfo; -import io.airlift.discovery.client.Announcer; import io.airlift.discovery.server.DynamicAnnouncementResource; +import io.airlift.discovery.server.ServiceResource; import io.airlift.event.client.EventClient; import io.airlift.http.server.HttpRequestEvent; import io.airlift.http.server.HttpServer; @@ -37,10 +37,13 @@ import io.airlift.http.server.TheAdminServlet; import io.airlift.http.server.TheServlet; import io.airlift.json.JsonCodecFactory; +import io.airlift.node.NodeConfig; import io.airlift.node.NodeInfo; import javax.servlet.Filter; +import java.util.UUID; + import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; import static io.airlift.event.client.EventBinder.eventBinder; import static io.airlift.http.server.HttpServerBinder.HttpResourceBinding; @@ -70,7 +73,6 @@ protected void setup(Binder binder) binder.bind(HttpServerConfig.class).toInstance(httpServerConfig); binder.bind(HttpServerInfo.class).in(Scopes.SINGLETON); - binder.bind(NodeInfo.class).toInstance(injector.getInstance(NodeInfo.class)); binder.bind(EventClient.class).toInstance(injector.getInstance(EventClient.class)); binder.bind(HttpServer.class).toProvider(HttpServerProvider.class).in(Scopes.SINGLETON); @@ -82,8 +84,10 @@ protected void setup(Binder binder) newExporter(binder).export(RequestStats.class).withGeneratedName(); eventBinder(binder).bindEventClient(HttpRequestEvent.class); + NodeConfig nodeConfig = injector.getInstance(NodeConfig.class); + nodeConfig.setNodeId(UUID.randomUUID().toString()); + binder.bind(NodeInfo.class).toInstance(new NodeInfo(nodeConfig)); binder.bind(AnnouncementHttpServerInfo.class).to(LocalAnnouncementHttpServerInfo.class).in(Scopes.SINGLETON); - binder.bind(Announcer.class).toInstance(injector.getInstance(Announcer.class)); binder.bind(BlockEncodingSerde.class).toInstance(injector.getInstance(BlockEncodingSerde.class)); binder.bind(TypeManager.class).toInstance(injector.getInstance(TypeManager.class)); binder.bind(SqlParser.class).toInstance(injector.getInstance(SqlParser.class)); @@ -98,6 +102,8 @@ protected void setup(Binder binder) discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator-ui"); // accept presto worker's announcement jaxrsBinder(binder).bindInstance(injector.getInstance(DynamicAnnouncementResource.class)); + // service info + jaxrsBinder(binder).bindInstance(injector.getInstance(ServiceResource.class)); // query execution visualizer jaxrsBinder(binder).bindInstance(injector.getInstance(QueryExecutionResource.class)); // query manager diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 36b65fff675e..19f1590d2933 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -133,10 +133,13 @@ public void run() ServerConfig serverConfig = injector.getInstance(ServerConfig.class); if (serverConfig.isCoordinator() && serverConfig.isEnabledUIonSecondaryPort()) { - Bootstrap uIApp = new Bootstrap(new JaxrsModule(true), + Bootstrap uIApp = new Bootstrap(new DiscoveryModule(), + new JaxrsModule(true), new CoordinatorUIHttpServerModule(injector)); Injector uIInjector = uIApp.doNotInitializeLogging().initialize(); log.info("UI runs on a seperate port: %d", serverConfig.getUIHttpPort()); + + uIInjector.getInstance(Announcer.class).start(); } log.info("======== SERVER STARTED ========"); From b5fe62f2e9e1d446604f8e54f865e509e15c26e5 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Tue, 17 Jan 2017 11:00:31 -0800 Subject: [PATCH 151/151] allow override the resouce binding code --- .../presto/server/CoordinatorUIHttpServerModule.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java index 13260f7e4d67..688c0f434ef0 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorUIHttpServerModule.java @@ -96,6 +96,11 @@ protected void setup(Binder binder) binder.bind(ObjectMapper.class).toInstance(injector.getInstance(ObjectMapper.class)); binder.bind(JsonCodecFactory.class).toInstance(injector.getInstance(JsonCodecFactory.class)); + resourceBinding(binder); + } + + protected void resourceBinding(Binder binder) + { // bind webapp httpServerBinder(binder).bindResource("/", "webapp").withWelcomeFile("index.html"); // presto coordinator ui announcement