Merge pull request #63 from dabaitu/twitter-master

Upgrade to 0.161

.travis.yml

@@ -5,6 +5,7 @@ env:
     - MAVEN_OPTS="-Xmx512M -XX:+ExitOnOutOfMemoryError"
     - MAVEN_SKIP_CHECKS_AND_DOCS="-Dair.check.skip-all=true -Dmaven.javadoc.skip=true"
     - MAVEN_FAST_INSTALL="-DskipTests $MAVEN_SKIP_CHECKS_AND_DOCS -B -q -T C1"
+    - ARTIFACTS_UPLOAD_PATH=travis_build_artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER}
   matrix:
     - MAVEN_CHECKS=true
     - TEST_SPECIFIC_MODULES=presto-tests
@@ -75,11 +76,6 @@ script:
     if [[ -v HIVE_TESTS ]]; then
       presto-hive-hadoop2/bin/run_on_docker.sh
     fi
-  - |
-    # Build presto-server-rpm for later artifact upload
-    if [[ -v DEPLOY_S3_ACCESS_KEY && -v PRODUCT_TESTS ]]; then
-       ./mvnw install $MAVEN_FAST_INSTALL -pl presto-server-rpm
-    fi
 
 before_cache:
   # Make the cache stable between builds by removing build output
@@ -104,12 +100,35 @@ before_deploy:
 deploy:
   on:
     all_branches: true
-    condition: -v DEPLOY_S3_ACCESS_KEY && -v PRODUCT_TESTS
+    condition: -v DEPLOY_S3_ACCESS_KEY && -v MAVEN_CHECKS
   provider: s3
   access_key_id: ${DEPLOY_S3_ACCESS_KEY}
   secret_access_key: ${DEPLOY_S3_SECRET_KEY}
   bucket: ${DEPLOY_S3_BUCKET}
   skip_cleanup: true
   local-dir: /tmp/artifacts
-  upload-dir: travis_build_artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_JOB_NUMBER}
+  upload-dir: ${ARTIFACTS_UPLOAD_PATH}
   acl: public_read
+
+after_script:
+- |
+  if [[ -v DEPLOY_S3_ACCESS_KEY ]]; then
+    sudo pip install awscli
+    export AWS_ACCESS_KEY_ID=${DEPLOY_S3_ACCESS_KEY}
+    export AWS_SECRET_ACCESS_KEY=${DEPLOY_S3_SECRET_KEY}
+
+    JOB_ARTIFACTS_URL_PREFIX=s3://${DEPLOY_S3_BUCKET}/${ARTIFACTS_UPLOAD_PATH}/travis_jobs/${TRAVIS_JOB_NUMBER}-run
+    JOB_RUN_ATTEMPTS=$( aws s3 ls ${JOB_ARTIFACTS_URL_PREFIX} | wc -l | tr -d '[:space:]' )
+    JOB_STATUS=$( [ "$TRAVIS_TEST_RESULT" == "0" ] && echo SUCCESS || echo FAILURE )
+
+    mkdir -p /tmp/job_artifacts/
+    rsync -av -m \
+      --include='**/' \
+      --include='**/surefire-reports/**.xml' \
+      --include='**/surefire-reports/emailable-report.html' \
+      --exclude='*' \
+      . /tmp/job_artifacts/
+    wget https://api.travis-ci.org/jobs/${TRAVIS_JOB_ID}/log.txt?deansi=true -O /tmp/job_artifacts/log.txt
+
+    aws s3 sync /tmp/job_artifacts ${JOB_ARTIFACTS_URL_PREFIX}_$((JOB_RUN_ATTEMPTS + 1))-${JOB_STATUS}
+  fi

README.md

@@ -38,7 +38,7 @@ After opening the project in IntelliJ, double check that the Java SDK is properl
 Presto comes with sample configuration that should work out-of-the-box for development. Use the following options to create a run configuration:
 
 * Main Class: `com.facebook.presto.server.PrestoServer`
-* VM Options: `-ea -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties`
+* VM Options: `-ea -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties`
 * Working directory: `$MODULE_DIR$`
 * Use classpath of module: `presto-main`
 

pom.xml

@@ -50,7 +50,7 @@
         <dep.packaging.version>${dep.airlift.version}</dep.packaging.version>
         <dep.slice.version>0.27</dep.slice.version>
         <dep.aws-sdk.version>1.11.30</dep.aws-sdk.version>
-        <dep.tempto.version>1.14</dep.tempto.version>
+        <dep.tempto.version>1.18</dep.tempto.version>
 
         <!--
         Versions newer than 6.9 appear to have an issue where the @BeforeClass method in
@@ -322,19 +322,19 @@
             <dependency>
                 <groupId>com.facebook.presto.hadoop</groupId>
                 <artifactId>hadoop-apache1</artifactId>
-                <version>0.3</version>
+                <version>0.4</version>
             </dependency>
 
             <dependency>
                 <groupId>com.facebook.presto.hadoop</groupId>
                 <artifactId>hadoop-apache2</artifactId>
-                <version>0.9</version>
+                <version>0.10</version>
             </dependency>
 
             <dependency>
                 <groupId>com.facebook.presto.hadoop</groupId>
                 <artifactId>hadoop-cdh4</artifactId>
-                <version>0.9</version>
+                <version>0.10</version>
             </dependency>
 
             <dependency>
@@ -871,9 +871,9 @@
                 </plugin>
 
                 <plugin>
-                    <groupId>kr.motd.maven</groupId>
+                    <groupId>io.airlift.maven.plugins</groupId>
                     <artifactId>sphinx-maven-plugin</artifactId>
-                    <version>1.3.1.Final</version>
+                    <version>2.0</version>
                 </plugin>
 
                 <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java

@@ -16,7 +16,6 @@
 import com.facebook.presto.spi.type.StandardTypes;
 import com.facebook.presto.spi.type.Type;
 import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
 import com.google.common.collect.ImmutableMap;
 import io.airlift.json.JsonCodec;
@@ -41,7 +40,7 @@ private MetadataUtil() {}
 
     static {
         ObjectMapperProvider provider = new ObjectMapperProvider();
-        provider.setJsonDeserializers(ImmutableMap.<Class<?>, JsonDeserializer<?>>of(Type.class, new TestingTypeDeserializer()));
+        provider.setJsonDeserializers(ImmutableMap.of(Type.class, new TestingTypeDeserializer()));
         JsonCodecFactory codecFactory = new JsonCodecFactory(provider);
         COLUMN_CODEC = codecFactory.jsonCodec(JdbcColumnHandle.class);
         TABLE_CODEC = codecFactory.jsonCodec(JdbcTableHandle.class);
@@ -51,7 +50,7 @@ private MetadataUtil() {}
     public static final class TestingTypeDeserializer
             extends FromStringDeserializer<Type>
     {
-        private final Map<String, Type> types = ImmutableMap.<String, Type>of(
+        private final Map<String, Type> types = ImmutableMap.of(
                 StandardTypes.BIGINT, BIGINT,
                 StandardTypes.VARCHAR, VARCHAR);
 

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcOutputTableHandle.java

@@ -13,7 +13,6 @@
  */
 package com.facebook.presto.plugin.jdbc;
 
-import com.facebook.presto.spi.type.Type;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.testng.annotations.Test;
@@ -33,7 +32,7 @@ public void testJsonRoundTrip()
                 "schema",
                 "table",
                 ImmutableList.of("abc", "xyz"),
-                ImmutableList.<Type>of(VARCHAR, VARCHAR),
+                ImmutableList.of(VARCHAR, VARCHAR),
                 "tmp_table",
                 "jdbc:junk",
                 ImmutableMap.of("user", "test"));

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSet.java

@@ -76,7 +76,7 @@ public void testGetColumnTypes()
                 new JdbcColumnHandle("test", "text", VARCHAR)));
         assertEquals(recordSet.getColumnTypes(), ImmutableList.of(BIGINT, BIGINT, VARCHAR));
 
-        recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.<JdbcColumnHandle>of());
+        recordSet = new JdbcRecordSet(jdbcClient, split, ImmutableList.of());
         assertEquals(recordSet.getColumnTypes(), ImmutableList.of());
     }
 

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSetProvider.java

@@ -111,52 +111,52 @@ public void testTupleDomain()
     {
         // single value
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.singleValue(VARCHAR, utf8Slice("foo")))
+                ImmutableMap.of(textColumn, Domain.singleValue(VARCHAR, utf8Slice("foo")))
         ));
 
         // multiple values (string)
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.singleValue(VARCHAR, utf8Slice("bar")))))
+                ImmutableMap.of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.singleValue(VARCHAR, utf8Slice("bar")))))
         ));
 
         // inequality (string)
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false))
+                ImmutableMap.of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false))
         ));
 
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false))
+                ImmutableMap.of(textColumn, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARCHAR, utf8Slice("foo"))), false))
         ));
 
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("foo"))), false))
+                ImmutableMap.of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("foo"))), false))
         ));
 
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThan(VARCHAR, utf8Slice("foo"))), false))
+                ImmutableMap.of(textColumn, Domain.create(ValueSet.ofRanges(Range.lessThan(VARCHAR, utf8Slice("foo"))), false))
         ));
 
         // is null
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.onlyNull(VARCHAR))
+                ImmutableMap.of(textColumn, Domain.onlyNull(VARCHAR))
         ));
 
         // not null
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.notNull(VARCHAR))
+                ImmutableMap.of(textColumn, Domain.notNull(VARCHAR))
         ));
 
         // specific value or null
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.onlyNull(VARCHAR))))
+                ImmutableMap.of(textColumn, Domain.union(ImmutableList.of(Domain.singleValue(VARCHAR, utf8Slice("foo")), Domain.onlyNull(VARCHAR))))
         ));
 
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(textColumn, Domain.create(ValueSet.ofRanges(Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true)), false))
+                ImmutableMap.of(textColumn, Domain.create(ValueSet.ofRanges(Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true)), false))
         ));
 
         getCursor(table, ImmutableList.of(textColumn, textShortColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(
+                ImmutableMap.of(
                         textColumn,
                         Domain.create(ValueSet.ofRanges(
                                         Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true),
@@ -174,7 +174,7 @@ public void testTupleDomain()
         ));
 
         getCursor(table, ImmutableList.of(textColumn, valueColumn), TupleDomain.withColumnDomains(
-                ImmutableMap.<ColumnHandle, Domain>of(
+                ImmutableMap.of(
                         textColumn,
                         Domain.create(ValueSet.ofRanges(
                                         Range.range(VARCHAR, utf8Slice("bar"), true, utf8Slice("foo"), true),

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcSplit.java

@@ -13,7 +13,6 @@
  */
 package com.facebook.presto.plugin.jdbc;
 
-import com.facebook.presto.spi.ColumnHandle;
 import com.facebook.presto.spi.predicate.TupleDomain;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -25,7 +24,7 @@
 
 public class TestJdbcSplit
 {
-    private final JdbcSplit split = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.<String, String>of(), TupleDomain.<ColumnHandle>all());
+    private final JdbcSplit split = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.of(), TupleDomain.all());
 
     @Test
     public void testAddresses()
@@ -34,7 +33,7 @@ public void testAddresses()
         assertEquals(split.getAddresses(), ImmutableList.of());
         assertEquals(split.isRemotelyAccessible(), true);
 
-        JdbcSplit jdbcSplit = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.<String, String>of(), TupleDomain.<ColumnHandle>all());
+        JdbcSplit jdbcSplit = new JdbcSplit("connectorId", "catalog", "schemaName", "tableName", "connectionUrl", ImmutableMap.of(), TupleDomain.all());
         assertEquals(jdbcSplit.getAddresses(), ImmutableList.of());
     }
 

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingDatabase.java

@@ -13,7 +13,6 @@
  */
 package com.facebook.presto.plugin.jdbc;
 
-import com.facebook.presto.spi.ColumnHandle;
 import com.facebook.presto.spi.ConnectorSplitSource;
 import com.facebook.presto.spi.SchemaTableName;
 import com.facebook.presto.spi.predicate.TupleDomain;
@@ -94,7 +93,7 @@ public JdbcSplit getSplit(String schemaName, String tableName)
             throws InterruptedException
     {
         JdbcTableHandle jdbcTableHandle = jdbcClient.getTableHandle(new SchemaTableName(schemaName, tableName));
-        JdbcTableLayoutHandle jdbcLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.<ColumnHandle>all());
+        JdbcTableLayoutHandle jdbcLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.all());
         ConnectorSplitSource splits = jdbcClient.getSplits(jdbcLayoutHandle);
         return (JdbcSplit) getOnlyElement(getFutureValue(splits.getNextBatch(1000)));
     }

presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkSchema.java

@@ -26,7 +26,7 @@ public class BenchmarkSchema
 
     public BenchmarkSchema(String name)
     {
-        this(name, ImmutableMap.<String, String>of());
+        this(name, ImmutableMap.of());
     }
 
     public BenchmarkSchema(String name, Map<String, String> tags)

presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/Suite.java

@@ -44,7 +44,7 @@ public class Suite
     public Suite(String name, Map<String, String> sessionProperties, Iterable<RegexTemplate> schemaNameTemplates, Iterable<Pattern> queryNamePatterns)
     {
         this.name = requireNonNull(name, "name is null");
-        this.sessionProperties = sessionProperties == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(sessionProperties);
+        this.sessionProperties = sessionProperties == null ? ImmutableMap.of() : ImmutableMap.copyOf(sessionProperties);
         this.schemaNameTemplates = ImmutableList.copyOf(requireNonNull(schemaNameTemplates, "schemaNameTemplates is null"));
         this.queryNamePatterns = ImmutableList.copyOf(requireNonNull(queryNamePatterns, "queryNamePatterns is null"));
     }

presto-benchmark/src/main/java/com/facebook/presto/benchmark/AbstractOperatorBenchmark.java

@@ -95,7 +95,7 @@ protected OperatorFactory createTableScanOperator(int operatorId, PlanNodeId pla
 
     protected OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types)
     {
-        return localQueryRunner.createHashProjectOperator(operatorId, planNodeId, types);
+        return LocalQueryRunner.createHashProjectOperator(operatorId, planNodeId, types);
     }
 
     protected abstract List<Driver> createDrivers(TaskContext taskContext);

presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkQueryRunner.java

@@ -51,7 +51,7 @@ public static LocalQueryRunner createLocalQueryRunner(boolean hashingEnabled)
         LocalQueryRunner localQueryRunner = new LocalQueryRunner(session);
 
         // add tpch
-        localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.<String, String>of());
+        localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of());
 
         return localQueryRunner;
     }

presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkSuite.java

@@ -38,7 +38,7 @@ public static List<AbstractBenchmark> createBenchmarks(LocalQueryRunner localQue
         Session optimizeHashSession = Session.builder(localQueryRunner.getDefaultSession())
                 .setSystemProperty(OPTIMIZE_HASH_GENERATION, "true")
                 .build();
-        return ImmutableList.<AbstractBenchmark>of(
+        return ImmutableList.of(
                 // hand built benchmarks
                 new CountAggregationBenchmark(localQueryRunner),
                 new DoubleSumAggregationBenchmark(localQueryRunner),

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java

@@ -133,7 +133,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
     public static class TpchQuery1Operator
             implements com.facebook.presto.operator.Operator // TODO: use import when Java 7 compiler bug is fixed
     {
-        private static final ImmutableList<Type> TYPES = ImmutableList.<Type>of(
+        private static final ImmutableList<Type> TYPES = ImmutableList.of(
                 VARCHAR,
                 VARCHAR,
                 DOUBLE,

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java

@@ -64,7 +64,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
         //    and quantity < 24;
         OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "extendedprice", "discount", "shipdate", "quantity");
 
-        FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), () -> new TpchQuery6Processor(), ImmutableList.<Type>of(DOUBLE));
+        FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), () -> new TpchQuery6Processor(), ImmutableList.of(DOUBLE));
 
         AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(
                 2,

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashBuildAndJoinBenchmark.java

@@ -21,7 +21,6 @@
 import com.facebook.presto.operator.LookupJoinOperators;
 import com.facebook.presto.operator.OperatorFactory;
 import com.facebook.presto.operator.TaskContext;
-import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.facebook.presto.testing.NullOutputOperator.NullOutputOperatorFactory;
@@ -69,7 +68,7 @@ protected List<Driver> createDrivers(TaskContext taskContext)
         OperatorFactory source = ordersTableScan;
         Optional<Integer> hashChannel = Optional.empty();
         if (hashEnabled) {
-            source = createHashProjectOperator(1, new PlanNodeId("test"), ImmutableList.<Type>of(BIGINT, DOUBLE));
+            source = createHashProjectOperator(1, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE));
             driversBuilder.add(source);
             hashChannel = Optional.of(2);
         }
@@ -87,7 +86,7 @@ protected List<Driver> createDrivers(TaskContext taskContext)
         source = lineItemTableScan;
         hashChannel = Optional.empty();
         if (hashEnabled) {
-            source = createHashProjectOperator(1, new PlanNodeId("test"), ImmutableList.<Type>of(BIGINT, BIGINT));
+            source = createHashProjectOperator(1, new PlanNodeId("test"), ImmutableList.of(BIGINT, BIGINT));
             joinDriversBuilder.add(source);
             hashChannel = Optional.of(2);
         }