twitter-forks · Yaliang · Sep 25, 2015 · Oct 1, 2015 · Oct 1, 2015 · Oct 1, 2015
diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
 
     <groupId>com.facebook.presto</groupId>
     <artifactId>presto-root</artifactId>
-    <version>0.161</version>
+    <version>0.157-tw-0.29</version>
     <packaging>pom</packaging>
 
     <name>presto-root</name>
@@ -28,9 +28,9 @@
     </licenses>
 
     <scm>
-        <connection>scm:git:git://github.com/facebook/presto.git</connection>
-        <url>https://github.com/facebook/presto</url>
-        <tag>0.161</tag>
+        <connection>scm:git:git://github.com/twitter-forks/presto.git</connection>
+        <url>https://github.com/twitter-forks/presto</url>
+        <tag>0.157-tw-0.29</tag>
     </scm>
 
     <properties>
@@ -91,6 +91,7 @@
         <module>presto-hive-cdh5</module>
         <module>presto-teradata-functions</module>
         <module>presto-example-http</module>
+        <module>twitter-eventlistener-plugin</module>
         <module>presto-local-file</module>
         <module>presto-tpch</module>
         <module>presto-raptor</module>
@@ -643,6 +644,12 @@
                 <version>3.2</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-pool2</artifactId>
+                <version>2.4.2</version>
+            </dependency>
+
             <dependency>
                 <groupId>commons-codec</groupId>
                 <artifactId>commons-codec</artifactId>

diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-accumulo</artifactId>

diff --git a/presto-array/pom.xml b/presto-array/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-array</artifactId>

diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-atop</artifactId>

diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-base-jdbc</artifactId>

diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-benchmark-driver</artifactId>

diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>presto-root</artifactId>
         <groupId>com.facebook.presto</groupId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-benchmark</artifactId>

diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-blackhole</artifactId>

diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-bytecode</artifactId>

diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-cassandra</artifactId>

diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-cli</artifactId>

diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java
@@ -17,6 +17,7 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
+import com.sun.security.auth.module.UnixSystem;
 import io.airlift.airline.Option;
 import io.airlift.http.client.spnego.KerberosConfig;
 import io.airlift.units.Duration;
@@ -72,15 +73,16 @@ public class ClientOptions
     @Option(name = "--keystore-password", title = "keystore password", description = "Keystore password")
     public String keystorePassword;
 
+    // Pick the user name for the logged in user.
+    // Do not let it be overridden by users.
+    public String user = new UnixSystem().getUsername();
+
     @Option(name = "--truststore-path", title = "truststore path", description = "Truststore path")
     public String truststorePath;
 
     @Option(name = "--truststore-password", title = "truststore password", description = "Truststore password")
     public String truststorePassword;
 
-    @Option(name = "--user", title = "user", description = "Username")
-    public String user = System.getProperty("user.name");
-
     @Option(name = "--source", title = "source", description = "Name of source making query")
     public String source = "presto-cli";
 

diff --git a/presto-client/pom.xml b/presto-client/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-client</artifactId>

diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-docs</artifactId>

diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-example-http</artifactId>

diff --git a/presto-hive-cdh4/pom.xml b/presto-hive-cdh4/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-hive-cdh4</artifactId>

diff --git a/presto-hive-cdh5/pom.xml b/presto-hive-cdh5/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-hive-cdh5</artifactId>

diff --git a/presto-hive-hadoop1/pom.xml b/presto-hive-hadoop1/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-hive-hadoop1</artifactId>

diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-hive-hadoop2</artifactId>

diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.161</version>
+        <version>0.157-tw-0.29</version>
     </parent>
 
     <artifactId>presto-hive</artifactId>
@@ -53,11 +53,54 @@
             <artifactId>hive-apache</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-recipes</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-framework</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-client</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-test</artifactId>
+            <version>2.8.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+            <version>3.4.6</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.101tec</groupId>
+            <artifactId>zkclient</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libthrift</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-pool2</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>aircompressor</artifactId>
@@ -93,6 +136,12 @@
             <artifactId>configuration</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.googlecode.json-simple</groupId>
+            <artifactId>json-simple</artifactId>
+            <version>1.1</version>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java
@@ -24,6 +24,7 @@
 import com.facebook.presto.spi.PrestoException;
 import com.facebook.presto.spi.StandardErrorCode;
 import com.facebook.presto.spi.predicate.TupleDomain;
+import com.facebook.presto.twitter.hive.util.UgiUtils;
 import com.google.common.collect.ImmutableList;
 import com.google.common.io.CharStreams;
 import io.airlift.units.DataSize;
@@ -40,11 +41,13 @@
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
@@ -151,8 +154,15 @@ public BackgroundHiveSplitLoader(
     public void start(HiveSplitSource splitSource)
     {
         this.hiveSplitSource = splitSource;
+
+        UserGroupInformation ugi = null;
+
+        if (HiveSessionProperties.getReadAsQueryUser(session)) {
+            ugi = UgiUtils.getUgi(session.getUser());
+        }
+
         for (int i = 0; i < maxPartitionBatchSize; i++) {
-            ResumableTasks.submit(executor, new HiveSplitLoaderTask());
+            ResumableTasks.submit(executor, new HiveSplitLoaderTask(ugi));
         }
     }
 
@@ -165,8 +175,30 @@ public void stop()
     private class HiveSplitLoaderTask
             implements ResumableTask
     {
+        private UserGroupInformation ugi;
+
+        public HiveSplitLoaderTask(UserGroupInformation ugi)
+        {
+            this.ugi = ugi;
+        }
+
         @Override
         public TaskStatus process()
+        {
+            if (ugi != null) {
+                try {
+                    return ugi.doAs((PrivilegedExceptionAction<TaskStatus>) this::doProcess);
+                }
+                catch (IOException | InterruptedException e) {
+                    throw new RuntimeException("Could not runAs " + session.getUser(), e);
+                }
+            }
+            else {
+                return doProcess();
+            }
+        }
+
+        private TaskStatus doProcess()
         {
             while (true) {
                 if (stopped) {

diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java
@@ -64,6 +64,8 @@ public class HiveClientConfig
 
     private boolean allowCorruptWritesForTesting;
 
+    private boolean readAsQueryUser = false;
+
     private Duration metastoreCacheTtl = new Duration(1, TimeUnit.HOURS);
     private Duration metastoreRefreshInterval = new Duration(1, TimeUnit.SECONDS);
     private long metastoreCacheMaximumSize = 10000;
@@ -305,6 +307,19 @@ public HiveClientConfig setAllowCorruptWritesForTesting(boolean allowCorruptWrit
         return this;
     }
 
+    public boolean getReadAsQueryUser()
+    {
+        return readAsQueryUser;
+    }
+
+    @Config("hive.read-as-query-user")
+    @ConfigDescription("When querying hive read data as the user submitting the query instead of as the presto daemon user")
+    public HiveClientConfig setReadAsQueryUser(boolean readAsQueryUser)
+    {
+        this.readAsQueryUser = readAsQueryUser;
+        return this;
+    }
+
     @NotNull
     public Duration getMetastoreCacheTtl()
     {