Merge pull request #123 from Yaliang/yaliangw/twitter/release-0.188-t…

…w-0.43 release 0.188-tw-0.43
twitter-forks · Nov 8, 2017 · 0239495 · 0239495
2 parents 04be4e2 + 0360705
commit 0239495
Show file tree

Hide file tree

Showing 185 changed files with 2,065 additions and 490 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -77,7 +77,7 @@ before_script:
 script:
   - |
     if [[ -v MAVEN_CHECKS ]]; then
-      ./mvnw install -DskipTests -B -T C1 -P !twitter-modules
+      ./mvnw install -DskipTests -B -T C1 -P 'ci,!twitter-modules'
     fi
   - |
     if [[ -v TEST_SPECIFIC_MODULES ]]; then

diff --git a/pom.xml b/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <groupId>io.airlift</groupId>
         <artifactId>airbase</artifactId>
-        <version>72</version>
+        <version>74</version>
     </parent>
 
     <groupId>com.facebook.presto</groupId>
     <artifactId>presto-root</artifactId>
-    <version>0.187-tw-0.42</version>
+    <version>0.188-tw-0.43</version>
     <packaging>pom</packaging>
 
     <name>presto-root</name>
@@ -30,7 +30,7 @@
     <scm>
         <connection>scm:git:git://github.com/twitter-forks/presto.git</connection>
         <url>https://github.com/twitter-forks/presto</url>
-        <tag>0.187-tw-0.42</tag>
+        <tag>0.188-tw-0.43</tag>
     </scm>
 
     <properties>
@@ -47,11 +47,11 @@
         <air.maven.version>3.3.9</air.maven.version>
 
         <dep.antlr.version>4.6</dep.antlr.version>
-        <dep.airlift.version>0.153</dep.airlift.version>
+        <dep.airlift.version>0.155</dep.airlift.version>
         <dep.packaging.version>${dep.airlift.version}</dep.packaging.version>
-        <dep.slice.version>0.31</dep.slice.version>
+        <dep.slice.version>0.32</dep.slice.version>
         <dep.aws-sdk.version>1.11.165</dep.aws-sdk.version>
-        <dep.okhttp.version>3.8.1</dep.okhttp.version>
+        <dep.okhttp.version>3.9.0</dep.okhttp.version>
         <dep.tempto.version>1.36</dep.tempto.version>
         <dep.testng.version>6.10</dep.testng.version>
         <dep.nifty.version>0.15.1</dep.nifty.version>

diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-accumulo</artifactId>

diff --git a/presto-array/pom.xml b/presto-array/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-array</artifactId>

diff --git a/presto-array/src/main/java/com/facebook/presto/array/BlockBigArray.java b/presto-array/src/main/java/com/facebook/presto/array/BlockBigArray.java
@@ -67,7 +67,7 @@ public void set(long index, Block value)
                     sizeOfBlocks -= size;
                     return;
                 }
-                if (trackedObjects.decrementReference(object) == 0) {
+                if (trackedObjects.decrementAndGet(object) == 0) {
                     // decrement the size only when it is the last reference
                     sizeOfBlocks -= size;
                 }
@@ -80,7 +80,7 @@ public void set(long index, Block value)
                     sizeOfBlocks += size;
                     return;
                 }
-                if (trackedObjects.incrementReference(object) == 1) {
+                if (trackedObjects.incrementAndGet(object) == 1) {
                     // increment the size only when it is the first reference
                     sizeOfBlocks += size;
                 }

diff --git a/presto-array/src/main/java/com/facebook/presto/array/ReferenceCountMap.java b/presto-array/src/main/java/com/facebook/presto/array/ReferenceCountMap.java
@@ -34,15 +34,15 @@ public ReferenceCountMap()
     /**
      * Increments the reference count of an object by 1 and returns the updated reference count
      */
-    public int incrementReference(Object key)
+    public int incrementAndGet(Object key)
     {
         return addTo(key, 1) + 1;
     }
 
     /**
      * Decrements the reference count of an object by 1 and returns the updated reference count
      */
-    public int decrementReference(Object key)
+    public int decrementAndGet(Object key)
     {
         int previousCount = addTo(key, -1);
         if (previousCount == 1) {

diff --git a/presto-array/src/main/java/com/facebook/presto/array/SliceBigArray.java b/presto-array/src/main/java/com/facebook/presto/array/SliceBigArray.java
@@ -21,6 +21,7 @@ public final class SliceBigArray
     private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceBigArray.class).instanceSize();
     private static final int SLICE_INSTANCE_SIZE = ClassLayout.parseClass(Slice.class).instanceSize();
     private final ObjectBigArray<Slice> array;
+    private final ReferenceCountMap trackedSlices = new ReferenceCountMap();
     private long sizeOfSlices;
 
     public SliceBigArray()
@@ -38,7 +39,7 @@ public SliceBigArray(Slice slice)
      */
     public long sizeOf()
     {
-        return INSTANCE_SIZE + array.sizeOf() + sizeOfSlices;
+        return INSTANCE_SIZE + array.sizeOf() + sizeOfSlices + trackedSlices.sizeOf();
     }
 
     /**
@@ -59,24 +60,10 @@ public Slice get(long index)
      */
     public void set(long index, Slice value)
     {
-        Slice currentValue = array.get(index);
-        if (currentValue != null) {
-            sizeOfSlices -= getSize(currentValue);
-        }
-        if (value != null) {
-            sizeOfSlices += getSize(value);
-        }
+        updateRetainedSize(index, value);
         array.set(index, value);
     }
 
-    // For now we approximate the retained size of a slice by object overhead plus length.
-    // In general this is more complicated than that as there may be multiple "view" slices
-    // pointing to the same backing memory of a slice.
-    private long getSize(Slice slice)
-    {
-        return slice.length() + SLICE_INSTANCE_SIZE;
-    }
-
     /**
      * Ensures this big array is at least the specified length.  If the array is smaller, segments
      * are added until the array is larger then the specified length.
@@ -85,4 +72,33 @@ public void ensureCapacity(long length)
     {
         array.ensureCapacity(length);
     }
+
+    private void updateRetainedSize(long index, Slice value)
+    {
+        Slice currentValue = array.get(index);
+        if (currentValue != null) {
+            int baseReferenceCount = trackedSlices.decrementAndGet(currentValue.getBase());
+            int sliceReferenceCount = trackedSlices.decrementAndGet(currentValue);
+            if (baseReferenceCount == 0) {
+                // it is the last referenced base
+                sizeOfSlices -= currentValue.getRetainedSize();
+            }
+            else if (sliceReferenceCount == 0) {
+                // it is the last referenced slice
+                sizeOfSlices -= SLICE_INSTANCE_SIZE;
+            }
+        }
+        if (value != null) {
+            int baseReferenceCount = trackedSlices.incrementAndGet(value.getBase());
+            int sliceReferenceCount = trackedSlices.incrementAndGet(value);
+            if (baseReferenceCount == 1) {
+                // it is the first referenced base
+                sizeOfSlices += value.getRetainedSize();
+            }
+            else if (sliceReferenceCount == 1) {
+                // it is the first referenced slice
+                sizeOfSlices += SLICE_INSTANCE_SIZE;
+            }
+        }
+    }
 }
diff --git a/presto-array/src/test/java/com/facebook/presto/array/TestBlockBigArray.java b/presto-array/src/test/java/com/facebook/presto/array/TestBlockBigArray.java
@@ -44,7 +44,7 @@ public void testRetainedSizeWithOverlappingBlocks()
         }
 
         ReferenceCountMap referenceCountMap = new ReferenceCountMap();
-        referenceCountMap.incrementReference(block);
+        referenceCountMap.incrementAndGet(block);
         long expectedSize = ClassLayout.parseClass(BlockBigArray.class).instanceSize()
                 + referenceCountMap.sizeOf()
                 + (new ObjectBigArray()).sizeOf()

diff --git a/presto-array/src/test/java/com/facebook/presto/array/TestSliceBigArray.java b/presto-array/src/test/java/com/facebook/presto/array/TestSliceBigArray.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.array;
+
+import io.airlift.slice.Slice;
+import org.openjdk.jol.info.ClassLayout;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static io.airlift.slice.SizeOf.sizeOf;
+import static io.airlift.slice.Slices.wrappedBuffer;
+import static org.testng.Assert.assertEquals;
+
+@Test(singleThreaded = true)
+public class TestSliceBigArray
+{
+    private static final long BIG_ARRAY_INSTANCE_SIZE = ClassLayout.parseClass(SliceBigArray.class).instanceSize() + new ReferenceCountMap().sizeOf() + new ObjectBigArray<Slice>().sizeOf();
+    private static final long SLICE_INSTANCE_SIZE = ClassLayout.parseClass(Slice.class).instanceSize();
+    private static final int CAPACITY = 32;
+    private final byte[] firstBytes = new byte[1234];
+    private final byte[] secondBytes = new byte[4567];
+    private SliceBigArray sliceBigArray;
+
+    @BeforeMethod
+    public void setup()
+    {
+        sliceBigArray = new SliceBigArray();
+        sliceBigArray.ensureCapacity(CAPACITY);
+    }
+
+    @Test
+    public void testSameSliceRetainedSize()
+    {
+        // same slice should be counted only once
+        Slice slice = wrappedBuffer(secondBytes, 201, 1501);
+        for (int i = 0; i < CAPACITY; i++) {
+            sliceBigArray.set(i, slice);
+            assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE);
+        }
+
+        // adding a new slice will increase the size
+        slice = wrappedBuffer(secondBytes, 201, 1501);
+        sliceBigArray.set(3, slice);
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 2);
+    }
+
+    @Test
+    public void testNullSlicesRetainedSize()
+    {
+        // add null values
+        sliceBigArray.set(0, null);
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE);
+
+        // replace null with a slice
+        sliceBigArray.set(0, wrappedBuffer(secondBytes, 201, 1501));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE);
+
+        // add another slice
+        sliceBigArray.set(1, wrappedBuffer(secondBytes, 201, 1501));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 2);
+
+        // replace slice with null
+        sliceBigArray.set(1, null);
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE);
+    }
+
+    @Test
+    public void testRetainedSize()
+    {
+        // add two elements
+        sliceBigArray.set(0, wrappedBuffer(firstBytes, 0, 100));
+        sliceBigArray.set(1, wrappedBuffer(secondBytes, 0, 100));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(firstBytes) + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 2);
+
+        // add two more
+        sliceBigArray.set(2, wrappedBuffer(firstBytes, 100, 200));
+        sliceBigArray.set(3, wrappedBuffer(secondBytes, 20, 150));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(firstBytes) + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 4);
+
+        // replace with different slices but the same base
+        sliceBigArray.set(2, wrappedBuffer(firstBytes, 11, 1200));
+        sliceBigArray.set(3, wrappedBuffer(secondBytes, 201, 1501));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(firstBytes) + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 4);
+
+        // replace with a different slice with a different base
+        sliceBigArray.set(0, wrappedBuffer(secondBytes, 11, 1200));
+        sliceBigArray.set(2, wrappedBuffer(secondBytes, 201, 1501));
+        assertEquals(sliceBigArray.sizeOf(), BIG_ARRAY_INSTANCE_SIZE + sizeOf(secondBytes) + SLICE_INSTANCE_SIZE * 4);
+    }
+}
diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-atop</artifactId>

diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-base-jdbc</artifactId>

diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchmark-driver</artifactId>

diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>presto-root</artifactId>
         <groupId>com.facebook.presto</groupId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchmark</artifactId>

diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashBuildAndJoinBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashBuildAndJoinBenchmark.java
@@ -93,7 +93,7 @@ protected List<Driver> createDrivers(TaskContext taskContext)
                 ImmutableList.of(),
                 1_500_000,
                 1,
-                new PagesIndex.TestingFactory(),
+                new PagesIndex.TestingFactory(false),
                 false,
                 SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
         driversBuilder.add(hashBuilder);

diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashBuildBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashBuildBenchmark.java
@@ -67,7 +67,7 @@ protected List<Driver> createDrivers(TaskContext taskContext)
                 ImmutableList.of(),
                 1_500_000,
                 1,
-                new PagesIndex.TestingFactory(),
+                new PagesIndex.TestingFactory(false),
                 false,
                 SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
         DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, true, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty());

diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java
@@ -76,7 +76,7 @@ protected List<Driver> createDrivers(TaskContext taskContext)
                     ImmutableList.of(),
                     1_500_000,
                     1,
-                    new PagesIndex.TestingFactory(),
+                    new PagesIndex.TestingFactory(false),
                     false,
                     SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
 

diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/OrderByBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/OrderByBenchmark.java
@@ -51,7 +51,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
                 ROWS,
                 ImmutableList.of(0),
                 ImmutableList.of(ASC_NULLS_LAST),
-                new PagesIndex.TestingFactory());
+                new PagesIndex.TestingFactory(false));
 
         return ImmutableList.of(tableScanOperator, limitOperator, orderByOperator);
     }

diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-benchto-benchmarks</artifactId>

diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-blackhole</artifactId>

diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-bytecode</artifactId>

diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.facebook.presto</groupId>
         <artifactId>presto-root</artifactId>
-        <version>0.187-tw-0.42</version>
+        <version>0.188-tw-0.43</version>
     </parent>
 
     <artifactId>presto-cassandra</artifactId>