215 deterministic ordering v2

planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/collection/BenchmarkKWayMerge.java

@@ -1,21 +1,23 @@
 package com.onthegomap.planetiler.collection;
 
+import java.nio.ByteBuffer;
 import java.time.Duration;
 import java.util.PriorityQueue;
 import java.util.Random;
 import java.util.function.IntFunction;
 import java.util.stream.IntStream;
 
 /**
- * Performance tests for {@link LongMinHeap} implementations.
+ * Performance tests for {@link SortableFeatureMinHeap} implementations.
  *
  * Times how long it takes to merge N sorted lists of random elements.
  */
 public class BenchmarkKWayMerge {
+
   public static void main(String[] args) {
     for (int i = 0; i < 4; i++) {
       System.err.println();
-      testMinHeap("quaternary", LongMinHeap::newArrayHeap);
+      testMinHeap("quaternary", SortableFeatureMinHeap::newArrayHeap);
       System.err.println(String.join("\t",
         "priorityqueue",
         Long.toString(testPriorityQueue(10).toMillis()),
@@ -25,7 +27,7 @@ public static void main(String[] args) {
     }
   }
 
-  private static void testMinHeap(String name, IntFunction<LongMinHeap> constructor) {
+  private static void testMinHeap(String name, IntFunction<SortableFeatureMinHeap> constructor) {
     System.err.println(String.join("\t",
       name,
       Long.toString(testUpdates(10, constructor).toMillis()),
@@ -36,20 +38,28 @@ private static void testMinHeap(String name, IntFunction<LongMinHeap> constructo
 
   private static final Random random = new Random();
 
-  private static long[][] getVals(int size) {
+  final static ByteBuffer byteBuffer = ByteBuffer.allocate(Long.BYTES);
+
+  private static SortableFeature newVal(long i) {
+    byteBuffer.clear().putLong(i);
+    return new SortableFeature(i, byteBuffer.array());
+  }
+
+  private static SortableFeature[][] getVals(int size) {
     int num = 10_000_000;
     return IntStream.range(0, size)
       .mapToObj(i -> random
         .longs(0, 1_000_000_000)
         .limit(num / size)
         .sorted()
-        .toArray()
-      ).toArray(long[][]::new);
+        .mapToObj(BenchmarkKWayMerge::newVal)
+        .toArray(SortableFeature[]::new)
+      ).toArray(SortableFeature[][]::new);
   }
 
-  private static Duration testUpdates(int size, IntFunction<LongMinHeap> heapFn) {
+  private static Duration testUpdates(int size, IntFunction<SortableFeatureMinHeap> heapFn) {
     int[] indexes = new int[size];
-    long[][] vals = getVals(size);
+    SortableFeature[][] vals = getVals(size);
     var heap = heapFn.apply(size);
     for (int i = 0; i < size; i++) {
       heap.push(i, vals[i][indexes[i]++]);
@@ -58,7 +68,7 @@ private static Duration testUpdates(int size, IntFunction<LongMinHeap> heapFn) {
     while (!heap.isEmpty()) {
       int id = heap.peekId();
       int index = indexes[id]++;
-      long[] valList = vals[id];
+      SortableFeature[] valList = vals[id];
       if (index < valList.length) {
         heap.updateHead(valList[index]);
       } else {
@@ -68,52 +78,24 @@ private static Duration testUpdates(int size, IntFunction<LongMinHeap> heapFn) {
     return Duration.ofNanos(System.nanoTime() - start);
   }
 
-  static class Item implements Comparable<Item> {
-    long value;
-    int id;
-
-    @Override
-    public int compareTo(Item o) {
-      return Long.compare(value, o.value);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) {
-        return true;
-      }
-      if (o == null || getClass() != o.getClass()) {
-        return false;
-      }
-
-      Item item = (Item) o;
-
-      return value == item.value;
-    }
-
-    @Override
-    public int hashCode() {
-      return (int) (value ^ (value >>> 32));
-    }
-  }
-
   private static Duration testPriorityQueue(int size) {
-    long[][] vals = getVals(size);
+    SortableFeature[][] vals = getVals(size);
     int[] indexes = new int[size];
-    PriorityQueue<Item> heap = new PriorityQueue<>();
+    PriorityQueue<SortableFeature> heap = new PriorityQueue<>();
     for (int i = 0; i < size; i++) {
-      Item item = new Item();
-      item.id = i;
-      item.value = vals[i][indexes[i]++];
+      byteBuffer.clear().putLong(i);
+      SortableFeature temp = vals[i][indexes[i]++];
+      SortableFeature item = new SortableFeature(temp.key(), byteBuffer.array());
       heap.offer(item);
     }
     var start = System.nanoTime();
     while (!heap.isEmpty()) {
       var item = heap.poll();
-      int index = indexes[item.id]++;
-      long[] valList = vals[item.id];
+      int id = (int) byteBuffer.clear().put(item.value()).rewind().getLong();
+      int index = indexes[id]++;
+      SortableFeature[] valList = vals[id];
       if (index < valList.length) {
-        item.value = valList[index];
+        item = valList[index];
         heap.offer(item);
       }
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java

@@ -30,11 +30,11 @@
 import com.onthegomap.planetiler.util.LayerAttrStats;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -377,8 +377,7 @@ public static List<Feature> decode(byte[] encoded) {
         }
 
         for (VectorTileProto.Tile.Feature feature : layer.getFeaturesList()) {
-          int tagsCount = feature.getTagsCount();
-          Map<String, Object> attrs = HashMap.newHashMap(tagsCount / 2);
+          Map<String, Object> attrs = new TreeMap<>();
           int tagIdx = 0;
           while (tagIdx < feature.getTagsCount()) {
             String key = keys.get(feature.getTags(tagIdx++));

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArrayLongMinHeap.java → planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ArraySortableFeatureMinHeap.java

@@ -36,25 +36,27 @@
  *
  * @see <a href="https://en.wikipedia.org/wiki/D-ary_heap">d-ary heap (wikipedia)</a>
  */
-class ArrayLongMinHeap implements LongMinHeap {
+class ArraySortableFeatureMinHeap implements SortableFeatureMinHeap {
   protected static final int NOT_PRESENT = -1;
   protected final int[] tree;
   protected final int[] positions;
   protected final long[] vals;
+  protected final SortableFeature[] sortableFeatures;
   protected final int max;
   protected int size;
 
   /**
    * @param elements the number of elements that can be stored in this heap. Currently the heap cannot be resized or
    *                 shrunk/trimmed after initial creation. elements-1 is the maximum id that can be stored in this heap
    */
-  ArrayLongMinHeap(int elements) {
+  ArraySortableFeatureMinHeap(int elements) {
     // we use an offset of one to make the arithmetic a bit simpler/more efficient, the 0th elements are not used!
     tree = new int[elements + 1];
     positions = new int[elements + 1];
     Arrays.fill(positions, NOT_PRESENT);
     vals = new long[elements + 1];
     vals[0] = Long.MIN_VALUE;
+    sortableFeatures = new SortableFeature[elements + 1];
     this.max = elements;
   }
 
@@ -77,7 +79,7 @@
   }
 
   @Override
-  public void push(int id, long value) {
+  public void push(int id, SortableFeature sf) {
     checkIdInRange(id);
     if (size == max) {
       throw new IllegalStateException("Cannot push anymore, the heap is already full. size: " + size);
@@ -89,7 +91,8 @@
     size++;
     tree[size] = id;
     positions[id] = size;
-    vals[size] = value;
+    vals[size] = sf.key();
+    sortableFeatures[size] = sf;
     percolateUp(size);
   }
 
@@ -100,25 +103,39 @@
   }
 
   @Override
-  public void update(int id, long value) {
+  public void update(int id, SortableFeature sf) {
     checkIdInRange(id);
     int index = positions[id];
     if (index < 0) {
       throw new IllegalStateException(
         "The heap does not contain: " + id + ". Use the contains method to check this before calling update");
     }
     long prev = vals[index];
+    long value = sf.key();
     vals[index] = value;
     if (value > prev) {
+      sortableFeatures[index] = sf;
       percolateDown(index);
     } else if (value < prev) {
+      sortableFeatures[index] = sf;
       percolateUp(index);
+    } else {
+      byte[] bytes = sf.value();
+      byte[] prevBytes = sortableFeatures[index].value();
+      sortableFeatures[index] = sf;
+      int compareResult = Arrays.compare(bytes, prevBytes);
+      if (compareResult > 0) {
+        percolateDown(index);
+      } else {
+        percolateUp(index);
+      }
     }
   }
 
   @Override
-  public void updateHead(long value) {
-    vals[1] = value;
+  public void updateHead(SortableFeature sf) {
+    vals[1] = sf.key();
+    sortableFeatures[1] = sf;
     percolateDown(1);
   }
 
@@ -128,15 +145,17 @@
   }
 
   @Override
-  public long peekValue() {
-    return vals[1];
+  public SortableFeature peekValue() {
+    return sortableFeatures[1];
   }
 
   @Override
   public int poll() {
     int id = peekId();
     tree[1] = tree[size];
     vals[1] = vals[size];
+    sortableFeatures[1] = sortableFeatures[size];
+    sortableFeatures[size] = null;
     positions[tree[1]] = 1;
     positions[id] = NOT_PRESENT;
     size--;
@@ -152,6 +171,26 @@
     size = 0;
   }
 
+  private void switchSortableFeatures(int index1, int index2) {
+    final SortableFeature temp = sortableFeatures[index1];
+    sortableFeatures[index1] = sortableFeatures[index2];
+    sortableFeatures[index2] = temp;
+  }
+
+  private byte[] getValue(SortableFeature sf) {
+    if (sf == null) {
+      return null;
+    }
+    return sf.value();
+  }
+
+  private boolean isLessThanParent(int index, int parent, long val, long parentValue) {
+    if (val == parentValue) {
+      return Arrays.compare(getValue(sortableFeatures[index]), getValue(sortableFeatures[parent])) < 0;
+    }
+    return val < parentValue;
+  }
+
   private void percolateUp(int index) {
     assert index != 0;
     if (index == 1) {
@@ -162,8 +201,9 @@
     // the finish condition (index==0) is covered here automatically because we set vals[0]=-inf
     int parent;
     long parentValue;
-    while (val < (parentValue = vals[parent = parent(index)])) {
+    while (isLessThanParent(index, parent = parent(index), val, parentValue = vals[parent])) {
       vals[index] = parentValue;
+      switchSortableFeatures(index, parent);
       positions[tree[index] = tree[parent]] = index;
       index = parent;
     }
@@ -187,7 +227,7 @@
     final int el = tree[index];
     final long val = vals[index];
     int child;
     while ((child = firstChild(index)) <= size) {
       // optimization: this is a very hot code path for performance of k-way merging,
       // so manually-unroll the loop over the 4 child elements to find the minimum value
       int minChild = child;
@@ -208,10 +248,14 @@
           }
         }
       }
-      if (minValue >= val) {
+      if (minValue > val) {
+        break;
+      } else if (minValue == val &&
+        Arrays.compare(sortableFeatures[minChild].value(), sortableFeatures[index].value()) >= 0) {
         break;
       }
       vals[index] = minValue;
+      switchSortableFeatures(index, minChild);
       positions[tree[index] = tree[minChild]] = index;
       index = minChild;
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java

@@ -253,7 +253,7 @@ public Iterator<SortableFeature> iterator(int shard, int shards) {
       }
     }
 
-    return LongMerger.mergeIterators(iterators);
+    return SortableFeatureMerger.mergeIterators(iterators);
   }
 
   public int chunks() {

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java

@@ -21,11 +21,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import javax.annotation.concurrent.NotThreadSafe;
@@ -404,7 +404,7 @@ private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) {
         GeometryType geomType = decodeGeomType(geomTypeAndScale);
         int scale = decodeScale(geomTypeAndScale);
         int mapSize = unpacker.unpackMapHeader();
-        Map<String, Object> attrs = HashMap.newHashMap(mapSize);
+        Map<String, Object> attrs = new TreeMap<>();
         for (int i = 0; i < mapSize; i++) {
           String key = commonValueStrings.decode(unpacker.unpackInt());
           Value v = unpacker.unpackValue();

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureSort.java

@@ -131,7 +131,7 @@ default ParallelIterator parallelIterator(Stats stats, int threads) {
         }
       }
     });
-    return new ParallelIterator(reader, LongMerger.mergeSuppliers(queues));
+    return new ParallelIterator(reader, SortableFeatureMerger.mergeSuppliers(queues));
   }
 
   record ParallelIterator(Worker reader, @Override Iterator<SortableFeature> iterator)

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/HasLongSortKey.java

@@ -4,7 +4,7 @@
  * An item with a {@code long key} that can be used for sorting/grouping.
  *
  * These items can be sorted or grouped by {@link FeatureSort}/{@link FeatureGroup} implementations. Sorted lists can
- * also be merged using {@link LongMerger}.
+ * also be merged using {@link SortableFeatureMerger}.
  */
 public interface HasLongSortKey {
   /** Value to sort/group items by. */

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/SortableFeature.java

@@ -6,7 +6,11 @@ public record SortableFeature(@Override long key, byte[] value) implements Compa
 
   @Override
   public int compareTo(SortableFeature o) {
-    return Long.compare(key, o.key);
+    int result = Long.compare(key, o.key);
+    if (result == 0) {
+      result = Arrays.compare(value, o.value);
+    }
+    return result;
   }
 
   @Override