Merge branch 'master' into docs_recreate_analyzers_1

docs/reference/indices/forcemerge.asciidoc

@@ -10,6 +10,11 @@ This call will block until the merge is complete. If the http connection is
 lost, the request will continue in the background, and any new requests will
 block until the previous force merge is complete.
 
+WARNING: Force merge should only be called against *read-only indices*. Running 
+force merge against a read-write index can cause very large segments to be produced 
+(>5Gb per segment), and the merge policy will never consider it for merging again until 
+it mostly consists of deleted docs. This can cause very large segments to remain in the shards.
+
 [source,js]
 --------------------------------------------------
 POST /twitter/_forcemerge

gradle.properties

@@ -1,2 +1,2 @@
 org.gradle.daemon=false
-org.gradle.jvmargs=-Xmx1536m
+org.gradle.jvmargs=-Xmx1792m

server/src/main/java/org/elasticsearch/common/cache/Cache.java

@@ -206,34 +206,33 @@ private static class CacheSegment<K, V> {
          */
         Entry<K, V> get(K key, long now, Predicate<Entry<K, V>> isExpired, Consumer<Entry<K, V>> onExpiration) {
             CompletableFuture<Entry<K, V>> future;
-            Entry<K, V> entry = null;
             try (ReleasableLock ignored = readLock.acquire()) {
                 future = map.get(key);
             }
             if (future != null) {
+                Entry<K, V> entry;
                 try {
-                    entry = future.handle((ok, ex) -> {
-                        if (ok != null && !isExpired.test(ok)) {
-                            segmentStats.hit();
-                            ok.accessTime = now;
-                            return ok;
-                        } else {
-                            segmentStats.miss();
-                            if (ok != null) {
-                                assert isExpired.test(ok);
-                                onExpiration.accept(ok);
-                            }
-                            return null;
-                        }
-                    }).get();
-                } catch (ExecutionException | InterruptedException e) {
+                    entry = future.get();
+                } catch (ExecutionException e) {
+                    assert future.isCompletedExceptionally();
+                    segmentStats.miss();
+                    return null;
+                } catch (InterruptedException e) {
                     throw new IllegalStateException(e);
                 }
-            }
-            else {
+                if (isExpired.test(entry)) {
+                    segmentStats.miss();
+                    onExpiration.accept(entry);
+                    return null;
+                } else {
+                    segmentStats.hit();
+                    entry.accessTime = now;
+                    return entry;
+                }
+            } else {
                 segmentStats.miss();
+                return null;
             }
-            return entry;
         }
 
         /**
@@ -269,30 +268,18 @@ Tuple<Entry<K, V>, Entry<K, V>> put(K key, V value, long now) {
         /**
          * remove an entry from the segment
          *
-         * @param key the key of the entry to remove from the cache
-         * @return the removed entry if there was one, otherwise null
+         * @param key       the key of the entry to remove from the cache
+         * @param onRemoval a callback for the removed entry
          */
-        Entry<K, V> remove(K key) {
+        void remove(K key, Consumer<CompletableFuture<Entry<K, V>>> onRemoval) {
             CompletableFuture<Entry<K, V>> future;
-            Entry<K, V> entry = null;
             try (ReleasableLock ignored = writeLock.acquire()) {
                 future = map.remove(key);
             }
             if (future != null) {
-                try {
-                    entry = future.handle((ok, ex) -> {
-                        if (ok != null) {
-                            segmentStats.eviction();
-                            return ok;
-                        } else {
-                            return null;
-                        }
-                    }).get();
-                } catch (ExecutionException | InterruptedException e) {
-                    throw new IllegalStateException(e);
-                }
+                segmentStats.eviction();
+                onRemoval.accept(future);
             }
-            return entry;
         }
 
         private static class SegmentStats {
@@ -476,12 +463,18 @@ private void put(K key, V value, long now) {
      */
     public void invalidate(K key) {
         CacheSegment<K, V> segment = getCacheSegment(key);
-        Entry<K, V> entry = segment.remove(key);
-        if (entry != null) {
-            try (ReleasableLock ignored = lruLock.acquire()) {
-                delete(entry, RemovalNotification.RemovalReason.INVALIDATED);
+        segment.remove(key, f -> {
+            try {
+                Entry<K, V> entry = f.get();
+                try (ReleasableLock ignored = lruLock.acquire()) {
+                    delete(entry, RemovalNotification.RemovalReason.INVALIDATED);
+                }
+            } catch (ExecutionException e) {
+                // ok
+            } catch (InterruptedException e) {
+                throw new IllegalStateException(e);
             }
-        }
+        });
     }
 
     /**
@@ -632,7 +625,7 @@ public void remove() {
             Entry<K, V> entry = current;
             if (entry != null) {
                 CacheSegment<K, V> segment = getCacheSegment(entry.key);
-                segment.remove(entry.key);
+                segment.remove(entry.key, f -> {});
                 try (ReleasableLock ignored = lruLock.acquire()) {
                     current = null;
                     delete(entry, RemovalNotification.RemovalReason.INVALIDATED);
@@ -717,7 +710,7 @@ private void evictEntry(Entry<K, V> entry) {
 
         CacheSegment<K, V> segment = getCacheSegment(entry.key);
         if (segment != null) {
-            segment.remove(entry.key);
+            segment.remove(entry.key, f -> {});
         }
         delete(entry, RemovalNotification.RemovalReason.EVICTED);
     }

server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java

@@ -24,6 +24,7 @@
 import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.search.SortedSetSortField;
 import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@@ -80,6 +81,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30416")
 public class SplitIndexIT extends ESIntegTestCase {
 
     @Override
@@ -283,7 +285,6 @@ public void assertAllUniqueDocs(SearchResponse response, int numDocs) {
         assertEquals(numDocs, ids.size());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30432")
     public void testSplitIndexPrimaryTerm() throws Exception {
         final List<Integer> factors = Arrays.asList(1, 2, 4, 8);
         final List<Integer> numberOfShardsFactors = randomSubsetOf(scaledRandomIntBetween(1, factors.size()), factors);

server/src/test/java/org/elasticsearch/common/cache/CacheTests.java

@@ -344,7 +344,6 @@ protected long now() {
         assertEquals(numberOfEntries, cache.stats().getEvictions());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30428")
     public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, InterruptedException {
         final int numberOfThreads = randomIntBetween(2, 32);
         final Cache<Integer, String> cache =

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java

@@ -22,11 +22,14 @@
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.core.TestXPackTransportClient;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
+import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 
@@ -39,7 +42,12 @@
 import java.nio.charset.StandardCharsets;
 import java.security.KeyStore;
 import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore;
 import static org.hamcrest.CoreMatchers.is;
@@ -65,12 +73,18 @@ protected boolean transportSSLEnabled() {
     }
 
     // no SSL exception as this is the exception is returned when connecting
-    public void testThatUnconfiguredCiphersAreRejected() {
+    public void testThatUnconfiguredCiphersAreRejected() throws Exception {
+        Set<String> supportedCiphers = Sets.newHashSet(SSLContext.getDefault().getSupportedSSLParameters().getCipherSuites());
+        Set<String> defaultXPackCiphers = Sets.newHashSet(XPackSettings.DEFAULT_CIPHERS);
+        final List<String> unconfiguredCiphers = new ArrayList<>(Sets.difference(supportedCiphers, defaultXPackCiphers));
+        Collections.shuffle(unconfiguredCiphers, random());
+        assumeFalse("the unconfigured ciphers list is empty", unconfiguredCiphers.isEmpty());
+
         try (TransportClient transportClient = new TestXPackTransportClient(Settings.builder()
                 .put(transportClientSettings())
                 .put("node.name", "programmatic_transport_client")
                 .put("cluster.name", internalCluster().getClusterName())
-                .putList("xpack.ssl.cipher_suites", "TLS_ECDH_anon_WITH_RC4_128_SHA", "SSL_RSA_WITH_3DES_EDE_CBC_SHA")
+                .putList("xpack.ssl.cipher_suites", unconfiguredCiphers)
                 .build(), LocalStateSecurity.class)) {
 
             TransportAddress transportAddress = randomFrom(internalCluster().getInstance(Transport.class).boundAddress().boundAddresses());