Merge branch 'master' into lazier-ingest-geoip

* master: (30 commits)
  Revert "[Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (elastic#35320)"
  Deprecate types in get_source and exist_source (elastic#36426)
  Fix duplicate phrase in shrink/split error message (elastic#36734)
  ingest: support default pipelines + bulk upserts (elastic#36618)
  TESTS:Debug Log. IndexStatsIT#testFilterCacheStats
  [Geo] Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach (elastic#35320)
  [TEST] fix float comparison in RandomObjects#getExpectedParsedValue
  Initialize startup `CcrRepositories` (elastic#36730)
  ingest: fix on_failure with Drop processor (elastic#36686)
  SNAPSHOTS: Adjust BwC Versions in Restore Logic (elastic#36718)
  [Painless] Add boxed type to boxed type casts for method/return (elastic#36571)
  Do not resolve addresses in remote connection info (elastic#36671)
  Add back one line removed by mistake regarding java version check and COMPAT jvm parameter existence
  Fixing line length for EnvironmentTests and RecoveryTests (elastic#36657)
  SQL: Fix translation of LIKE/RLIKE keywords (elastic#36672)
  [DOCS] Adds monitoring requirement for ingest node (elastic#36665)
  SNAPSHOTS: Disable BwC Tests Until elastic#36659 Landed (elastic#36709)
  Add doc's sequence number + primary term to GetResult and use it for updates (elastic#36680)
  [CCR] Add time since last auto follow fetch to auto follow stats (elastic#36542)
  Watcher accounts constructed lazily (elastic#36656)
  ...

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -64,17 +64,11 @@
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]threadpool[/\\]ThreadPool.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]aliases[/\\]IndexAliasesIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]deps[/\\]joda[/\\]SimpleJodaTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]EnvironmentTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]NodeEnvironmentTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]explain[/\\]ExplainActionIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]get[/\\]GetActionIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indexing[/\\]IndexActionIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RecoveryWhileUnderLoadIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RelocationIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]TruncatedRecoveryIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponseTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasRoutingIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]SimpleRoutingIT.java" checks="LineLength" />

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -268,8 +268,14 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
     }
 
     static Request sourceExists(GetRequest getRequest) {
-        Request request = new Request(HttpHead.METHOD_NAME, endpoint(getRequest.index(), getRequest.type(), getRequest.id(), "_source"));
-
+        String optionalType = getRequest.type();
+        String endpoint;
+        if (optionalType.equals(MapperService.SINGLE_MAPPING_NAME)) {
+            endpoint = endpoint(getRequest.index(), "_source", getRequest.id());
+        } else {
+            endpoint = endpoint(getRequest.index(), optionalType, getRequest.id(), "_source");
+        }
+        Request request = new Request(HttpHead.METHOD_NAME, endpoint);
         Params parameters = new Params(request);
         parameters.withPreference(getRequest.preference());
         parameters.withRouting(getRequest.routing());

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/AutoFollowStats.java

@@ -39,6 +39,10 @@ public final class AutoFollowStats {
     static final ParseField RECENT_AUTO_FOLLOW_ERRORS = new ParseField("recent_auto_follow_errors");
     static final ParseField LEADER_INDEX = new ParseField("leader_index");
     static final ParseField AUTO_FOLLOW_EXCEPTION = new ParseField("auto_follow_exception");
+    static final ParseField AUTO_FOLLOWED_CLUSTERS = new ParseField("auto_followed_clusters");
+    static final ParseField CLUSTER_NAME = new ParseField("cluster_name");
+    static final ParseField TIME_SINCE_LAST_CHECK_MILLIS = new ParseField("time_since_last_check_millis");
+    static final ParseField LAST_SEEN_METADATA_VERSION = new ParseField("last_seen_metadata_version");
 
     @SuppressWarnings("unchecked")
     static final ConstructingObjectParser<AutoFollowStats, Void> STATS_PARSER = new ConstructingObjectParser<>("auto_follow_stats",
@@ -48,6 +52,10 @@ public final class AutoFollowStats {
             (Long) args[2],
             new TreeMap<>(
                 ((List<Map.Entry<String, ElasticsearchException>>) args[3])
+                    .stream()
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))),
+            new TreeMap<>(
+                ((List<Map.Entry<String, AutoFollowedCluster>>) args[4])
                     .stream()
                     .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
         ));
@@ -57,33 +65,47 @@ public final class AutoFollowStats {
             "auto_follow_stats_errors",
             args -> new AbstractMap.SimpleEntry<>((String) args[0], (ElasticsearchException) args[1]));
 
+    private static final ConstructingObjectParser<Map.Entry<String, AutoFollowedCluster>, Void> AUTO_FOLLOWED_CLUSTERS_PARSER =
+        new ConstructingObjectParser<>(
+            "auto_followed_clusters",
+            args -> new AbstractMap.SimpleEntry<>((String) args[0], new AutoFollowedCluster((Long) args[1], (Long) args[2])));
+
     static {
         AUTO_FOLLOW_EXCEPTIONS_PARSER.declareString(ConstructingObjectParser.constructorArg(), LEADER_INDEX);
         AUTO_FOLLOW_EXCEPTIONS_PARSER.declareObject(
             ConstructingObjectParser.constructorArg(),
             (p, c) -> ElasticsearchException.fromXContent(p),
             AUTO_FOLLOW_EXCEPTION);
 
+        AUTO_FOLLOWED_CLUSTERS_PARSER.declareString(ConstructingObjectParser.constructorArg(), CLUSTER_NAME);
+        AUTO_FOLLOWED_CLUSTERS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), TIME_SINCE_LAST_CHECK_MILLIS);
+        AUTO_FOLLOWED_CLUSTERS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_SEEN_METADATA_VERSION);
+
         STATS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), NUMBER_OF_FAILED_INDICES_AUTO_FOLLOWED);
         STATS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), NUMBER_OF_FAILED_REMOTE_CLUSTER_STATE_REQUESTS);
         STATS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), NUMBER_OF_SUCCESSFUL_INDICES_AUTO_FOLLOWED);
         STATS_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), AUTO_FOLLOW_EXCEPTIONS_PARSER,
             RECENT_AUTO_FOLLOW_ERRORS);
+        STATS_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), AUTO_FOLLOWED_CLUSTERS_PARSER,
+            AUTO_FOLLOWED_CLUSTERS);
     }
 
     private final long numberOfFailedFollowIndices;
     private final long numberOfFailedRemoteClusterStateRequests;
     private final long numberOfSuccessfulFollowIndices;
     private final NavigableMap<String, ElasticsearchException> recentAutoFollowErrors;
+    private final NavigableMap<String, AutoFollowedCluster> autoFollowedClusters;
 
     AutoFollowStats(long numberOfFailedFollowIndices,
                     long numberOfFailedRemoteClusterStateRequests,
                     long numberOfSuccessfulFollowIndices,
-                    NavigableMap<String, ElasticsearchException> recentAutoFollowErrors) {
+                    NavigableMap<String, ElasticsearchException> recentAutoFollowErrors,
+                    NavigableMap<String, AutoFollowedCluster> autoFollowedClusters) {
         this.numberOfFailedFollowIndices = numberOfFailedFollowIndices;
         this.numberOfFailedRemoteClusterStateRequests = numberOfFailedRemoteClusterStateRequests;
         this.numberOfSuccessfulFollowIndices = numberOfSuccessfulFollowIndices;
         this.recentAutoFollowErrors = recentAutoFollowErrors;
+        this.autoFollowedClusters = autoFollowedClusters;
     }
 
     public long getNumberOfFailedFollowIndices() {
@@ -102,4 +124,27 @@ public NavigableMap<String, ElasticsearchException> getRecentAutoFollowErrors()
         return recentAutoFollowErrors;
     }
 
+    public NavigableMap<String, AutoFollowedCluster> getAutoFollowedClusters() {
+        return autoFollowedClusters;
+    }
+
+    public static class AutoFollowedCluster {
+
+        private final long timeSinceLastCheckMillis;
+        private final long lastSeenMetadataVersion;
+
+        public AutoFollowedCluster(long timeSinceLastCheckMillis, long lastSeenMetadataVersion) {
+            this.timeSinceLastCheckMillis = timeSinceLastCheckMillis;
+            this.lastSeenMetadataVersion = lastSeenMetadataVersion;
+        }
+
+        public long getTimeSinceLastCheckMillis() {
+            return timeSinceLastCheckMillis;
+        }
+
+        public long getLastSeenMetadataVersion() {
+            return lastSeenMetadataVersion;
+        }
+    }
+
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -73,6 +73,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.rankeval.PrecisionAtK;
@@ -115,6 +116,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.StringJoiner;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -156,6 +158,58 @@ public void testGetWithType() {
         getAndExistsWithTypeTest(RequestConverters::get, HttpGet.METHOD_NAME);
     }
 
+    public void testSourceExists() throws IOException {
+        doTestSourceExists((index, id) -> new GetRequest(index, id));
+    }
+
+    public void testSourceExistsWithType() throws IOException {
+        String type = frequently() ? randomAlphaOfLengthBetween(3, 10) : MapperService.SINGLE_MAPPING_NAME;
+        doTestSourceExists((index, id) -> new GetRequest(index, type, id));
+    }
+
+    private static void doTestSourceExists(BiFunction<String, String, GetRequest> requestFunction) throws IOException {
+        String index = randomAlphaOfLengthBetween(3, 10);
+        String id = randomAlphaOfLengthBetween(3, 10);
+        final GetRequest getRequest = requestFunction.apply(index, id);
+
+        Map<String, String> expectedParams = new HashMap<>();
+        if (randomBoolean()) {
+            String preference = randomAlphaOfLengthBetween(3, 10);
+            getRequest.preference(preference);
+            expectedParams.put("preference", preference);
+        }
+        if (randomBoolean()) {
+            String routing = randomAlphaOfLengthBetween(3, 10);
+            getRequest.routing(routing);
+            expectedParams.put("routing", routing);
+        }
+        if (randomBoolean()) {
+            boolean realtime = randomBoolean();
+            getRequest.realtime(realtime);
+            if (realtime == false) {
+                expectedParams.put("realtime", "false");
+            }
+        }
+        if (randomBoolean()) {
+            boolean refresh = randomBoolean();
+            getRequest.refresh(refresh);
+            if (refresh) {
+                expectedParams.put("refresh", "true");
+            }
+        }
+        Request request = RequestConverters.sourceExists(getRequest);
+        assertEquals(HttpHead.METHOD_NAME, request.getMethod());
+        String type = getRequest.type();
+        if (type.equals(MapperService.SINGLE_MAPPING_NAME)) {
+            assertEquals("/" + index + "/_source/" + id, request.getEndpoint());
+        } else {
+            assertEquals("/" + index + "/" + type + "/" + id + "/_source", request.getEndpoint());
+        }
+
+        assertEquals(expectedParams, request.getParameters());
+        assertNull(request.getEntity());
+    }
+
     public void testMultiGet() throws IOException {
         Map<String, String> expectedParams = new HashMap<>();
         MultiGetRequest multiGetRequest = new MultiGetRequest();

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client.ccr;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.ccr.AutoFollowStats.AutoFollowedCluster;
 import org.elasticsearch.client.ccr.IndicesFollowStats.ShardFollowStats;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -185,6 +186,19 @@ private static void toXContent(CcrStatsResponse response, XContentBuilder builde
                     builder.endObject();
                 }
                 builder.endArray();
+                builder.startArray(AutoFollowStats.AUTO_FOLLOWED_CLUSTERS.getPreferredName());
+                for (Map.Entry<String, AutoFollowedCluster> entry : autoFollowStats.getAutoFollowedClusters().entrySet()) {
+                    builder.startObject();
+                    {
+                        builder.field(AutoFollowStats.CLUSTER_NAME.getPreferredName(), entry.getKey());
+                        builder.field(AutoFollowStats.TIME_SINCE_LAST_CHECK_MILLIS.getPreferredName(),
+                            entry.getValue().getTimeSinceLastCheckMillis());
+                        builder.field(AutoFollowStats.LAST_SEEN_METADATA_VERSION.getPreferredName(),
+                            entry.getValue().getLastSeenMetadataVersion());
+                    }
+                    builder.endObject();
+                }
+                builder.endArray();
             }
             builder.endObject();
 
@@ -315,11 +329,16 @@ private static AutoFollowStats randomAutoFollowStats() {
         for (int i = 0; i < count; i++) {
             readExceptions.put("" + i, new ElasticsearchException(new IllegalStateException("index [" + i + "]")));
         }
+        final NavigableMap<String, AutoFollowedCluster> autoFollowClusters = new TreeMap<>();
+        for (int i = 0; i < count; i++) {
+            autoFollowClusters.put("" + i, new AutoFollowedCluster(randomLong(), randomNonNegativeLong()));
+        }
         return new AutoFollowStats(
             randomNonNegativeLong(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            readExceptions
+            readExceptions,
+            autoFollowClusters
         );
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -1265,7 +1265,6 @@ public void testGet() throws Exception {
             assertEquals(3, getResponse.getSourceAsMap().size());
             //tag::get-response
             String index = getResponse.getIndex();
-            String type = getResponse.getType();
             String id = getResponse.getId();
             if (getResponse.isExists()) {
                 long version = getResponse.getVersion();

docs/plugins/ingest-attachment.asciidoc

@@ -63,6 +63,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "attachment": {
@@ -74,7 +76,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 To specify only some fields to be extracted:
@@ -146,6 +148,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 35,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "attachment": {
@@ -157,7 +161,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 [source,js]
@@ -194,6 +198,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id_2",
   "_version": 1,
+  "_seq_no": 40,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "max_size": 5,
@@ -206,7 +212,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 [[ingest-attachment-with-arrays]]
@@ -285,6 +291,8 @@ Returns this:
   "_type" : "_doc",
   "_id" : "my_id",
   "_version" : 1,
+  "_seq_no" : 50,
+  "_primary_term" : 1,
   "found" : true,
   "_source" : {
     "attachments" : [
@@ -312,7 +320,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Note that the `target_field` needs to be set, otherwise the

docs/plugins/ingest-geoip.asciidoc

@@ -75,6 +75,8 @@ Which returns:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 55,
+  "_primary_term": 1,
   "_source": {
     "ip": "8.8.8.8",
     "geoip": {
@@ -85,7 +87,7 @@ Which returns:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/]
 
 Here is an example that uses the default country database and adds the
 geographical information to the `geo` field based on the `ip` field`. Note that
@@ -124,6 +126,8 @@ returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 65,
+  "_primary_term": 1,
   "_source": {
     "ip": "8.8.8.8",
     "geo": {
@@ -133,7 +137,7 @@ returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Not all IP addresses find geo information from the database, When this
@@ -174,13 +178,15 @@ Which returns:
   "_type" : "_doc",
   "_id" : "my_id",
   "_version" : 1,
+  "_seq_no" : 71,
+  "_primary_term": 1,
   "found" : true,
   "_source" : {
     "ip" : "80.231.5.0"
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 [[ingest-geoip-mappings-note]]
 ===== Recognizing Location as a Geopoint