Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x:
  Use correct formatting for links (#29460)
  Revert "Adds a new auto-interval date histogram (#28993)"
  Revert "fix typo"
  fix typo
  Adds a new auto-interval date histogram (#28993)
  [Rollup] Replace RollupIT with a ESRestTestCase version (#31977)
  [Rollup] Fix duplicate field names in test (#32075)
  [Tests] Fix failure due to changes exception message (#32036)
  [Test] Mute MlJobIT#testDeleteJobAfterMissingAliases
  Replace Ingest ScriptContext with Custom Interface (#32003) (#32060)
  Cleanup Duplication in `PainlessScriptEngine` (#31991) (#32061)
  HLRC: Add xpack usage api (#31975)
  Clean Up Snapshot Create Rest API (#31779)

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -105,6 +105,7 @@
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -1092,6 +1093,13 @@ static Request xPackInfo(XPackInfoRequest infoRequest) {
         return request;
     }
 
+    static Request xpackUsage(XPackUsageRequest usageRequest) {
+        Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(usageRequest.masterNodeTimeout());
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

@@ -174,7 +174,7 @@ public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryReques
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
      * API on elastic.co</a>
      */
-    public CreateSnapshotResponse createSnapshot(CreateSnapshotRequest createSnapshotRequest, RequestOptions options)
+    public CreateSnapshotResponse create(CreateSnapshotRequest createSnapshotRequest, RequestOptions options)
         throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(createSnapshotRequest, RequestConverters::createSnapshot, options,
             CreateSnapshotResponse::fromXContent, emptySet());
@@ -186,7 +186,7 @@ public CreateSnapshotResponse createSnapshot(CreateSnapshotRequest createSnapsho
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
      * API on elastic.co</a>
      */
-    public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, RequestOptions options,
+    public void createAsync(CreateSnapshotRequest createSnapshotRequest, RequestOptions options,
                                     ActionListener<CreateSnapshotResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(createSnapshotRequest, RequestConverters::createSnapshot, options,
             CreateSnapshotResponse::fromXContent, listener, emptySet());

client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java

@@ -22,6 +22,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageResponse;
 
 import java.io.IOException;
 
@@ -70,4 +72,25 @@ public void infoAsync(XPackInfoRequest request, RequestOptions options,
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackInfo, options,
             XPackInfoResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Fetch usage information about X-Pack features from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public XPackUsageResponse usage(XPackUsageRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xpackUsage, options,
+            XPackUsageResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously fetch usage information about X-Pack features from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void usageAsync(XPackUsageRequest request, RequestOptions options, ActionListener<XPackUsageResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options,
+            XPackUsageResponse::fromXContent, listener, emptySet());
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java

@@ -57,8 +57,8 @@ private PutRepositoryResponse createTestRepository(String repository, String typ
     private CreateSnapshotResponse createTestSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
         // assumes the repository already exists
 
-        return execute(createSnapshotRequest, highLevelClient().snapshot()::createSnapshot,
-            highLevelClient().snapshot()::createSnapshotAsync);
+        return execute(createSnapshotRequest, highLevelClient().snapshot()::create,
+            highLevelClient().snapshot()::createAsync);
     }
 
     public void testCreateRepository() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java

@@ -35,12 +35,17 @@
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageResponse;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.is;
+
 /**
  * Documentation for miscellaneous APIs in the high level java client.
  * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
@@ -130,6 +135,50 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testXPackUsage() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::x-pack-usage-execute
+            XPackUsageRequest request = new XPackUsageRequest();
+            XPackUsageResponse response = client.xpack().usage(request, RequestOptions.DEFAULT);
+            //end::x-pack-usage-execute
+
+            //tag::x-pack-usage-response
+            Map<String, Map<String, Object>> usages = response.getUsages();
+            Map<String, Object> monitoringUsage = usages.get("monitoring");
+            assertThat(monitoringUsage.get("available"), is(true));
+            assertThat(monitoringUsage.get("enabled"), is(true));
+            assertThat(monitoringUsage.get("collection_enabled"), is(false));
+            //end::x-pack-usage-response
+        }
+        {
+            XPackUsageRequest request = new XPackUsageRequest();
+            // tag::x-pack-usage-execute-listener
+            ActionListener<XPackUsageResponse> listener = new ActionListener<XPackUsageResponse>() {
+                @Override
+                public void onResponse(XPackUsageResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-usage-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-usage-execute-async
+            client.xpack().usageAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-usage-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     public void testInitializationFromClientBuilder() throws IOException {
         //tag::rest-high-level-client-init
         RestHighLevelClient client = new RestHighLevelClient(

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

@@ -337,8 +337,9 @@ public void testSearchRequestAggregations() throws IOException {
                     Range range = aggregations.get("by_company"); // <1>
                     // end::search-request-aggregations-get-wrongCast
                 } catch (ClassCastException ex) {
-                    assertEquals("org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms"
-                            + " cannot be cast to org.elasticsearch.search.aggregations.bucket.range.Range", ex.getMessage());
+                    String message = ex.getMessage();
+                    assertThat(message, containsString("org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms"));
+                    assertThat(message, containsString("org.elasticsearch.search.aggregations.bucket.range.Range"));
                 }
                 assertEquals(3, elasticBucket.getDocCount());
                 assertEquals(30, avg, 0.0);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -420,14 +420,20 @@ public void testSnapshotCreate() throws IOException {
         // end::create-snapshot-request-waitForCompletion
 
         // tag::create-snapshot-execute
-        CreateSnapshotResponse response = client.snapshot().createSnapshot(request, RequestOptions.DEFAULT);
+        CreateSnapshotResponse response = client.snapshot().create(request, RequestOptions.DEFAULT);
         // end::create-snapshot-execute
 
         // tag::create-snapshot-response
         RestStatus status = response.status(); // <1>
         // end::create-snapshot-response
 
         assertEquals(RestStatus.OK, status);
+
+        // tag::create-snapshot-response-snapshot-info
+        SnapshotInfo snapshotInfo = response.getSnapshotInfo(); // <1>
+        // end::create-snapshot-response-snapshot-info
+
+        assertNotNull(snapshotInfo);
     }
 
     public void testSnapshotCreateAsync() throws InterruptedException {
@@ -455,7 +461,7 @@ public void onFailure(Exception exception) {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::create-snapshot-execute-async
-            client.snapshot().createSnapshotAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.snapshot().createAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::create-snapshot-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));

docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc

@@ -0,0 +1,54 @@
+[[java-rest-high-x-pack-usage]]
+=== X-Pack Usage API
+
+[[java-rest-high-x-pack-usage-execution]]
+==== Execution
+
+Detailed information about the usage of features from {xpack} can be
+retrieved using the `usage()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-info-response]]
+==== Response
+
+The returned `XPackUsageResponse` contains a `Map` keyed by feature name.
+Every feature map has an `available` key, indicating whether that
+feature is available given the current license, and an `enabled` key,
+indicating whether that feature is currently enabled. Other keys
+are specific to each feature.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-response]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-usage-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute-async]
+--------------------------------------------------
+<1> The call to execute the usage api and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `XPackUsageResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument

docs/java-rest/high-level/snapshot/create_snapshot.asciidoc

@@ -73,11 +73,22 @@ include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-r
 [[java-rest-high-snapshot-create-snapshot-sync]]
 ==== Synchronous Execution
 
+Execute a `CreateSnapshotRequest` synchronously to receive a `CreateSnapshotResponse`.
+
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-execute]
 --------------------------------------------------
 
+Retrieve the `SnapshotInfo` from a `CreateSnapshotResponse` when the snapshot is fully created.
+(The `waitForCompletion` parameter is `true`).
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-response-snapshot-info]
+--------------------------------------------------
+<1> The `SnapshotInfo` object.
+
 [[java-rest-high-snapshot-create-snapshot-async]]
 ==== Asynchronous Execution
 

docs/reference/how-to/recipes.asciidoc

@@ -3,8 +3,8 @@
 
 This section includes a few recipes to help with common problems:
 
-* mixing-exact-search-with-stemming
-* consistent-scoring
+* <<mixing-exact-search-with-stemming>>
+* <<consistent-scoring>>
 
 include::recipes/stemming.asciidoc[]
 include::recipes/scoring.asciidoc[]

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java

@@ -26,7 +26,7 @@
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.IngestScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
@@ -71,10 +71,8 @@ public final class ScriptProcessor extends AbstractProcessor {
      */
     @Override
     public void execute(IngestDocument document) {
-        ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
-        ExecutableScript executableScript = factory.newInstance(script.getParams());
-        executableScript.setNextVar("ctx",  document.getSourceAndMetadata());
-        executableScript.run();
+        IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
+        factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata());
     }
 
     @Override
@@ -144,7 +142,7 @@ public ScriptProcessor create(Map<String, Processor.Factory> registry, String pr
 
             // verify script is able to be compiled before successfully creating processor.
             try {
-                scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
+                scriptService.compile(script, IngestScript.CONTEXT);
             } catch (ScriptException e) {
                 throw newConfigurationException(TYPE, processorTag, scriptPropertyUsed, e);
             }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java

@@ -58,9 +58,7 @@ protected boolean ignoreExternalCluster() {
     public static class CustomScriptPlugin extends MockScriptPlugin {
         @Override
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
-            return Collections.singletonMap("my_script", script -> {
-                @SuppressWarnings("unchecked")
-                Map<String, Object> ctx = (Map) script.get("ctx");
+            return Collections.singletonMap("my_script", ctx -> {
                 ctx.put("z", 0);
                 return null;
             });

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java

@@ -19,47 +19,51 @@
 
 package org.elasticsearch.ingest.common;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.core.Is.is;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class ScriptProcessorTests extends ESTestCase {
 
     public void testScripting() throws Exception {
         int randomBytesIn = randomInt();
         int randomBytesOut = randomInt();
         int randomBytesTotal = randomBytesIn + randomBytesOut;
-
-        ScriptService scriptService = mock(ScriptService.class);
-        Script script = mockScript("_script");
-        ExecutableScript.Factory factory = mock(ExecutableScript.Factory.class);
-        ExecutableScript executableScript = mock(ExecutableScript.class);
-        when(scriptService.compile(script, ExecutableScript.INGEST_CONTEXT)).thenReturn(factory);
-        when(factory.newInstance(any())).thenReturn(executableScript);
+        String scriptName = "script";
+        ScriptService scriptService = new ScriptService(Settings.builder().build(),
+            Collections.singletonMap(
+                Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
+                    Script.DEFAULT_SCRIPT_LANG,
+                    Collections.singletonMap(
+                        scriptName, ctx -> {
+                            ctx.put("bytes_total", randomBytesTotal);
+                            return null;
+                        }
+                    )
+                )
+            ),
+            new HashMap<>(ScriptModule.CORE_CONTEXTS)
+        );
+        Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
 
         Map<String, Object> document = new HashMap<>();
         document.put("bytes_in", randomInt());
         document.put("bytes_out", randomInt());
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        doAnswer(invocationOnMock ->  {
-            ingestDocument.setFieldValue("bytes_total", randomBytesTotal);
-            return null;
-        }).when(executableScript).run();
-
         ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);
 
         processor.execute(ingestDocument);