Merge remote-tracking branch 'elastic/7.0' into notable-highlights-7.0

* elastic/7.0: (50 commits)
  Fix more broken links in plugins docs
  Fix archives links in plugins docs
  Disable integTest when Docker is not available (elastic#40585)
  Add docs for bundled jdk (elastic#40487)
  [DOCS] Correct keystore commands for Email and Jira actions in Watcher (elastic#40417) (elastic#40613)
  Add usage indicators for the bundled JDK (elastic#40616)
  Add ability to mute and mute flaky fixture (elastic#40630)
  Test fixtures krb5 (elastic#40297)
  Update docs for the DFR similarity (elastic#40579)
  Update ingest jdocs that a null return value will drop the current document. (elastic#40359)
  [DOCS] Document common settings for snapshot repository plugins (elastic#40475) (elastic#40607)
  [DOCS] Fixes formatting in breaking changes
  Handle null retention leases in WaitForNoFollowersStep (elastic#40477)
  Correct ILM metadata minimum compatibility version (elastic#40569)
  Mute SpecificMasterNodesIT.testElectOnlyBetweenMasterNodes()
  Mute testHttpInput
  Include functions' aliases in the list of functions (elastic#40584)
  Optimise rejection of out-of-range `long` values (elastic#40325)
  Add docs for cluster.remote.*.proxy setting (elastic#40281)
  Mute WatchAckTests.testAckAllActions
  ...

.ci/matrix-runtime-javas.yml

@@ -9,6 +9,8 @@ ES_RUNTIME_JAVA:
   - java8
   - java8fips
   - java11
+  - java12
   - openjdk12
   - zulu8
   - zulu11
+  - zulu12

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -31,7 +31,6 @@
 import org.gradle.api.Task;
 import org.gradle.api.plugins.BasePlugin;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
-import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.TaskContainer;
 
 import java.lang.reflect.InvocationTargetException;
@@ -72,6 +71,8 @@ public void apply(Project project) {
                 pullFixture.setEnabled(false);
                 return;
             }
+            preProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
+            postProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
 
             project.apply(spec -> spec.plugin(BasePlugin.class));
             project.apply(spec -> spec.plugin(DockerComposePlugin.class));
@@ -95,20 +96,26 @@ public void apply(Project project) {
                 (name, port) -> postProcessFixture.getExtensions()
                     .getByType(ExtraPropertiesExtension.class).set(name, port)
             );
+            extension.fixtures.add(project);
         }
 
-        extension.fixtures.all(fixtureProject -> project.evaluationDependsOn(fixtureProject.getPath()));
+        extension.fixtures
+            .matching(fixtureProject -> fixtureProject.equals(project) == false)
+            .all(fixtureProject ->  project.evaluationDependsOn(fixtureProject.getPath()));
+
+        conditionTaskByType(tasks, extension, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"));
+        conditionTaskByType(tasks, extension, getTaskClass("org.elasticsearch.gradle.test.RestIntegTestTask"));
+        conditionTaskByType(tasks, extension, TestingConventionsTasks.class);
+        conditionTaskByType(tasks, extension, ComposeUp.class);
+
         if (dockerComposeSupported(project) == false) {
             project.getLogger().warn(
                 "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
                     "but none could be found so these will be skipped", project.getPath()
             );
-            disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"));
-            // conventions are not honored when the tasks are disabled
-            disableTaskByType(tasks, TestingConventionsTasks.class);
-            disableTaskByType(tasks, ComposeUp.class);
             return;
         }
+
         tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
             extension.fixtures.all(fixtureProject -> {
                 fixtureProject.getTasks().matching(it -> it.getName().equals("buildFixture")).all(buildFixture ->
@@ -122,11 +129,22 @@ public void apply(Project project) {
                     fixtureProject,
                     (name, port) -> setSystemProperty(task, name, port)
                 );
+                task.dependsOn(fixtureProject.getTasks().getByName("postProcessFixture"));
             })
         );
 
     }
 
+    private void conditionTaskByType(TaskContainer tasks, TestFixtureExtension extension, Class<? extends DefaultTask> taskClass) {
+        tasks.withType(
+            taskClass,
+            task -> task.onlyIf(spec ->
+                extension.fixtures.stream()
+                    .anyMatch(fixtureProject -> fixtureProject.getTasks().getByName("buildFixture").getEnabled() == false) == false
+            )
+        );
+    }
+
     private void configureServiceInfoForTask(Task task, Project fixtureProject, BiConsumer<String, Integer> consumer) {
         // Configure ports for the tests as system properties.
         // We only know these at execution time so we need to do it in doFirst
@@ -155,7 +173,6 @@ private void configureServiceInfoForTask(Task task, Project fixtureProject, BiCo
         );
     }
 
-    @Input
     public boolean dockerComposeSupported(Project project) {
         if (OS.current().equals(OS.WINDOWS)) {
             return false;

client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreRequest.java

@@ -108,10 +108,26 @@ public GraphExploreRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    /**
+     * The document types to execute the explore against. Defaults to be executed against
+     * all types.
+     *
+     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
+     * filter on a field on the document.
+     */
+    @Deprecated
     public String[] types() {
         return this.types;
     }
 
+    /**
+     * The document types to execute the explore request against. Defaults to be executed against
+     * all types.
+     *
+     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
+     * filter on a field on the document.
+     */
+    @Deprecated
     public GraphExploreRequest types(String... types) {
         this.types = types;
         return this;

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -137,7 +137,7 @@ public void testDelete() throws IOException {
             ElasticsearchException exception = expectThrows(ElasticsearchException.class,
                 () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync));
             assertEquals(RestStatus.CONFLICT, exception.status());
-            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][" + docId + "]: " +
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[" + docId + "]: " +
                 "version conflict, required seqNo [2], primary term [2]. current document has seqNo [3] and primary term [1]]",
                 exception.getMessage());
             assertEquals("index", exception.getMetadata("es.index").get(0));
@@ -166,7 +166,7 @@ public void testDelete() throws IOException {
                 execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
             });
             assertEquals(RestStatus.CONFLICT, exception.status());
-            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][" +
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[" +
                 docId + "]: version conflict, current version [12] is higher or equal to the one provided [10]]", exception.getMessage());
             assertEquals("index", exception.getMetadata("es.index").get(0));
         }
@@ -301,7 +301,7 @@ public void testGet() throws IOException {
             ElasticsearchException exception = expectThrows(ElasticsearchException.class,
                     () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync));
             assertEquals(RestStatus.CONFLICT, exception.status());
-            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, " + "reason=[_doc][id]: " +
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, " + "reason=[id]: " +
                     "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage());
             assertEquals("index", exception.getMetadata("es.index").get(0));
         }
@@ -527,7 +527,7 @@ public void testIndex() throws IOException {
                 execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync);
             });
             assertEquals(RestStatus.CONFLICT, exception.status());
-            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][id]: " +
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[id]: " +
                          "version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]",
                 exception.getMessage());
             assertEquals("index", exception.getMetadata("es.index").get(0));
@@ -574,7 +574,7 @@ public void testIndex() throws IOException {
             });
 
             assertEquals(RestStatus.CONFLICT, exception.status());
-            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][with_create_op_type]: " +
+            assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[with_create_op_type]: " +
                          "version conflict, document already exists (current version [1])]", exception.getMessage());
         }
     }

distribution/archives/build.gradle

@@ -54,10 +54,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
       into('config') {
         dirMode 0750
         fileMode 0660
-        with configFiles(distributionType, oss)
+        with configFiles(distributionType, oss, jdk)
       }
       into('bin') {
-        with binFiles(distributionType, oss)
+        with binFiles(distributionType, oss, jdk)
       }
       if (jdk) {
         into('jdk') {

distribution/build.gradle

@@ -248,8 +248,12 @@ for (String platform : ['linux', 'darwin', 'windows']) {
     }
     it.includeEmptyDirs false
   }
+  String extractDir = "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
   project.task("extract${platform.capitalize()}Jdk", type: Copy) {
-    into "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
+    doFirst {
+      project.delete(extractDir)
+    }
+    into extractDir
     if (extension.equals('zip')) {
       from({ zipTree(jdkConfig.singleFile) }, removeRootDir)
     } else {
@@ -317,13 +321,13 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       from project(':distribution').buildTransportModules
     }
 
-    configFiles = { distributionType, oss ->
+    configFiles = { distributionType, oss, jdk ->
       copySpec {
         with copySpec {
           // main config files, processed with distribution specific substitutions
           from '../src/config'
           exclude 'log4j2.properties' // this is handled separately below
-          MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss))
+          MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
         }
         if (oss) {
           from project(':distribution').buildOssLog4jConfig
@@ -335,23 +339,23 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    binFiles = { distributionType, oss ->
+    binFiles = { distributionType, oss, jdk ->
       copySpec {
         // non-windows files, for all distributions 
         with copySpec {
           from '../src/bin'
           exclude '*.exe'
           exclude '*.bat'
           eachFile { it.setMode(0755) }
-          MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss))
+          MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
         }
         // windows files, only for zip
         if (distributionType == 'zip') {
           with copySpec {
             from '../src/bin'
             include '*.bat'
             filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
-            MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss))
+            MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
           }
           with copySpec {
             from '../src/bin'
@@ -448,7 +452,7 @@ task run(type: RunTask) {
  * </dl>
  */
 subprojects {
-  ext.expansionsForDistribution = { distributionType, oss ->
+  ext.expansionsForDistribution = { distributionType, oss, jdk ->
     final String defaultHeapSize = "1g"
     final String packagingPathData = "path.data: /var/lib/elasticsearch"
     final String pathLogs = "/var/log/elasticsearch"
@@ -541,6 +545,10 @@ subprojects {
         'zip': 'zip'
       ],
 
+      'es.bundled_jdk': [
+        'def': jdk ? 'true' : 'false'
+      ],
+
       'license.name': [
         'deb': oss ? 'ASL-2.0' : 'Elastic-License'
       ],

distribution/bwc/build.gradle

@@ -163,7 +163,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
                         'JAVA_HOME',
                         getJavaHome(it, Integer.parseInt(
                                 lines
-                                        .findAll({ it.startsWith("ES_BUILD_JAVA=")})
+                                        .findAll({ it.startsWith("ES_BUILD_JAVA=") })
                                         .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                                         .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
                                         .join("!!")

distribution/docker/src/docker/Dockerfile

@@ -41,9 +41,10 @@ FROM centos:7
 
 ENV ELASTIC_CONTAINER true
 
-RUN yum update -y && \
+RUN for iter in {1..10}; do yum update -y && \
     yum install -y nc unzip wget which && \
-    yum clean all
+    yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; \
+    (exit $exit_code)
 
 RUN groupadd -g 1000 elasticsearch && \
     adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \

distribution/packages/build.gradle

@@ -61,23 +61,23 @@ buildscript {
   }
 }
 
-void addProcessFilesTask(String type, boolean oss) {
-  String packagingFiles = "build/packaging/${ oss ? 'oss-' : ''}${type}"
+void addProcessFilesTask(String type, boolean oss, boolean jdk) {
+  String packagingFiles = "build/packaging/${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
 
-  String taskName = "process${oss ? 'Oss' : ''}${type.capitalize()}Files"
+  String taskName = "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files"
   task(taskName, type: Copy) {
     into packagingFiles
 
     with copySpec {
       from 'src/common'
       from "src/${type}"
-      MavenFilteringHack.filter(it, expansionsForDistribution(type, oss))
+      MavenFilteringHack.filter(it, expansionsForDistribution(type, oss, jdk))
     }
 
     into('etc/elasticsearch') {
-      with configFiles(type, oss)
+      with configFiles(type, oss, jdk)
     }
-    MavenFilteringHack.filter(it, expansionsForDistribution(type, oss))
+    MavenFilteringHack.filter(it, expansionsForDistribution(type, oss, jdk))
 
     doLast {
       // create empty dirs, we set the permissions when configuring the packages
@@ -91,17 +91,21 @@ void addProcessFilesTask(String type, boolean oss) {
     }
   }
 }
-addProcessFilesTask('deb', true)
-addProcessFilesTask('deb', false)
-addProcessFilesTask('rpm', true)
-addProcessFilesTask('rpm', false)
+addProcessFilesTask('deb', true, true)
+addProcessFilesTask('deb', true, false)
+addProcessFilesTask('deb', false, true)
+addProcessFilesTask('deb', false, false)
+addProcessFilesTask('rpm', true, true)
+addProcessFilesTask('rpm', true, false)
+addProcessFilesTask('rpm', false, true)
+addProcessFilesTask('rpm', false, false)
 
 // Common configuration that is package dependent. This can't go in ospackage
 // since we have different templated files that need to be consumed, but the structure
 // is the same
 Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
   return {
-    dependsOn "process${oss ? 'Oss' : ''}${type.capitalize()}Files"
+    dependsOn "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files"
     packageName "elasticsearch${oss ? '-oss' : ''}"
     arch (type == 'deb' ? 'amd64' : 'X86_64')
     // Follow elasticsearch's file naming convention
@@ -110,7 +114,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
 
     String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
     destinationDir = file("${prefix}/build/distributions")
-    String packagingFiles = "build/packaging/${oss ? 'oss-' : ''}${type}"
+    String packagingFiles = "build/packaging/${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
 
     String scripts = "${packagingFiles}/scripts"
     preInstall file("${scripts}/preinst")
@@ -125,7 +129,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
     // specify it again explicitly for copying common files
     into('/usr/share/elasticsearch') {
       into('bin') {
-        with binFiles(type, oss)
+        with binFiles(type, oss, jdk)
       }
       from(rootProject.projectDir) {
         include 'README.textile'
@@ -202,7 +206,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       createDirectoryEntry true
       fileType CONFIG | NOREPLACE
     }
-    String envFile = expansionsForDistribution(type, false)['path.env']
+    String envFile = expansionsForDistribution(type, oss, jdk)['path.env']
     configurationFile envFile
     into(new File(envFile).getParent()) {
       fileType CONFIG | NOREPLACE

distribution/src/bin/elasticsearch

@@ -29,6 +29,7 @@ if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null;
     -Des.path.conf="$ES_PATH_CONF" \
     -Des.distribution.flavor="$ES_DISTRIBUTION_FLAVOR" \
     -Des.distribution.type="$ES_DISTRIBUTION_TYPE" \
+    -Des.bundled_jdk="$ES_BUNDLED_JDK" \
     -cp "$ES_CLASSPATH" \
     org.elasticsearch.bootstrap.Elasticsearch \
     "$@"
@@ -40,6 +41,7 @@ else
     -Des.path.conf="$ES_PATH_CONF" \
     -Des.distribution.flavor="$ES_DISTRIBUTION_FLAVOR" \
     -Des.distribution.type="$ES_DISTRIBUTION_TYPE" \
+    -Des.bundled_jdk="$ES_BUNDLED_JDK" \
     -cp "$ES_CLASSPATH" \
     org.elasticsearch.bootstrap.Elasticsearch \
     "$@" \

distribution/src/bin/elasticsearch-env

@@ -82,6 +82,7 @@ ES_PATH_CONF=`cd "$ES_PATH_CONF"; pwd`
 
 ES_DISTRIBUTION_FLAVOR=${es.distribution.flavor}
 ES_DISTRIBUTION_TYPE=${es.distribution.type}
+ES_BUNDLED_JDK=${es.bundled_jdk}
 
 if [ -z "$ES_TMPDIR" ]; then
   ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`

distribution/src/bin/elasticsearch-env.bat

@@ -56,6 +56,7 @@ for %%I in ("%ES_PATH_CONF%..") do set ES_PATH_CONF=%%~dpfI
 
 set ES_DISTRIBUTION_FLAVOR=${es.distribution.flavor}
 set ES_DISTRIBUTION_TYPE=${es.distribution.type}
+set ES_BUNDLED_JDK=${es.bundled_jdk}
 
 if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory""`) do set ES_TMPDIR=%%a

distribution/src/bin/elasticsearch-service.bat

@@ -164,7 +164,7 @@ if "%JVM_SS%" == "" (
   goto:eof
 )
 
-set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.path.conf="%ES_PATH_CONF%";-Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%";-Des.distribution.type="%ES_DISTRIBUTION_TYPE%"
+set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.path.conf="%ES_PATH_CONF%";-Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%";-Des.distribution.type="%ES_DISTRIBUTION_TYPE%";-Des.bundled_jdk="%ES_BUNDLED_JDK%"
 
 if "%ES_START_TYPE%" == "" set ES_START_TYPE=manual
 if "%ES_STOP_TIMEOUT%" == "" set ES_STOP_TIMEOUT=0