Merge remote-tracking branch 'elastic/6.x' into ccr-6.x

* elastic/6.x:
  Tests: Fix XPack upgrade tests (#32352)
  Remove invalid entry from 6.3.2 release notes
  Number of utilities for writing gradle integration tests (#32282)
  Determine the minimum gradle version based on the wrapper (#32226)
  Enable FIPS JVM in CI (#32330)
  Build: Fix jarHell error I caused by last backport
  Build: Shadow x-pack:protocol into x-pack:plugin:core (#32240)

.ci/matrix-runtime-javas.yml

@@ -7,4 +7,5 @@
 
 ES_RUNTIME_JAVA:
   - java8
+  - java8fips
   - java10

CONTRIBUTING.md

@@ -180,7 +180,7 @@ If your changes affect only the documentation, run:
 ```sh
 ./gradlew -p docs check
 ```
-For more information about testing code examples in the documentation, see 
+For more information about testing code examples in the documentation, see
 https://github.com/elastic/elasticsearch/blob/master/docs/README.asciidoc
 
 ### Project layout
@@ -271,6 +271,39 @@ the `qa` subdirectory functions just like the top level `qa` subdirectory. The
 Elasticsearch process. The `transport-client` subdirectory contains extensions
 to Elasticsearch's standard transport client to work properly with x-pack.
 
+### Gradle Build
+
+We use Gradle to build Elasticsearch because it is flexible enough to not only
+build and package Elasticsearch, but also orchestrate all of the ways that we
+have to test Elasticsearch.
+
+#### Configurations
+
+Gradle organizes dependencies and build artifacts into "configurations" and
+allows you to use these configurations arbitrarilly. Here are some of the most
+common configurations in our build and how we use them:
+
+<dl>
+<dt>`compile`</dt><dd>Code that is on the classpath at both compile and
+runtime. If the [`shadow`][shadow-plugin] plugin is applied to the project then
+this code is bundled into the jar produced by the project.</dd>
+<dt>`runtime`</dt><dd>Code that is not on the classpath at compile time but is
+on the classpath at runtime. We mostly use this configuration to make sure that
+we do not accidentally compile against dependencies of our dependencies also
+known as "transitive" dependencies".</dd>
+<dt>`compileOnly`</dt><dd>Code that is on the classpath at comile time but that
+should not be shipped with the project because it is "provided" by the runtime
+somehow. Elasticsearch plugins use this configuration to include dependencies
+that are bundled with Elasticsearch's server.</dd>
+<dt>`shadow`</dt><dd>Only available in projects with the shadow plugin. Code
+that is on the classpath at both compile and runtime but it *not* bundled into
+the jar produced by the project. If you depend on a project with the `shadow`
+plugin then you need to depend on this configuration because it will bring
+along all of the dependencies you need at runtime.</dd>
+<dt>`testCompile`</dt><dd>Code that is on the classpath for compiling tests
+that are part of this project but not production code. The canonical example
+of this is `junit`.</dd>
+</dl>
 
 Contributing as part of a class
 -------------------------------
@@ -300,3 +333,5 @@ especially when they are unlikely to become long time contributors.
 Finally, we require that you run `./gradlew check` before submitting a
 non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
+
+[shadow-plugin]: https://github.com/johnrengelman/shadow

build.gradle

@@ -16,21 +16,17 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
-
-import org.gradle.api.tasks.wrapper.Wrapper
-import org.gradle.api.tasks.wrapper.Wrapper.DistributionType
 import org.gradle.util.GradleVersion
 import org.gradle.util.DistributionLocator
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.apache.tools.ant.filters.ReplaceTokens
 
 import java.nio.file.Files
 import java.nio.file.Path
@@ -510,6 +506,31 @@ allprojects {
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
+allprojects {
+  /*
+   * IntelliJ and Eclipse don't know about the shadow plugin so when we're
+   * in "IntelliJ mode" or "Eclipse mode" add "runtime" dependencies
+   * eveywhere where we see a "shadow" dependency which will cause them to
+   * reference shadowed projects directly rather than rely on the shadowing
+   * to include them. This is the correct thing for it to do because it
+   * doesn't run the jar shadowing at all. This isn't needed for the project
+   * itself because the IDE configuration is done by SourceSets but it is
+   * *is* needed for projects that depends on the project doing the shadowing.
+   * Without this they won't properly depend on the shadowed project.
+   */
+  if (isEclipse || isIdea) {
+	configurations.all { Configuration configuration ->
+	  dependencies.all { Dependency dep ->
+		if (dep instanceof ProjectDependency) {
+		  if (dep.getTargetConfiguration() == 'shadow') {
+			configuration.dependencies.add(project.dependencies.project(path: dep.dependencyProject.path, configuration: 'runtime'))
+		  }
+		}
+	  }
+	}
+  }
+}
+
 // we need to add the same --debug-jvm option as
 // the real RunTask has, so we can pass it through
 class Run extends DefaultTask {
@@ -531,7 +552,7 @@ task run(type: Run) {
 }
 
 wrapper {
-    distributionType = DistributionType.ALL
+    distributionType = 'ALL'
     doLast {
         final DistributionLocator locator = new DistributionLocator()
         final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
@@ -540,6 +561,10 @@ wrapper {
         final String sha256Sum = new String(sha256Uri.toURL().bytes)
         wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
         println "Added checksum to wrapper properties"
+        // Update build-tools to reflect the Gradle upgrade
+        // TODO: we can remove this once we have tests to make sure older versions work.
+        project(':build-tools').file('src/main/resources/minimumGradleVersion').text = gradleVersion
+        println "Updated minimum Gradle Version"
     }
 }
 

buildSrc/build.gradle

@@ -25,8 +25,9 @@ plugins {
 
 group = 'org.elasticsearch.gradle'
 
-if (GradleVersion.current() < GradleVersion.version('4.9')) {
-  throw new GradleException('Gradle 4.9+ is required to build elasticsearch')
+String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()
+if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion)) {
+    throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
 }
 
 if (JavaVersion.current() < JavaVersion.VERSION_1_8) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import org.apache.commons.io.IOUtils
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.eclipse.jgit.lib.Constants
 import org.eclipse.jgit.lib.RepositoryBuilder
@@ -53,6 +54,7 @@ import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
 import org.gradle.util.GradleVersion
 
+import java.nio.charset.StandardCharsets
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
 /**
@@ -67,8 +69,13 @@ class BuildPlugin implements Plugin<Project> {
                 + 'elasticearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
-        if (GradleVersion.current() < GradleVersion.version('4.9')) {
-            throw new GradleException('Gradle 4.9+ is required to use elasticsearch.build plugin')
+        final String minimumGradleVersion
+        InputStream is = getClass().getResourceAsStream("/minimumGradleVersion")
+        try { minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString()) } finally { is.close() }
+        if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion.trim())) {
+            throw new GradleException(
+                    "Gradle ${minimumGradleVersion}+ is required to use elasticsearch.build plugin"
+            )
         }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
@@ -153,14 +160,6 @@ class BuildPlugin implements Plugin<Project> {
             }
             println "  Random Testing Seed   : ${project.testSeed}"
 
-            // enforce Gradle version
-            final GradleVersion currentGradleVersion = GradleVersion.current();
-
-            final GradleVersion minGradle = GradleVersion.version('4.3')
-            if (currentGradleVersion < minGradle) {
-                throw new GradleException("${minGradle} or above is required to build Elasticsearch")
-            }
-
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
                 final String message =
@@ -391,6 +390,9 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
         project.configurations.compileOnly.dependencies.all(disableTransitiveDeps)
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.configurations.shadow.dependencies.all(disableTransitiveDeps)
+        }
     }
 
     /** Adds repositories used by ES dependencies */
@@ -883,11 +885,20 @@ class BuildPlugin implements Plugin<Project> {
         project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
             it.group.startsWith('org.elasticsearch') == false
         } - project.configurations.compileOnly
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.dependencyLicenses.dependencies += project.configurations.shadow.fileCollection {
+                it.group.startsWith('org.elasticsearch') == false
+            }
+        }
     }
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
         deps.runtimeConfiguration = project.configurations.runtime
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            deps.runtimeConfiguration = project.configurations.create('infoDeps')
+            deps.runtimeConfiguration.extendsFrom(project.configurations.runtime, project.configurations.shadow)
+        }
         deps.compileOnlyConfiguration = project.configurations.compileOnly
         project.afterEvaluate {
             deps.mappings = project.dependencyLicenses.mappings

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -48,18 +48,6 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
-        project.plugins.withType(ShadowPlugin).whenPluginAdded {
-            /*
-             * We've not tested these plugins together and we're fairly sure
-             * they aren't going to work properly as is *and* we're not really
-             * sure *why* you'd want to shade stuff in plugins. So we throw an
-             * exception here to make you come and read this comment. If you
-             * have a need for shadow while building plugins then know that you
-             * are probably going to have to fight with gradle for a while....
-             */
-            throw new InvalidUserDataException('elasticsearch.esplugin is not '
-                    + 'compatible with com.github.johnrengelman.shadow');
-        }
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
@@ -151,8 +139,13 @@ public class PluginBuildPlugin extends BuildPlugin {
                 include(buildProperties.descriptorOutput.name)
             }
             from pluginMetadata // metadata (eg custom security policy)
-            from project.jar // this plugin's jar
-            from project.configurations.runtime - project.configurations.compileOnly // the dep jars
+            /*
+             * If the plugin is using the shadow plugin then we need to bundle
+             * "shadow" things rather than the default jar and dependencies so
+             * we don't hit jar hell.
+             */
+            from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar }
+            from { project.plugins.hasPlugin(ShadowPlugin) ? project.configurations.shadow : project.configurations.runtime - project.configurations.compileOnly }
             // extra files for the plugin to go into the zip
             from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
             from('src/main') {

buildSrc/src/main/resources/minimumGradleVersion

@@ -0,0 +1 @@
+4.9

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java

@@ -1,6 +1,11 @@
 package org.elasticsearch.gradle.test;
 
+import org.gradle.testkit.runner.GradleRunner;
+
 import java.io.File;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
 
@@ -13,4 +18,47 @@ protected File getProjectDir(String name) {
         return new File(root, name);
     }
 
+    protected GradleRunner getGradleRunner(String sampleProject) {
+        return GradleRunner.create()
+            .withProjectDir(getProjectDir(sampleProject))
+            .withPluginClasspath();
+    }
+
+    protected File getBuildDir(String name) {
+        return new File(getProjectDir(name), "build");
+    }
+
+    protected void assertOutputContains(String output, String... lines) {
+        for (String line : lines) {
+            assertOutputContains(output, line);
+        }
+        List<Integer> index = Stream.of(lines).map(line -> output.indexOf(line)).collect(Collectors.toList());
+        if (index.equals(index.stream().sorted().collect(Collectors.toList())) == false) {
+            fail("Expected the following lines to appear in this order:\n" +
+                Stream.of(lines).map(line -> "   - `" + line + "`").collect(Collectors.joining("\n")) +
+                "\nBut they did not. Output is:\n\n```" + output + "\n```\n"
+            );
+        }
+    }
+
+    protected void assertOutputContains(String output, String line) {
+        assertTrue(
+            "Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output,
+            output.contains(line)
+        );
+    }
+
+    protected void assertOutputDoesNotContain(String output, String line) {
+        assertFalse(
+            "Expected the following line not to be in output:\n\n" + line + "\n\nOutput is:\n" + output,
+            output.contains(line)
+        );
+    }
+
+    protected void assertOutputDoesNotContain(String output, String... lines) {
+        for (String line : lines) {
+            assertOutputDoesNotContain(line);
+        }
+    }
+
 }

docs/reference/release-notes/6.3.asciidoc

@@ -40,9 +40,6 @@ SQL::
 * Allow long literals {pull}31777[#31777] (issue: {issue}31750[#31750])
 * Fix stackoverflow on getObject and timestamp conversion {pull}31735[#31735] (issue: {issue}31734[#31734])
 
-Search::
-* Fix multi level nested sort {pull}32204[#32204] (issues: {issue}31554[#31554], {issue}31776[#31776], {issue}31783[#31783], {issue}32130[#32130])
-
 Security::
 * Detect old trial licenses and mimic behaviour {pull}32209[#32209]
 

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java

@@ -84,7 +84,7 @@ public void testIndexTemplatesCreated() throws Exception {
         List<String> expectedTemplates = new ArrayList<>();
         // Watcher creates its templates as soon as the first watcher node connects
         expectedTemplates.add(".triggered_watches");
-        expectedTemplates.add(".watch-history-8");
+        expectedTemplates.add(".watch-history-9");
         expectedTemplates.add(".watches");
         if (masterIsNewVersion()) {
             // Everything else waits until the master is upgraded to create its templates

x-pack/docs/build.gradle

@@ -30,7 +30,7 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
@@ -265,7 +265,7 @@ setups['farequote_index'] = '''
                   airline:
                      type: keyword
                   doc_count:
-                     type: integer       
+                     type: integer
 '''
 setups['farequote_data'] = setups['farequote_index'] + '''
   - do:
@@ -278,7 +278,7 @@ setups['farequote_data'] = setups['farequote_index'] + '''
             {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5}
             {"index": {"_id":"2"}}
             {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23}
-            {"index": {"_id":"3"}}            
+            {"index": {"_id":"3"}}
             {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42}
 '''
 setups['farequote_job'] = setups['farequote_data'] + '''
@@ -310,7 +310,7 @@ setups['farequote_datafeed'] = setups['farequote_job'] + '''
           "job_id":"farequote",
           "indexes":"farequote"
           }
-'''          
+'''
 setups['server_metrics_index'] = '''
   - do:
         indices.create:

x-pack/license-tools/build.gradle

@@ -1,7 +1,7 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compile project(xpackModule('core'))
+  compile project(path: xpackModule('core'), configuration: 'shadow')
   compile "org.elasticsearch:elasticsearch:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
 }
@@ -17,7 +17,7 @@ task buildZip(type: Zip, dependsOn: jar) {
   into(parentDir + '/lib') {
     from jar
     from configurations.runtime
-  } 
+  }
   into(parentDir + '/bin') {
     from 'bin'
   }