Test clusters ( was clusterformation ) - part 3

.ci/java-versions.properties

@@ -5,4 +5,5 @@
 # are 'java' or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA=java10
-ES_RUNTIME_JAVA=java8
+# FIXME: revert once CI is updated
+ES_RUNTIME_JAVA=java10

build.gradle

@@ -37,7 +37,7 @@ if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
 }
 
 // common maven publishing configuration
-subprojects {
+allprojects {
   group = 'org.elasticsearch'
   version = VersionProperties.elasticsearch.toString()
   description = "Elasticsearch subproject ${project.path}"
@@ -52,6 +52,8 @@ if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
 }
 String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE.txt"
 
+BuildPlugin.configureRepositories(project)
+
 subprojects {
   // Default to the apache license
   project.ext.licenseName = 'The Apache Software License, Version 2.0'
@@ -199,7 +201,7 @@ task branchConsistency {
   dependsOn verifyVersions, verifyBwcTestsEnabled
 }
 
-subprojects {
+allprojects {
   // ignore missing javadocs
   tasks.withType(Javadoc) { Javadoc javadoc ->
     // the -quiet here is because of a bug in gradle, in that adding a string option
@@ -370,7 +372,7 @@ gradle.projectsEvaluated {
         return
       }
       dependencies.all { Dependency dep ->
-        Project upstreamProject = dependencyToProject(dep)
+        Project upstreamProject = project.ext.dependencyToProject(dep)
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

buildSrc/build.gradle

@@ -16,8 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 import java.nio.file.Files
-import org.gradle.util.GradleVersion
 
 plugins {
   id 'java-gradle-plugin'
@@ -31,8 +31,10 @@ if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion)) {
     throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
 }
 
-if (JavaVersion.current() < JavaVersion.VERSION_1_8) {
-  throw new GradleException('Java 1.8 is required to build elasticsearch gradle tools')
+// TODO this will not work in CI as of this writing
+// take advantage of the new Process API in 9
+if (JavaVersion.current() < JavaVersion.VERSION_1_9) {
+  throw new GradleException('Java 1.9 is required to build elasticsearch gradle tools')
 }
 
 if (project == rootProject) {
@@ -130,6 +132,7 @@ if (project == rootProject) {
     }
     mavenCentral()
   }
+
   test {
     include "**/*Tests.class"
     exclude "**/*IT.class"
@@ -182,6 +185,10 @@ if (project != rootProject) {
   }
   check.dependsOn(integTest)
 
+  // TODO needs to be updated in CI, needs update in NamingConventions
+  targetCompatibility = "1.9"
+  sourceCompatibility = "1.9"
+
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -4,25 +4,22 @@ import com.carrotsearch.ant.tasks.junit4.ListenersList
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
 import groovy.xml.NamespaceBuilder
 import groovy.xml.NamespaceBuilderSupport
-import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
-import org.apache.tools.ant.Project
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
-import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
 import org.gradle.api.specs.Spec
 import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
+import org.gradle.tooling.BuildException
 import org.gradle.util.ConfigureUtil
 
 import javax.inject.Inject
@@ -264,7 +261,11 @@ class RandomizedTestingTask extends DefaultTask {
                         throw new InvalidUserDataException('Seed should be ' +
                             'set on the project instead of a system property')
                     }
-                    sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    if (prop.getValue() instanceof Closure) {
+                        sysproperty key: prop.getKey(), value: (prop.getValue() as Closure).call().toString()
+                    } else {
+                        sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    }
                 }
                 systemProperty 'tests.seed', project.testSeed
                 for (Map.Entry<String, Object> envvar : environmentVariables) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -18,12 +18,18 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.Distribution
 import org.elasticsearch.gradle.NoticeTask
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.testclusters.ElasticsearchNode
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Project
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
@@ -42,12 +48,24 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
+        project.pluginManager.apply(TestClustersPlugin.class)
+        NamedDomainObjectContainer<ElasticsearchNode> nodeExtension =
+                TestClustersPlugin.getNodeExtension(project)
+        ElasticsearchNode esNode = nodeExtension.create("${project.getPath()}##node") {
+            // TODO: change this to INTEG_TEST_ZIP when we add support to install plugins
+            it.distribution = Distribution.ZIP
+            it.version = VersionProperties.elasticsearch
+        }
+
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
+        createIntegTestTask(project, esNode)
+        createBundleTask(project)
+        project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
+        project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
         project.afterEvaluate {
-            boolean isXPackModule = project.path.startsWith(':x-pack:plugin')
-            boolean isModule = project.path.startsWith(':modules:') || isXPackModule
+            boolean isModule = project.path.startsWith(':modules:') || project.path.startsWith(':x-pack:plugin')
             String name = project.pluginProperties.extension.name
             project.archivesBaseName = name
 
@@ -56,20 +74,17 @@ public class PluginBuildPlugin extends BuildPlugin {
 
             configurePublishing(project)
 
-            project.integTestCluster.dependsOn(project.bundlePlugin)
             project.tasks.run.dependsOn(project.bundlePlugin)
             if (isModule) {
-                project.integTestCluster.module(project)
                 project.tasks.run.clusterConfig.module(project)
                 project.tasks.run.clusterConfig.distribution = System.getProperty(
                         'run.distribution', 'integ-test-zip'
                 )
             } else {
-                project.integTestCluster.plugin(project.path)
                 project.tasks.run.clusterConfig.plugin(project.path)
             }
 
-            if (isModule == false || isXPackModule) {
+            if (isModule == false || project.path.startsWith(':x-pack:plugin')) {
                 addNoticeGeneration(project)
             }
 
@@ -78,10 +93,6 @@ public class PluginBuildPlugin extends BuildPlugin {
                 skipIntegTestInDisguise = true
             }
         }
-        createIntegTestTask(project)
-        createBundleTask(project)
-        project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
-        project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
     private void configurePublishing(Project project) {
@@ -125,10 +136,33 @@ public class PluginBuildPlugin extends BuildPlugin {
     }
 
     /** Adds an integTest task which runs rest tests */
-    private static void createIntegTestTask(Project project) {
-        RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+    private static void createIntegTestTask(Project project, ElasticsearchNode esNode) {
+        final RestIntegTestTask integTest
+        if (System.hasProperty('enableTestClusters')) {
+            // emulate external cluster
+            System.properties.put("tests.rest.cluster", "future")
+            System.properties.put("tests.cluster", "future")
+            integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+            RandomizedTestingTask runner = project.tasks.getByName('integTestRunner')
+            runner.configure {
+                useCluster esNode
+                systemProperty('tests.rest.cluster', {-> esNode.getHttpSocketURI()})
+                systemProperty('tests.config.dir', {-> esNode.getCopyOfConfDir()})
+                systemProperty('tests.cluster', {-> esNode.getTransportPortURI()})
+            }
+        } else {
+            integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+            project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
+            project.afterEvaluate {
+                project.integTestCluster.dependsOn(project.bundlePlugin)
+                if (project.path.startsWith(':modules:') || project.path.startsWith(':x-pack:plugin')) {
+                    project.integTestCluster.module(project)
+                } else {
+                    project.integTestCluster.plugin(project.path)
+                }
+            }
+        }
         integTest.mustRunAfter(project.precommit, project.test)
-        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
         project.check.dependsOn(integTest)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -65,16 +65,10 @@ class ClusterFormationTasks {
          * snapshots survive failures / test runs and there is no simple way
          * today to fix that. */
         if (config.cleanShared) {
-          Task cleanup = project.tasks.create(
-            name: "${prefix}#prepareCluster.cleanShared",
-            type: Delete,
-            dependsOn: startDependencies) {
-              delete sharedDir
-              doLast {
-                  sharedDir.mkdirs()
-              }
-          }
-          startDependencies = cleanup
+            runner.doFirst {
+                project.delete(sharedDir)
+                sharedDir.mkdirs()
+            }
         }
         List<Task> startTasks = []
         List<NodeInfo> nodes = []

buildSrc/src/main/java/org/elasticsearch/GradleServicesAdapter.java

@@ -21,28 +21,23 @@
 import org.gradle.api.Action;
 import org.gradle.api.Project;
 import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.WorkResult;
-import org.gradle.process.ExecResult;
-import org.gradle.process.JavaExecSpec;
 
 import java.io.File;
 
-/**
- * Facilitate access to Gradle services without a direct dependency on Project.
- *
- * In a future release Gradle will offer service injection, this adapter plays that role until that time.
+/*
+ * In a future release Gradle will offer service injection, this is a temporary measure until that that time.
  * It exposes the service methods that are part of the public API as the classes implementing them are not.
  * Today service injection is <a href="https://github.com/gradle/gradle/issues/2363">not available</a> for
- * extensions.
+ * plugins.
  *
- * Everything exposed here must be thread safe. That is the very reason why project is not passed in directly.
  */
 public class GradleServicesAdapter {
+    private final Project project;
 
-    public final Project project;
-
-    public GradleServicesAdapter(Project project) {
+    private GradleServicesAdapter(Project project) {
         this.project = project;
     }
 
@@ -58,11 +53,12 @@ public WorkResult sync(Action<? super CopySpec> action) {
         return project.sync(action);
     }
 
-    public ExecResult javaexec(Action<? super JavaExecSpec> action) {
-        return project.javaexec(action);
-    }
 
     public FileTree zipTree(File zipPath) {
         return project.zipTree(zipPath);
     }
+
+    public FileCollection fileTree(File dir) {
+        return project.fileTree(dir);
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java

@@ -20,17 +20,23 @@
 
 public enum Distribution {
 
-    INTEG_TEST("integ-test-zip"),
-    ZIP("zip"),
-    ZIP_OSS("zip-oss");
+    INTEG_TEST_ZIP("integ-test", "zip"),
+    ZIP("elasticsearch", "zip"),
+    ZIP_OSS("elasticsearch-oss", "zip");
 
-    private final String name;
+    private final String fileName;
+    private final String extension;
 
-    Distribution(String name) {
-        this.name = name;
+    Distribution(String fileName, String extension) {
+        this.fileName = fileName;
+        this.extension = extension;
     }
 
-    public String getName() {
-        return name;
+    public String getFileName() {
+        return fileName;
+    }
+
+    public String getExtension() {
+        return extension;
     }
 }