From e23d2071fcb94a1a7b14be259e25aba23a55d683 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 9 Apr 2019 19:09:30 -0700 Subject: [PATCH 01/32] work in progress --- build.gradle | 1 + buildSrc/build.gradle | 35 +-- .../elasticsearch/gradle/JdkJarHellCheck.java | 0 .../org/elasticsearch/gradle/LoggedExec.java | 0 .../org/elasticsearch/gradle/Version.java | 0 .../gradle/VersionProperties.java | 0 .../info/GlobalBuildInfoGeneratingTask.java | 80 ++++++ .../gradle/info/GlobalBuildInfoPlugin.java | 255 ++++++++++++++++++ .../elasticsearch/gradle/info/JavaHome.java | 38 +++ ...elasticsearch.global-build-info.properties | 1 + 10 files changed, 377 insertions(+), 33 deletions(-) rename buildSrc/src/main/{minimumRuntime => java}/org/elasticsearch/gradle/JdkJarHellCheck.java (100%) rename buildSrc/src/main/{minimumRuntime => java}/org/elasticsearch/gradle/LoggedExec.java (100%) rename buildSrc/src/main/{minimumRuntime => java}/org/elasticsearch/gradle/Version.java (100%) rename buildSrc/src/main/{minimumRuntime => java}/org/elasticsearch/gradle/VersionProperties.java (100%) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties diff --git a/build.gradle b/build.gradle index c6ace445e9310..83e8f941eae29 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder plugins { id 'com.gradle.build-scan' version '2.2.1' id 'base' + id 'elasticsearch.global-build-info' } if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) { buildScan { diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 70f50f2c541ee..875708725aa0c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -69,39 +69,9 @@ processResources { if (JavaVersion.current() < JavaVersion.VERSION_11) { throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools') } -// Gradle 4.10 does not support setting this to 11 yet -targetCompatibility = "10" -sourceCompatibility = "10" - -// We have a few classes that need to be compiled for older java versions because these are used to run checks against -// those -sourceSets { - minimumRuntime { - // We only want Java here, but the Groovy doesn't configure javadoc correctly if we don't define this as groovy - groovy { - srcDirs = ['src/main/minimumRuntime'] - } - } -} -compileMinimumRuntimeGroovy { - // We can't use BuildPlugin here, so read from file - String minimumRuntimeVersion = file('src/main/resources/minimumRuntimeVersion').text.trim() - targetCompatibility = minimumRuntimeVersion - sourceCompatibility = minimumRuntimeVersion -} -dependencies { - if (project.ext.has("isEclipse") == false || project.ext.isEclipse == false) { - // eclipse is confused if this is set explicitly - compile sourceSets.minimumRuntime.output - } - minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}" - minimumRuntimeCompile localGroovy() - minimumRuntimeCompile gradleApi() -} -jar { - from sourceSets.minimumRuntime.output -} +targetCompatibility = '11' +sourceCompatibility = '11' /***************************************************************************** * Dependencies used by the entire build * @@ -176,7 +146,6 @@ if (project != rootProject) { dependenciesInfo.enabled = false forbiddenApisMain.enabled = false forbiddenApisTest.enabled = false - forbiddenApisMinimumRuntime.enabled = false jarHell.enabled = false thirdPartyAudit.enabled = false diff --git a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java similarity index 100% rename from buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/JdkJarHellCheck.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java diff --git a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/LoggedExec.java b/buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java similarity index 100% rename from buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/LoggedExec.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java diff --git a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/Version.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java similarity index 100% rename from buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/Version.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/Version.java diff --git a/buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java similarity index 100% rename from buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java new file mode 100644 index 0000000000000..51ff6fef4088b --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java @@ -0,0 +1,80 @@ +package org.elasticsearch.gradle.info; + +import org.gradle.api.DefaultTask; +import org.gradle.api.JavaVersion; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; + +import java.io.File; +import java.util.Map; + +@CacheableTask +public class GlobalBuildInfoGeneratingTask extends DefaultTask { + + private JavaVersion minimumCompilerVersion; + private JavaVersion minimumRuntimeVersion; + private File compilerJavaHome; + private File runtimeJavaHome; + private Map javaVersions; + private File outputFile; + + @Input + public JavaVersion getMinimumCompilerVersion() { + return minimumCompilerVersion; + } + + public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { + this.minimumCompilerVersion = minimumCompilerVersion; + } + + @Input + public JavaVersion getMinimumRuntimeVersion() { + return minimumRuntimeVersion; + } + + public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { + this.minimumRuntimeVersion = minimumRuntimeVersion; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getCompilerJavaHome() { + return compilerJavaHome; + } + + public void setCompilerJavaHome(File compilerJavaHome) { + this.compilerJavaHome = compilerJavaHome; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getRuntimeJavaHome() { + return runtimeJavaHome; + } + + public void setRuntimeJavaHome(File runtimeJavaHome) { + this.runtimeJavaHome = runtimeJavaHome; + } + + @Input + public Map getJavaVersions() { + return javaVersions; + } + + public void setJavaVersions(Map javaVersions) { + this.javaVersions = javaVersions; + } + + @OutputFile + public File getOutputFile() { + return outputFile; + } + + public void setOutputFile(File outputFile) { + this.outputFile = outputFile; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java new file mode 100644 index 0000000000000..8af9df7e51533 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -0,0 +1,255 @@ +package org.elasticsearch.gradle.info; + +import org.apache.tools.ant.taskdefs.condition.Os; +import org.gradle.api.GradleException; +import org.gradle.api.JavaVersion; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.internal.jvm.Jvm; +import org.gradle.process.ExecResult; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class GlobalBuildInfoPlugin implements Plugin { + private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); + + @Override + public void apply(Project project) { + if (project != project.getRootProject()) { + throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); + } + + JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion")); + JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion")); + + File compilerJavaHome = findCompilerJavaHome(); + File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome); + + final Map javaVersions = new HashMap<>(); + for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) { + if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) { + javaVersions.put(version, findJavaHome(Integer.toString(version))); + } + } + + project.getTasks().register("generateGlobalBuildInfo", GlobalBuildInfoGeneratingTask.class, task -> { + task.setMinimumCompilerVersion(minimumCompilerVersion); + task.setMinimumRuntimeVersion(minimumRuntimeVersion); + task.setCompilerJavaHome(compilerJavaHome); + task.setRuntimeJavaHome(runtimeJavaHome); + task.setOutputFile(new File(project.getBuildDir(), "global-build-info")); + }); + + String javaVendor = System.getProperty("java.vendor"); + String gradleJavaVersion = System.getProperty("java.version"); + String gradleJavaVersionDetails = javaVendor + " " + gradleJavaVersion + " [" + System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version") + "]"; + + String compilerJavaVersionDetails = gradleJavaVersionDetails; + JavaVersion compilerJavaVersionEnum = JavaVersion.current(); + String runtimeJavaVersionDetails = gradleJavaVersionDetails; + JavaVersion runtimeJavaVersionEnum = JavaVersion.current(); + + try { + if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { + compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome); + compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome)); + } + + if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { + runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome); + runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + String inFipsJvmScript = "print(java.security.Security.getProviders()[0].name.toLowerCase().contains(\"fips\"));"; + boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript)); + + // Build debugging info + LOGGER.lifecycle("======================================="); + LOGGER.lifecycle("Elasticsearch Build Hamster says Hello!"); + LOGGER.lifecycle(" Gradle Version : " + project.getGradle().getGradleVersion()); + LOGGER.lifecycle(" OS Info : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + " (" + System.getProperty("os.arch") + ")"); + if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) { + LOGGER.lifecycle(" Compiler JDK Version : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")"); + LOGGER.lifecycle(" Compiler java.home : " + compilerJavaHome); + LOGGER.lifecycle(" Runtime JDK Version : " + runtimeJavaVersionEnum + " (" +runtimeJavaVersionDetails + ")"); + LOGGER.lifecycle(" Runtime java.home : " + runtimeJavaHome); + LOGGER.lifecycle(" Gradle JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")"); + LOGGER.lifecycle(" Gradle java.home : " + gradleJavaHome); + } else { + LOGGER.lifecycle(" JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")"); + LOGGER.lifecycle(" JAVA_HOME : " + gradleJavaHome); + } + //LOGGER.lifecycle(" Random Testing Seed : " + project.property("testSeed")); + LOGGER.lifecycle("======================================="); + + // enforce Java version + if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) { + final String message = + "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" + + " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"; + throw new GradleException(message); + } + + if (runtimeJavaVersionEnum.compareTo(minimumRuntimeVersion) < 0) { + final String message = + "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" + + " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"; + throw new GradleException(message); + } + + for (final Map.Entry javaVersionEntry : javaVersions.entrySet()) { + final String javaHome = javaVersionEntry.getValue(); + if (javaHome == null) { + continue; + } + JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, new File(javaHome))); + final JavaVersion expectedJavaVersionEnum; + final int version = javaVersionEntry.getKey(); + if (version < 9) { + expectedJavaVersionEnum = JavaVersion.toVersion("1." + version); + } else { + expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version)); + } + if (javaVersionEnum != expectedJavaVersionEnum) { + final String message = + "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" + + " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"; + throw new GradleException(message); + } + } + + JavaVersion finalCompilerJavaVersionEnum = compilerJavaVersionEnum; + JavaVersion finalRuntimeJavaVersionEnum = runtimeJavaVersionEnum; + project.allprojects(p -> { + ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); + + ext.set("compilerJavaHome", compilerJavaHome); + ext.set("runtimeJavaHome", runtimeJavaHome); + ext.set("compilerJavaVersion", finalCompilerJavaVersionEnum); + ext.set("runtimeJavaVersion", finalRuntimeJavaVersionEnum); + ext.set("isRuntimeJavaHomeSet", compilerJavaHome.equals(runtimeJavaHome) == false); + ext.set("javaVersions", javaVersions); + ext.set("minimumCompilerVersion", minimumCompilerVersion); + ext.set("minimumRuntimeVersion", minimumRuntimeVersion); + ext.set("inFipsJvm", inFipsJvm); + ext.set("gradleJavaVersion", JavaVersion.toVersion(gradleJavaVersion)); + }); + } + + private static File findCompilerJavaHome() { + String compilerJavaHome = System.getenv("JAVA_HOME"); + String compilerJavaProperty = System.getProperty("compiler.java"); + + if (compilerJavaProperty != null) { + compilerJavaHome = findJavaHome(compilerJavaProperty); + } + + // if JAVA_HOME is not set,so we use the JDK that Gradle was run with. + return Objects.requireNonNullElseGet(new File(compilerJavaHome), () -> Jvm.current().getJavaHome()); + } + + private static File findRuntimeJavaHome(final File compilerJavaHome) { + String runtimeJavaProperty = System.getProperty("runtime.java"); + + if (runtimeJavaProperty != null) { + return new File(findJavaHome(runtimeJavaProperty)); + } + + return Objects.requireNonNullElse(new File(System.getenv("RUNTIME_JAVA_HOME")), compilerJavaHome); + } + + private static String findJavaHome(String version) { + String versionedVarName = getJavaHomeEnvVarName(version); + String versionedJavaHome = System.getenv(versionedVarName); + if (versionedJavaHome == null) { + throw new GradleException( + "$versionedVarName must be set to build Elasticsearch. " + + "Note that if the variable was just set you might have to run `./gradlew --stop` for " + + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details." + ); + } + return versionedJavaHome; + } + + private static String getJavaHomeEnvVarName(String version) { + return "JAVA" + version + "_HOME"; + } + + private static String getResourceContents(String resourcePath) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))) { + StringBuffer buffer = new StringBuffer(); + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + if (buffer.length() != 0) { + buffer.append('\n'); + } + buffer.append(line); + } + + return buffer.toString(); + } catch (IOException e) { + throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e); + } + } + + /** Finds printable java version of the given JAVA_HOME */ + private static String findJavaVersionDetails(Project project, File javaHome) { + String versionInfoScript = "print(" + + "java.lang.System.getProperty(\"java.vendor\") + \" \" + java.lang.System.getProperty(\"java.version\") + " + + "\" [\" + java.lang.System.getProperty(\"java.vm.name\") + \" \" + java.lang.System.getProperty(\"java.vm.version\") + \"]\");"; + return runJavaAsScript(project, javaHome, versionInfoScript).trim(); + } + + /** Finds the parsable java specification version */ + private static String findJavaSpecificationVersion(Project project, File javaHome) { + String versionScript = "print(java.lang.System.getProperty(\"java.specification.version\"));"; + return runJavaAsScript(project, javaHome, versionScript); + } + + /** Runs the given javascript using jjs from the jdk, and returns the output */ + private static String runJavaAsScript(Project project, File javaHome, String script) { + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // gradle/groovy does not properly escape the double quote for windows + script = script.replace("\"", "\\\""); + } + File jrunscriptPath = new File(javaHome, "bin/jrunscript"); + String finalScript = script; + ExecResult result = project.exec(spec -> { + spec.setExecutable(jrunscriptPath); + spec.args("-e", finalScript); + spec.setStandardOutput(stdout); + spec.setErrorOutput(stderr); + spec.setIgnoreExitValue(true); + }); + + try { + if (result.getExitValue() != 0) { + LOGGER.error("STDOUT:"); + stdout.toString("UTF-8").lines().forEach(LOGGER::error); + LOGGER.error("STDERR:"); + stderr.toString("UTF-8").lines().forEach(LOGGER::error); + result.rethrowFailure(); + } + return stdout.toString("UTF-8").trim(); + } catch (UnsupportedEncodingException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java new file mode 100644 index 0000000000000..5b9b5eb3508b0 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java @@ -0,0 +1,38 @@ +package org.elasticsearch.gradle.info; + +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; + +import java.io.File; + +public class JavaHome { + private Integer version; + private File javaHome; + + private JavaHome(int version, File javaHome) { + this.version = version; + this.javaHome = javaHome; + } + + public static JavaHome of(int version, File javaHome) { + return new JavaHome(version, javaHome); + } + + @Input + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } + + @InputDirectory + public File getJavaHome() { + return javaHome; + } + + public void setJavaHome(File javaHome) { + this.javaHome = javaHome; + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties new file mode 100644 index 0000000000000..7428707877242 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties @@ -0,0 +1 @@ +implementation-class=org.elasticsearch.gradle.info.GlobalBuildInfoPlugin \ No newline at end of file From fd193d348d07adf603bb4b61b7e5ee01a2cff635 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 9 Apr 2019 19:25:04 -0700 Subject: [PATCH 02/32] wip --- .../elasticsearch/gradle/BuildPlugin.groovy | 121 ------------------ 1 file changed, 121 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3d8ba1f3b36a5..eac95483a6fce 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -112,133 +112,12 @@ class BuildPlugin implements Plugin { configureDependenciesInfo(project) } - - /** Performs checks on the build environment and prints information about the build environment. */ static void globalBuildInfo(Project project) { if (project.rootProject.ext.has('buildChecksDone') == false) { - JavaVersion minimumRuntimeVersion = JavaVersion.toVersion( - BuildPlugin.class.getClassLoader().getResourceAsStream("minimumRuntimeVersion").text.trim() - ) - JavaVersion minimumCompilerVersion = JavaVersion.toVersion( - BuildPlugin.class.getClassLoader().getResourceAsStream("minimumCompilerVersion").text.trim() - ) - String compilerJavaHome = findCompilerJavaHome() - String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome) - File gradleJavaHome = Jvm.current().javaHome - - final Map javaVersions = [:] - for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) { - if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) { - javaVersions.put(version, findJavaHome(version.toString())); - } - } - - String javaVendor = System.getProperty('java.vendor') - String gradleJavaVersion = System.getProperty('java.version') - String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" + - " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]" - - String compilerJavaVersionDetails = gradleJavaVersionDetails - JavaVersion compilerJavaVersionEnum = JavaVersion.current() - if (new File(compilerJavaHome).canonicalPath != gradleJavaHome.canonicalPath) { - compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome) - compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome)) - } - - String runtimeJavaVersionDetails = gradleJavaVersionDetails - JavaVersion runtimeJavaVersionEnum = JavaVersion.current() - if (new File(runtimeJavaHome).canonicalPath != gradleJavaHome.canonicalPath) { - runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome) - runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome)) - } - - String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));' - boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript)) - - // Build debugging info - println '=======================================' - println 'Elasticsearch Build Hamster says Hello!' - println " Gradle Version : ${project.gradle.gradleVersion}" - println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})" - if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) { - println " Compiler JDK Version : ${compilerJavaVersionEnum} (${compilerJavaVersionDetails})" - println " Compiler java.home : ${compilerJavaHome}" - println " Runtime JDK Version : ${runtimeJavaVersionEnum} (${runtimeJavaVersionDetails})" - println " Runtime java.home : ${runtimeJavaHome}" - println " Gradle JDK Version : ${JavaVersion.toVersion(gradleJavaVersion)} (${gradleJavaVersionDetails})" - println " Gradle java.home : ${gradleJavaHome}" - } else { - println " JDK Version : ${JavaVersion.toVersion(gradleJavaVersion)} (${gradleJavaVersionDetails})" - println " JAVA_HOME : ${gradleJavaHome}" - } - println " Random Testing Seed : ${project.testSeed}" - println '=======================================' - - // enforce Java version - if (compilerJavaVersionEnum < minimumCompilerVersion) { - final String message = - "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" + - " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]" - throw new GradleException(message) - } - - if (runtimeJavaVersionEnum < minimumRuntimeVersion) { - final String message = - "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" + - " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]" - throw new GradleException(message) - } - - for (final Map.Entry javaVersionEntry : javaVersions.entrySet()) { - final String javaHome = javaVersionEntry.getValue() - if (javaHome == null) { - continue - } - JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome)) - final JavaVersion expectedJavaVersionEnum - final int version = javaVersionEntry.getKey() - if (version < 9) { - expectedJavaVersionEnum = JavaVersion.toVersion("1." + version) - } else { - expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version)) - } - if (javaVersionEnum != expectedJavaVersionEnum) { - final String message = - "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" + - " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]" - throw new GradleException(message) - } - } - - project.rootProject.ext.compilerJavaHome = compilerJavaHome - project.rootProject.ext.runtimeJavaHome = runtimeJavaHome - project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum - project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum - project.rootProject.ext.isRuntimeJavaHomeSet = compilerJavaHome.equals(runtimeJavaHome) == false - project.rootProject.ext.javaVersions = javaVersions project.rootProject.ext.buildChecksDone = true - project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion - project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion - project.rootProject.ext.inFipsJvm = inFipsJvm - project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion) - project.rootProject.ext.java9Home = "${-> findJavaHome("9")}" project.rootProject.ext.defaultParallel = findDefaultParallel(project.rootProject) } - - project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion - project.sourceCompatibility = project.rootProject.ext.minimumRuntimeVersion - - // set java home for each project, so they dont have to find it in the root project - project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome - project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome - project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion - project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion - project.ext.isRuntimeJavaHomeSet = project.rootProject.ext.isRuntimeJavaHomeSet - project.ext.javaVersions = project.rootProject.ext.javaVersions - project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm - project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion - project.ext.java9Home = project.rootProject.ext.java9Home } static void requireDocker(final Task task) { From 0643838fe2ee47ab62347ae669f8359f5810913d Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 10 Apr 2019 09:14:04 -0700 Subject: [PATCH 03/32] more wip --- .../elasticsearch/gradle/BuildPlugin.groovy | 226 ++++------------- .../gradle/precommit/PrecommitTasks.groovy | 31 +-- .../gradle/test/RestIntegTestTask.groovy | 39 +-- .../test/StandaloneRestTestPlugin.groovy | 1 - .../info/GenerateGlobalBuildInfoTask.java | 237 ++++++++++++++++++ .../info/GlobalBuildInfoGeneratingTask.java | 80 ------ .../gradle/info/GlobalBuildInfoPlugin.java | 212 ++++++---------- .../elasticsearch/gradle/info/JavaHome.java | 5 + .../gradle/info/PrintGlobalBuildInfoTask.java | 39 +++ .../gradle/precommit/ThirdPartyAuditTask.java | 13 +- server/build.gradle | 6 +- 11 files changed, 430 insertions(+), 459 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3d8ba1f3b36a5..0d0c06f8172dd 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -23,6 +23,7 @@ import org.apache.commons.io.IOUtils import org.apache.tools.ant.taskdefs.condition.Os import org.eclipse.jgit.lib.Constants import org.eclipse.jgit.lib.RepositoryBuilder +import org.elasticsearch.gradle.info.JavaHome import org.elasticsearch.gradle.precommit.PrecommitTasks import org.elasticsearch.gradle.test.ErrorReportingTestListener import org.gradle.api.GradleException @@ -54,6 +55,7 @@ import org.gradle.api.tasks.javadoc.Javadoc import org.gradle.api.tasks.testing.Test import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.internal.jvm.Jvm +import org.gradle.process.CommandLineArgumentProvider import org.gradle.process.ExecResult import org.gradle.process.ExecSpec import org.gradle.util.GradleVersion @@ -99,7 +101,6 @@ class BuildPlugin implements Plugin { project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) setupSeed(project) - globalBuildInfo(project) configureRepositories(project) project.ext.versions = VersionProperties.versions configureSourceSets(project) @@ -112,135 +113,6 @@ class BuildPlugin implements Plugin { configureDependenciesInfo(project) } - - - /** Performs checks on the build environment and prints information about the build environment. */ - static void globalBuildInfo(Project project) { - if (project.rootProject.ext.has('buildChecksDone') == false) { - JavaVersion minimumRuntimeVersion = JavaVersion.toVersion( - BuildPlugin.class.getClassLoader().getResourceAsStream("minimumRuntimeVersion").text.trim() - ) - JavaVersion minimumCompilerVersion = JavaVersion.toVersion( - BuildPlugin.class.getClassLoader().getResourceAsStream("minimumCompilerVersion").text.trim() - ) - String compilerJavaHome = findCompilerJavaHome() - String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome) - File gradleJavaHome = Jvm.current().javaHome - - final Map javaVersions = [:] - for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) { - if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) { - javaVersions.put(version, findJavaHome(version.toString())); - } - } - - String javaVendor = System.getProperty('java.vendor') - String gradleJavaVersion = System.getProperty('java.version') - String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" + - " [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]" - - String compilerJavaVersionDetails = gradleJavaVersionDetails - JavaVersion compilerJavaVersionEnum = JavaVersion.current() - if (new File(compilerJavaHome).canonicalPath != gradleJavaHome.canonicalPath) { - compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome) - compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome)) - } - - String runtimeJavaVersionDetails = gradleJavaVersionDetails - JavaVersion runtimeJavaVersionEnum = JavaVersion.current() - if (new File(runtimeJavaHome).canonicalPath != gradleJavaHome.canonicalPath) { - runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome) - runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome)) - } - - String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));' - boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript)) - - // Build debugging info - println '=======================================' - println 'Elasticsearch Build Hamster says Hello!' - println " Gradle Version : ${project.gradle.gradleVersion}" - println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})" - if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) { - println " Compiler JDK Version : ${compilerJavaVersionEnum} (${compilerJavaVersionDetails})" - println " Compiler java.home : ${compilerJavaHome}" - println " Runtime JDK Version : ${runtimeJavaVersionEnum} (${runtimeJavaVersionDetails})" - println " Runtime java.home : ${runtimeJavaHome}" - println " Gradle JDK Version : ${JavaVersion.toVersion(gradleJavaVersion)} (${gradleJavaVersionDetails})" - println " Gradle java.home : ${gradleJavaHome}" - } else { - println " JDK Version : ${JavaVersion.toVersion(gradleJavaVersion)} (${gradleJavaVersionDetails})" - println " JAVA_HOME : ${gradleJavaHome}" - } - println " Random Testing Seed : ${project.testSeed}" - println '=======================================' - - // enforce Java version - if (compilerJavaVersionEnum < minimumCompilerVersion) { - final String message = - "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" + - " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]" - throw new GradleException(message) - } - - if (runtimeJavaVersionEnum < minimumRuntimeVersion) { - final String message = - "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" + - " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]" - throw new GradleException(message) - } - - for (final Map.Entry javaVersionEntry : javaVersions.entrySet()) { - final String javaHome = javaVersionEntry.getValue() - if (javaHome == null) { - continue - } - JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome)) - final JavaVersion expectedJavaVersionEnum - final int version = javaVersionEntry.getKey() - if (version < 9) { - expectedJavaVersionEnum = JavaVersion.toVersion("1." + version) - } else { - expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version)) - } - if (javaVersionEnum != expectedJavaVersionEnum) { - final String message = - "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" + - " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]" - throw new GradleException(message) - } - } - - project.rootProject.ext.compilerJavaHome = compilerJavaHome - project.rootProject.ext.runtimeJavaHome = runtimeJavaHome - project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum - project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum - project.rootProject.ext.isRuntimeJavaHomeSet = compilerJavaHome.equals(runtimeJavaHome) == false - project.rootProject.ext.javaVersions = javaVersions - project.rootProject.ext.buildChecksDone = true - project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion - project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion - project.rootProject.ext.inFipsJvm = inFipsJvm - project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion) - project.rootProject.ext.java9Home = "${-> findJavaHome("9")}" - project.rootProject.ext.defaultParallel = findDefaultParallel(project.rootProject) - } - - project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion - project.sourceCompatibility = project.rootProject.ext.minimumRuntimeVersion - - // set java home for each project, so they dont have to find it in the root project - project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome - project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome - project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion - project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion - project.ext.isRuntimeJavaHomeSet = project.rootProject.ext.isRuntimeJavaHomeSet - project.ext.javaVersions = project.rootProject.ext.javaVersions - project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm - project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion - project.ext.java9Home = project.rootProject.ext.java9Home - } - static void requireDocker(final Task task) { final Project rootProject = task.project.rootProject if (rootProject.hasProperty('requiresDocker') == false) { @@ -420,7 +292,8 @@ class BuildPlugin implements Plugin { /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ static String getJavaHome(final Task task, final int version) { requireJavaHome(task, version) - return task.project.javaVersions.get(version) + def javaVersions = task.project.javaVersions as List + return javaVersions.find { it.version == version } } private static String findRuntimeJavaHome(final String compilerJavaHome) { @@ -728,13 +601,12 @@ class BuildPlugin implements Plugin { project.afterEvaluate { project.tasks.withType(JavaCompile) { final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(it.targetCompatibility) - final compilerJavaHomeFile = new File(project.compilerJavaHome) // we only fork if the Gradle JDK is not the same as the compiler JDK - if (compilerJavaHomeFile.canonicalPath == Jvm.current().javaHome.canonicalPath) { + if (project.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { options.fork = false } else { options.fork = true - options.forkOptions.javaHome = compilerJavaHomeFile + options.forkOptions.javaHome = project.compilerJavaHome } /* * -path because gradle will send in paths that don't always exist. @@ -758,13 +630,12 @@ class BuildPlugin implements Plugin { } // also apply release flag to groovy, which is used in build-tools project.tasks.withType(GroovyCompile) { - final compilerJavaHomeFile = new File(project.compilerJavaHome) // we only fork if the Gradle JDK is not the same as the compiler JDK - if (compilerJavaHomeFile.canonicalPath == Jvm.current().javaHome.canonicalPath) { + if (project.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { options.fork = false } else { options.fork = true - options.forkOptions.javaHome = compilerJavaHomeFile + options.forkOptions.javaHome = project.compilerJavaHome options.compilerArgs << '--release' << JavaVersion.toVersion(it.targetCompatibility).majorVersion } } @@ -914,11 +785,25 @@ class BuildPlugin implements Plugin { project.mkdir(test.workingDir) } - def listener = new ErrorReportingTestListener(test.testLogging, testOutputDir) + ErrorReportingTestListener listener = new ErrorReportingTestListener(test.testLogging, testOutputDir) test.extensions.add(ErrorReportingTestListener, 'errorReportingTestListener', listener) addTestOutputListener(listener) addTestListener(listener) + /* + * We use lazy-evaluated strings in order to configure system properties whose value will not be known until + * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated + * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due + * to the GStrings containing references to non-serializable objects. + * + * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added + * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the + * build cache key or up to date checking. + */ + SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider() + test.jvmArgumentProviders.add(nonInputProperties) + test.ext.nonInputProperties = nonInputProperties + executable = "${project.runtimeJavaHome}/bin/java" workingDir = project.file("${project.buildDir}/testrun/${test.name}") maxParallelForks = project.rootProject.ext.defaultParallel @@ -928,11 +813,9 @@ class BuildPlugin implements Plugin { jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}", "-Xms${System.getProperty('tests.heap.size', '512m')}", '-XX:+HeapDumpOnOutOfMemoryError', - "-XX:HeapDumpPath=$heapdumpDir" + "-XX:HeapDumpPath=$heapdumpDir", + '--illegal-access=warn' - if (project.runtimeJavaVersion >= JavaVersion.VERSION_1_9) { - jvmArgs '--illegal-access=warn' - } if (System.getProperty('tests.jvm.argline')) { jvmArgs System.getProperty('tests.jvm.argline').split(" ") @@ -953,13 +836,14 @@ class BuildPlugin implements Plugin { 'tests.task': path, 'tests.security.manager': 'true', 'tests.seed': project.testSeed, - 'jna.nosys': 'true', - 'compiler.java': project.ext.compilerJavaVersion.getMajorVersion() + 'jna.nosys': 'true' + + nonInputProperties.systemProperty('compiler.java', "${-> project.ext.compilerJavaVersion.getMajorVersion()}") if (project.ext.inFipsJvm) { - systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS" + nonInputProperties.systemProperty('runtime.java', "${-> project.ext.runtimeJavaVersion.getMajorVersion()} FIPS") } else { - systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + nonInputProperties.systemProperty('runtime.java', "${-> project.ext.runtimeJavaVersion.getMajorVersion()}") } // TODO: remove setting logging level via system property systemProperty 'tests.logger.level', 'WARN' @@ -995,41 +879,6 @@ class BuildPlugin implements Plugin { } } - private static int findDefaultParallel(Project project) { - if (project.file("/proc/cpuinfo").exists()) { - // Count physical cores on any Linux distro ( don't count hyper-threading ) - Map socketToCore = [:] - String currentID = "" - project.file("/proc/cpuinfo").readLines().forEach({ line -> - if (line.contains(":")) { - List parts = line.split(":", 2).collect({it.trim()}) - String name = parts[0], value = parts[1] - // the ID of the CPU socket - if (name == "physical id") { - currentID = value - } - // Number of cores not including hyper-threading - if (name == "cpu cores") { - assert currentID.isEmpty() == false - socketToCore[currentID] = Integer.valueOf(value) - currentID = "" - } - } - }) - return socketToCore.values().sum() - } else if ('Mac OS X'.equals(System.getProperty('os.name'))) { - // Ask macOS to count physical CPUs for us - ByteArrayOutputStream stdout = new ByteArrayOutputStream() - project.exec { - executable 'sysctl' - args '-n', 'hw.physicalcpu' - standardOutput = stdout - } - return Integer.parseInt(stdout.toString('UTF-8').trim()) - } - return Runtime.getRuntime().availableProcessors() / 2 - } - private static configurePrecommit(Project project) { Task precommit = PrecommitTasks.create(project, true) project.check.dependsOn(precommit) @@ -1111,4 +960,19 @@ class BuildPlugin implements Plugin { }) } } + + private static class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider { + private final Map systemProperties = [:] + + void systemProperty(String key, Object value) { + systemProperties.put(key, value) + } + + @Override + Iterable asArguments() { + return systemProperties.collect { key, value -> + "-D${key}=${value.toString()}".toString() + } + } + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index 0316acef922c4..0157302c31550 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -115,15 +115,13 @@ class PrecommitTasks { } private static Task configureThirdPartyAudit(Project project) { - ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') - thirdPartyAuditTask.configure { - dependsOn(buildResources) - signatureFile = buildResources.copy("forbidden/third-party-audit.txt") - javaHome = project.runtimeJavaHome - targetCompatibility = project.runtimeJavaVersion + return project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) { task -> + task.dependsOn(buildResources) + task.signatureFile = buildResources.copy("forbidden/third-party-audit.txt") + task.javaHome = project.runtimeJavaHome + task.targetCompatibility.set(project.provider({ project.runtimeJavaVersion })) } - return thirdPartyAuditTask } private static Task configureForbiddenApisCli(Project project) { @@ -131,15 +129,18 @@ class PrecommitTasks { ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources') project.tasks.withType(CheckForbiddenApis) { dependsOn(buildResources) - targetCompatibility = project.runtimeJavaVersion.getMajorVersion() - if (project.runtimeJavaVersion > JavaVersion.VERSION_11) { - doLast { - project.logger.info( - "Forbidden APIs does not support java version past 11. Will use the signatures from 11 for ", - project.runtimeJavaVersion - ) + doFirst { + // we need to defer this configuration since we don't know the runtime java version until execution time + targetCompatibility = project.runtimeJavaVersion.getMajorVersion() + if (project.runtimeJavaVersion > JavaVersion.VERSION_11) { + doLast { + project.logger.info( + "Forbidden APIs does not support java version past 11. Will use the signatures from 11 for ", + project.runtimeJavaVersion + ) + } + targetCompatibility = JavaVersion.VERSION_11.getMajorVersion() } - targetCompatibility = JavaVersion.VERSION_11.getMajorVersion() } bundledSignatures = [ "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index df26a5a07cb21..2698623ba3ae0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -86,51 +86,24 @@ class RestIntegTestTask extends DefaultTask { runner.include('**/*IT.class') runner.systemProperty('tests.rest.load_packaged', 'false') - /* - * We use lazy-evaluated strings in order to configure system properties whose value will not be known until - * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated - * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due - * to the GStrings containing references to non-serializable objects. - * - * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added - * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the - * build cache key or up to date checking. - */ - def nonInputProperties = new CommandLineArgumentProvider() { - private final Map systemProperties = [:] - - void systemProperty(String key, Object value) { - systemProperties.put(key, value) - } - - @Override - Iterable asArguments() { - return systemProperties.collect { key, value -> - "-D${key}=${value.toString()}".toString() - } - } - } - runner.jvmArgumentProviders.add(nonInputProperties) - runner.ext.nonInputProperties = nonInputProperties - if (System.getProperty("tests.rest.cluster") == null) { if (System.getProperty("tests.cluster") != null) { throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null") } if (usesTestclusters == true) { ElasticsearchCluster cluster = project.testClusters."${name}" - nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }") - nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }") - nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }") + runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }") + runner.nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }") + runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }") } else { // we pass all nodes to the rest cluster to allow the clients to round-robin between them // this is more realistic than just talking to a single node - nonInputProperties.systemProperty('tests.rest.cluster', "${-> nodes.collect { it.httpUri() }.join(",")}") - nonInputProperties.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}") + runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> nodes.collect { it.httpUri() }.join(",")}") + runner.nonInputProperties.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}") // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass // both as separate sysprops - nonInputProperties.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}") + runner.nonInputProperties.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}") // dump errors and warnings from cluster log on failure TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 2a858206ebd72..ba9c64cb0e945 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -57,7 +57,6 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(JavaBasePlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) - BuildPlugin.globalBuildInfo(project) BuildPlugin.configureRepositories(project) BuildPlugin.configureTestTasks(project) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java new file mode 100644 index 0000000000000..1c8b3de697756 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -0,0 +1,237 @@ +package org.elasticsearch.gradle.info; + +import org.elasticsearch.gradle.OS; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.JavaVersion; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.Nested; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.TaskAction; +import org.gradle.internal.jvm.Jvm; +import org.gradle.process.ExecResult; + +import javax.inject.Inject; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@CacheableTask +public class GenerateGlobalBuildInfoTask extends DefaultTask { + + private JavaVersion minimumCompilerVersion; + private JavaVersion minimumRuntimeVersion; + private File compilerJavaHome; + private File runtimeJavaHome; + private List javaVersions; + private final RegularFileProperty outputFile; + + @Inject + public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) { + this.outputFile = objectFactory.fileProperty(); + } + + @TaskAction + public void generate() { + String javaVendor = System.getProperty("java.vendor"); + String gradleJavaVersion = System.getProperty("java.version"); + String gradleJavaVersionDetails = javaVendor + " " + gradleJavaVersion + " [" + System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version") + "]"; + + String compilerJavaVersionDetails = gradleJavaVersionDetails; + JavaVersion compilerJavaVersionEnum = JavaVersion.current(); + String runtimeJavaVersionDetails = gradleJavaVersionDetails; + JavaVersion runtimeJavaVersionEnum = JavaVersion.current(); + File gradleJavaHome = Jvm.current().getJavaHome(); + + try { + if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { + compilerJavaVersionDetails = findJavaVersionDetails(compilerJavaHome); + compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(compilerJavaHome)); + } + + if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { + runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome); + runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile.getAsFile().get()))) { + writer.write(" Gradle Version : " + getProject().getGradle().getGradleVersion() + "\n"); + writer.write(" OS Info : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + " (" + System.getProperty("os.arch") + ")\n"); + if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) { + writer.write(" Compiler JDK Version : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")\n"); + writer.write(" Compiler java.home : " + compilerJavaHome + "\n"); + writer.write(" Runtime JDK Version : " + runtimeJavaVersionEnum + " (" + runtimeJavaVersionDetails + ")\n"); + writer.write(" Runtime java.home : " + runtimeJavaHome + "\n"); + writer.write(" Gradle JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")\n"); + writer.write(" Gradle java.home : " + gradleJavaHome); + } else { + writer.write(" JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")\n"); + writer.write(" JAVA_HOME : " + gradleJavaHome); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // enforce Java version + if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) { + String message = "The compiler java.home must be set to a JDK installation directory for Java " + minimumCompilerVersion + + " but is [" + compilerJavaHome + "] corresponding to [" + compilerJavaVersionEnum +"]"; + throw new GradleException(message); + } + + if (runtimeJavaVersionEnum.compareTo(minimumRuntimeVersion) < 0) { + String message = "The runtime java.home must be set to a JDK installation directory for Java " + minimumRuntimeVersion + + " but is [" + runtimeJavaHome + "] corresponding to [" + runtimeJavaVersionEnum + "]"; + throw new GradleException(message); + } + + for (JavaHome javaVersion : javaVersions) { + File javaHome = javaVersion.getJavaHome(); + if (javaHome == null) { + continue; + } + JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(javaHome)); + JavaVersion expectedJavaVersionEnum; + int version = javaVersion.getVersion(); + if (version < 9) { + expectedJavaVersionEnum = JavaVersion.toVersion("1." + version); + } else { + expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version)); + } + if (javaVersionEnum != expectedJavaVersionEnum) { + String message = "The environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java " + + expectedJavaVersionEnum + " but is [" + javaHome + "] corresponding to [" + javaVersionEnum + "]"; + throw new GradleException(message); + } + } + + JavaVersion finalCompilerJavaVersionEnum = compilerJavaVersionEnum; + JavaVersion finalRuntimeJavaVersionEnum = runtimeJavaVersionEnum; + + getProject().getRootProject().allprojects(p -> { + ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); + ext.set("compilerJavaVersion", finalCompilerJavaVersionEnum); + ext.set("runtimeJavaVersion", finalRuntimeJavaVersionEnum); + }); + } + + /** + * Finds printable java version of the given JAVA_HOME + */ + private String findJavaVersionDetails(File javaHome) { + String versionInfoScript = "print(" + + "java.lang.System.getProperty(\"java.vendor\") + \" \" + java.lang.System.getProperty(\"java.version\") + " + + "\" [\" + java.lang.System.getProperty(\"java.vm.name\") + \" \" + java.lang.System.getProperty(\"java.vm.version\") + \"]\");"; + return runJavaAsScript(javaHome, versionInfoScript).trim(); + } + + /** + * Finds the parsable java specification version + */ + private String findJavaSpecificationVersion(File javaHome) { + String versionScript = "print(java.lang.System.getProperty(\"java.specification.version\"));"; + return runJavaAsScript(javaHome, versionScript); + } + + + /** + * Runs the given javascript using jjs from the jdk, and returns the output + */ + private String runJavaAsScript(File javaHome, String script) { + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + if (OS.current() == OS.WINDOWS) { + // gradle/groovy does not properly escape the double quote for windows + script = script.replace("\"", "\\\""); + } + File jrunscriptPath = new File(javaHome, "bin/jrunscript"); + String finalScript = script; + ExecResult result = getProject().exec(spec -> { + spec.setExecutable(jrunscriptPath); + spec.args("-e", finalScript); + spec.setStandardOutput(stdout); + spec.setErrorOutput(stderr); + spec.setIgnoreExitValue(true); + }); + + if (result.getExitValue() != 0) { + getLogger().error("STDOUT:"); + stdout.toString(UTF_8).lines().forEach(getLogger()::error); + getLogger().error("STDERR:"); + stderr.toString(UTF_8).lines().forEach(getLogger()::error); + result.rethrowFailure(); + } + return stdout.toString(UTF_8).trim(); + } + + @Input + public JavaVersion getMinimumCompilerVersion() { + return minimumCompilerVersion; + } + + public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { + this.minimumCompilerVersion = minimumCompilerVersion; + } + + @Input + public JavaVersion getMinimumRuntimeVersion() { + return minimumRuntimeVersion; + } + + public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { + this.minimumRuntimeVersion = minimumRuntimeVersion; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getCompilerJavaHome() { + return compilerJavaHome; + } + + public void setCompilerJavaHome(File compilerJavaHome) { + this.compilerJavaHome = compilerJavaHome; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getRuntimeJavaHome() { + return runtimeJavaHome; + } + + public void setRuntimeJavaHome(File runtimeJavaHome) { + this.runtimeJavaHome = runtimeJavaHome; + } + + @Nested + public List getJavaVersions() { + return javaVersions; + } + + public void setJavaVersions(List javaVersions) { + this.javaVersions = javaVersions; + } + + @OutputFile + public RegularFileProperty getOutputFile() { + return outputFile; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java deleted file mode 100644 index 51ff6fef4088b..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoGeneratingTask.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.elasticsearch.gradle.info; - -import org.gradle.api.DefaultTask; -import org.gradle.api.JavaVersion; -import org.gradle.api.tasks.CacheableTask; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputDirectory; -import org.gradle.api.tasks.OutputFile; -import org.gradle.api.tasks.PathSensitive; -import org.gradle.api.tasks.PathSensitivity; - -import java.io.File; -import java.util.Map; - -@CacheableTask -public class GlobalBuildInfoGeneratingTask extends DefaultTask { - - private JavaVersion minimumCompilerVersion; - private JavaVersion minimumRuntimeVersion; - private File compilerJavaHome; - private File runtimeJavaHome; - private Map javaVersions; - private File outputFile; - - @Input - public JavaVersion getMinimumCompilerVersion() { - return minimumCompilerVersion; - } - - public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { - this.minimumCompilerVersion = minimumCompilerVersion; - } - - @Input - public JavaVersion getMinimumRuntimeVersion() { - return minimumRuntimeVersion; - } - - public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { - this.minimumRuntimeVersion = minimumRuntimeVersion; - } - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - public File getCompilerJavaHome() { - return compilerJavaHome; - } - - public void setCompilerJavaHome(File compilerJavaHome) { - this.compilerJavaHome = compilerJavaHome; - } - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - public File getRuntimeJavaHome() { - return runtimeJavaHome; - } - - public void setRuntimeJavaHome(File runtimeJavaHome) { - this.runtimeJavaHome = runtimeJavaHome; - } - - @Input - public Map getJavaVersions() { - return javaVersions; - } - - public void setJavaVersions(Map javaVersions) { - this.javaVersions = javaVersions; - } - - @OutputFile - public File getOutputFile() { - return outputFile; - } - - public void setOutputFile(File outputFile) { - this.outputFile = outputFile; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 8af9df7e51533..92d0ad952dd02 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -1,30 +1,29 @@ package org.elasticsearch.gradle.info; -import org.apache.tools.ant.taskdefs.condition.Os; +import org.elasticsearch.gradle.OS; import org.gradle.api.GradleException; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.logging.Logger; -import org.gradle.api.logging.Logging; import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.internal.jvm.Jvm; -import org.gradle.process.ExecResult; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.UncheckedIOException; -import java.io.UnsupportedEncodingException; -import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.stream.Collectors; public class GlobalBuildInfoPlugin implements Plugin { - private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); + private static Integer _defaultParallel = null; @Override public void apply(Project project) { @@ -38,117 +37,46 @@ public void apply(Project project) { File compilerJavaHome = findCompilerJavaHome(); File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome); - final Map javaVersions = new HashMap<>(); + final List javaVersions = new ArrayList<>(); for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) { if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) { - javaVersions.put(version, findJavaHome(Integer.toString(version))); + javaVersions.add(JavaHome.of(version, new File(findJavaHome(Integer.toString(version))))); } } - project.getTasks().register("generateGlobalBuildInfo", GlobalBuildInfoGeneratingTask.class, task -> { + GenerateGlobalBuildInfoTask generateTask = project.getTasks().create("generateGlobalBuildInfo", GenerateGlobalBuildInfoTask.class, task -> { + task.setJavaVersions(javaVersions); task.setMinimumCompilerVersion(minimumCompilerVersion); task.setMinimumRuntimeVersion(minimumRuntimeVersion); task.setCompilerJavaHome(compilerJavaHome); task.setRuntimeJavaHome(runtimeJavaHome); - task.setOutputFile(new File(project.getBuildDir(), "global-build-info")); + task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info")); }); - String javaVendor = System.getProperty("java.vendor"); - String gradleJavaVersion = System.getProperty("java.version"); - String gradleJavaVersionDetails = javaVendor + " " + gradleJavaVersion + " [" + System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version") + "]"; - - String compilerJavaVersionDetails = gradleJavaVersionDetails; - JavaVersion compilerJavaVersionEnum = JavaVersion.current(); - String runtimeJavaVersionDetails = gradleJavaVersionDetails; - JavaVersion runtimeJavaVersionEnum = JavaVersion.current(); - - try { - if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { - compilerJavaVersionDetails = findJavaVersionDetails(project, compilerJavaHome); - compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, compilerJavaHome)); - } - - if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { - runtimeJavaVersionDetails = findJavaVersionDetails(project, runtimeJavaHome); - runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome)); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> { + task.getBuildInfoFile().set(generateTask.getOutputFile()); + }); - String inFipsJvmScript = "print(java.security.Security.getProviders()[0].name.toLowerCase().contains(\"fips\"));"; - boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript)); - - // Build debugging info - LOGGER.lifecycle("======================================="); - LOGGER.lifecycle("Elasticsearch Build Hamster says Hello!"); - LOGGER.lifecycle(" Gradle Version : " + project.getGradle().getGradleVersion()); - LOGGER.lifecycle(" OS Info : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + " (" + System.getProperty("os.arch") + ")"); - if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) { - LOGGER.lifecycle(" Compiler JDK Version : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")"); - LOGGER.lifecycle(" Compiler java.home : " + compilerJavaHome); - LOGGER.lifecycle(" Runtime JDK Version : " + runtimeJavaVersionEnum + " (" +runtimeJavaVersionDetails + ")"); - LOGGER.lifecycle(" Runtime java.home : " + runtimeJavaHome); - LOGGER.lifecycle(" Gradle JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")"); - LOGGER.lifecycle(" Gradle java.home : " + gradleJavaHome); - } else { - LOGGER.lifecycle(" JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")"); - LOGGER.lifecycle(" JAVA_HOME : " + gradleJavaHome); - } - //LOGGER.lifecycle(" Random Testing Seed : " + project.property("testSeed")); - LOGGER.lifecycle("======================================="); - - // enforce Java version - if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) { - final String message = - "the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" + - " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"; - throw new GradleException(message); - } + project.getExtensions().getByType(ExtraPropertiesExtension.class).set("defaultParallel", findDefaultParallel(project)); - if (runtimeJavaVersionEnum.compareTo(minimumRuntimeVersion) < 0) { - final String message = - "the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" + - " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"; - throw new GradleException(message); - } - - for (final Map.Entry javaVersionEntry : javaVersions.entrySet()) { - final String javaHome = javaVersionEntry.getValue(); - if (javaHome == null) { - continue; - } - JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, new File(javaHome))); - final JavaVersion expectedJavaVersionEnum; - final int version = javaVersionEntry.getKey(); - if (version < 9) { - expectedJavaVersionEnum = JavaVersion.toVersion("1." + version); - } else { - expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version)); - } - if (javaVersionEnum != expectedJavaVersionEnum) { - final String message = - "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" + - " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"; - throw new GradleException(message); - } - } - - JavaVersion finalCompilerJavaVersionEnum = compilerJavaVersionEnum; - JavaVersion finalRuntimeJavaVersionEnum = runtimeJavaVersionEnum; project.allprojects(p -> { + // Make sure than any task execution generates and prints build info + p.getTasks().all(task -> { + if (task != generateTask && task != printTask) { + task.dependsOn(printTask); + } + }); + ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); ext.set("compilerJavaHome", compilerJavaHome); ext.set("runtimeJavaHome", runtimeJavaHome); - ext.set("compilerJavaVersion", finalCompilerJavaVersionEnum); - ext.set("runtimeJavaVersion", finalRuntimeJavaVersionEnum); ext.set("isRuntimeJavaHomeSet", compilerJavaHome.equals(runtimeJavaHome) == false); ext.set("javaVersions", javaVersions); ext.set("minimumCompilerVersion", minimumCompilerVersion); ext.set("minimumRuntimeVersion", minimumRuntimeVersion); - ext.set("inFipsJvm", inFipsJvm); - ext.set("gradleJavaVersion", JavaVersion.toVersion(gradleJavaVersion)); + ext.set("inFipsJvm", true); + ext.set("gradleJavaVersion", Jvm.current().getJavaVersion()); }); } @@ -161,7 +89,7 @@ private static File findCompilerJavaHome() { } // if JAVA_HOME is not set,so we use the JDK that Gradle was run with. - return Objects.requireNonNullElseGet(new File(compilerJavaHome), () -> Jvm.current().getJavaHome()); + return compilerJavaHome == null ? Jvm.current().getJavaHome() : new File(compilerJavaHome); } private static File findRuntimeJavaHome(final File compilerJavaHome) { @@ -171,7 +99,7 @@ private static File findRuntimeJavaHome(final File compilerJavaHome) { return new File(findJavaHome(runtimeJavaProperty)); } - return Objects.requireNonNullElse(new File(System.getenv("RUNTIME_JAVA_HOME")), compilerJavaHome); + return System.getenv("RUNTIME_JAVA_HOME") == null ? compilerJavaHome : new File(System.getenv("RUNTIME_JAVA_HOME")); } private static String findJavaHome(String version) { @@ -207,49 +135,53 @@ private static String getResourceContents(String resourcePath) { } } - /** Finds printable java version of the given JAVA_HOME */ - private static String findJavaVersionDetails(Project project, File javaHome) { - String versionInfoScript = "print(" + - "java.lang.System.getProperty(\"java.vendor\") + \" \" + java.lang.System.getProperty(\"java.version\") + " + - "\" [\" + java.lang.System.getProperty(\"java.vm.name\") + \" \" + java.lang.System.getProperty(\"java.vm.version\") + \"]\");"; - return runJavaAsScript(project, javaHome, versionInfoScript).trim(); - } - - /** Finds the parsable java specification version */ - private static String findJavaSpecificationVersion(Project project, File javaHome) { - String versionScript = "print(java.lang.System.getProperty(\"java.specification.version\"));"; - return runJavaAsScript(project, javaHome, versionScript); - } + private static int findDefaultParallel(Project project) { + // Since it costs IO to compute this, and is done at configuration time we want to cache this if possible + // It's safe to store this in a static variable since it's just a primitive so leaking memory isn't an issue + if (_defaultParallel == null) { + File cpuInfoFile = new File("/proc/cpuinfo"); + if (cpuInfoFile.exists()) { + // Count physical cores on any Linux distro ( don't count hyper-threading ) + Map socketToCore = new HashMap<>(); + String currentID = ""; + + try (BufferedReader reader = new BufferedReader(new FileReader(cpuInfoFile))) { + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + if (line.contains(":")) { + List parts = Arrays.stream(line.split(":", 2)).map(String::trim).collect(Collectors.toList()); + String name = parts.get(0); + String value = parts.get(1); + // the ID of the CPU socket + if (name.equals("physical id")) { + currentID = value; + } + // Number of cores not including hyper-threading + if (name.equals("cpu cores")) { + assert currentID.isEmpty() == false; + socketToCore.put("currentID", Integer.valueOf(value)); + currentID = ""; + } + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + _defaultParallel = socketToCore.values().stream().mapToInt(i -> i).sum(); + } else if (OS.current() == OS.MAC) { + // Ask macOS to count physical CPUs for us + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + project.exec(spec -> { + spec.setExecutable("sysctl"); + spec.args("-n", "hw.physicalcpu"); + spec.setStandardOutput(stdout); + }); + + _defaultParallel = Integer.parseInt(stdout.toString().trim()); + } - /** Runs the given javascript using jjs from the jdk, and returns the output */ - private static String runJavaAsScript(Project project, File javaHome, String script) { - ByteArrayOutputStream stdout = new ByteArrayOutputStream(); - ByteArrayOutputStream stderr = new ByteArrayOutputStream(); - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // gradle/groovy does not properly escape the double quote for windows - script = script.replace("\"", "\\\""); + _defaultParallel = Runtime.getRuntime().availableProcessors() / 2; } - File jrunscriptPath = new File(javaHome, "bin/jrunscript"); - String finalScript = script; - ExecResult result = project.exec(spec -> { - spec.setExecutable(jrunscriptPath); - spec.args("-e", finalScript); - spec.setStandardOutput(stdout); - spec.setErrorOutput(stderr); - spec.setIgnoreExitValue(true); - }); - try { - if (result.getExitValue() != 0) { - LOGGER.error("STDOUT:"); - stdout.toString("UTF-8").lines().forEach(LOGGER::error); - LOGGER.error("STDERR:"); - stderr.toString("UTF-8").lines().forEach(LOGGER::error); - result.rethrowFailure(); - } - return stdout.toString("UTF-8").trim(); - } catch (UnsupportedEncodingException e) { - throw new UncheckedIOException(e); - } + return _defaultParallel; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java index 5b9b5eb3508b0..086860a25fcf6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java @@ -2,6 +2,9 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import java.io.File; @@ -28,6 +31,8 @@ public void setVersion(Integer version) { } @InputDirectory + @Optional + @PathSensitive(PathSensitivity.RELATIVE) public File getJavaHome() { return javaHome; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java new file mode 100644 index 0000000000000..7e458724a3a16 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -0,0 +1,39 @@ +package org.elasticsearch.gradle.info; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.resources.TextResource; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import javax.inject.Inject; + +public class PrintGlobalBuildInfoTask extends DefaultTask { + + private RegularFileProperty buildInfoFile; + + @Inject + public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) { + this.buildInfoFile = objectFactory.fileProperty(); + } + + @TaskAction + public void print() { + getLogger().quiet("======================================="); + getLogger().quiet("Elasticsearch Build Hamster says Hello!"); + getLogger().quiet(getBuildInfoText().asString()); + getLogger().quiet(" Random Testing Seed : " + getProject().property("testSeed")); + getLogger().quiet("======================================="); + } + + @InputFile + public RegularFileProperty getBuildInfoFile() { + return buildInfoFile; + } + + private TextResource getBuildInfoText() { + return getProject().getResources().getText().fromFile(buildInfoFile.getAsFile().get()); + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 8ec979420c0e4..563ea0761ddde 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.io.output.NullOutputStream; +import org.apache.tools.ant.taskdefs.Java; import org.elasticsearch.gradle.JdkJarHellCheck; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; @@ -26,6 +27,8 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.file.FileTree; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; import org.gradle.api.specs.Spec; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Input; @@ -76,17 +79,13 @@ public class ThirdPartyAuditTask extends DefaultTask { private String javaHome; - private JavaVersion targetCompatibility; + private final Property targetCompatibility = getProject().getObjects().property(JavaVersion.class); @Input - public JavaVersion getTargetCompatibility() { + public Property getTargetCompatibility() { return targetCompatibility; } - public void setTargetCompatibility(JavaVersion targetCompatibility) { - this.targetCompatibility = targetCompatibility; - } - @InputFiles @PathSensitive(PathSensitivity.NAME_ONLY) public Configuration getForbiddenAPIsConfiguration() { @@ -276,7 +275,7 @@ private void extractJars(Set jars) { // pther version specific implementation of said classes. IntStream.rangeClosed( Integer.parseInt(JavaVersion.VERSION_1_9.getMajorVersion()), - Integer.parseInt(targetCompatibility.getMajorVersion()) + Integer.parseInt(targetCompatibility.get().getMajorVersion()) ).forEach(majorVersion -> getProject().copy(spec -> { spec.from(getProject().zipTree(jar)); spec.into(jarExpandDir); diff --git a/server/build.gradle b/server/build.gradle index f1c17f8dbe00c..8c84f90607ebb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -56,8 +56,10 @@ if (!isEclipse && !isIdea) { } forbiddenApisJava12 { - if (project.runtimeJavaVersion < JavaVersion.VERSION_12) { - targetCompatibility = JavaVersion.VERSION_12.getMajorVersion() + doFirst { + if (project.runtimeJavaVersion < JavaVersion.VERSION_12) { + targetCompatibility = JavaVersion.VERSION_12.getMajorVersion() + } } } From 5512fff81277ccdbcb904b4f96b09feff2911b11 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 17 Apr 2019 09:07:42 -0700 Subject: [PATCH 04/32] tweaks --- .../elasticsearch/gradle/BuildPlugin.groovy | 20 ++++++++++++------- .../gradle/plugin/PluginBuildPlugin.groovy | 5 +++-- .../info/GenerateGlobalBuildInfoTask.java | 1 - .../gradle/info/GlobalBuildInfoPlugin.java | 10 +++++----- .../gradle/info/PrintGlobalBuildInfoTask.java | 1 - 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 0d0c06f8172dd..ebde61b84acb1 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -19,6 +19,7 @@ package org.elasticsearch.gradle import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin +import groovy.transform.CompileStatic import org.apache.commons.io.IOUtils import org.apache.tools.ant.taskdefs.condition.Os import org.eclipse.jgit.lib.Constants @@ -43,6 +44,7 @@ import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.credentials.HttpHeaderCredentials import org.gradle.api.execution.TaskActionListener import org.gradle.api.execution.TaskExecutionGraph +import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaPlugin import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.publish.maven.plugins.MavenPublishPlugin @@ -68,6 +70,7 @@ import java.util.regex.Matcher /** * Encapsulates build configuration for elasticsearch projects. */ +@CompileStatic class BuildPlugin implements Plugin { @Override @@ -102,7 +105,7 @@ class BuildPlugin implements Plugin { setupSeed(project) configureRepositories(project) - project.ext.versions = VersionProperties.versions + project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions) configureSourceSets(project) configureCompile(project) configureJavadoc(project) @@ -115,6 +118,8 @@ class BuildPlugin implements Plugin { static void requireDocker(final Task task) { final Project rootProject = task.project.rootProject + ExtraPropertiesExtension ext = rootProject.extensions.getByType(ExtraPropertiesExtension) + if (rootProject.hasProperty('requiresDocker') == false) { /* * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks @@ -142,11 +147,11 @@ class BuildPlugin implements Plugin { throw new IllegalArgumentException( "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]") } - rootProject.rootProject.ext.buildDocker = buildDocker - rootProject.rootProject.ext.requiresDocker = [] + + ext.set('buildDocker', buildDocker) + ext.set('requiresDocker', []) rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> - final List tasks = - ((List)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { " ${it.path}".toString()} + final List tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List).collect { " ${it.path}".toString()} if (tasks.isEmpty() == false) { /* * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not @@ -199,8 +204,9 @@ class BuildPlugin implements Plugin { } } } - if (rootProject.buildDocker) { - rootProject.requiresDocker.add(task) + + if (ext.get('buildDocker')) { + (ext.get('requiresDocker') as List).add(task) } else { task.enabled = false } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 9d303fcb78dc0..7d4e1ecc9a426 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -28,6 +28,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.test.RunTask import org.elasticsearch.gradle.testclusters.TestClustersPlugin import org.gradle.api.InvalidUserDataException +import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.publish.maven.MavenPublication @@ -43,13 +44,13 @@ import java.util.regex.Pattern /** * Encapsulates build configuration for an Elasticsearch plugin. */ -class PluginBuildPlugin extends BuildPlugin { +class PluginBuildPlugin implements Plugin { public static final String PLUGIN_EXTENSION_NAME = 'esplugin' @Override void apply(Project project) { - super.apply(project) + project.pluginManager.apply(BuildPlugin) PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 1c8b3de697756..dfcecca01f73e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -34,7 +34,6 @@ @CacheableTask public class GenerateGlobalBuildInfoTask extends DefaultTask { - private JavaVersion minimumCompilerVersion; private JavaVersion minimumRuntimeVersion; private File compilerJavaHome; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 92d0ad952dd02..6342a0b57bbeb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -121,15 +121,15 @@ private static String getJavaHomeEnvVarName(String version) { private static String getResourceContents(String resourcePath) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))) { - StringBuffer buffer = new StringBuffer(); + StringBuilder b = new StringBuilder(); for (String line = reader.readLine(); line != null; line = reader.readLine()) { - if (buffer.length() != 0) { - buffer.append('\n'); + if (b.length() != 0) { + b.append('\n'); } - buffer.append(line); + b.append(line); } - return buffer.toString(); + return b.toString(); } catch (IOException e) { throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index 7e458724a3a16..240f0f136db46 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -10,7 +10,6 @@ import javax.inject.Inject; public class PrintGlobalBuildInfoTask extends DefaultTask { - private RegularFileProperty buildInfoFile; @Inject From fdbbb38c1520d42ea569c683176d935639931e4b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 14:06:44 -0700 Subject: [PATCH 05/32] Convert BuildPlugin to static Groovy --- build.gradle | 2 +- .../elasticsearch/gradle/BuildPlugin.groovy | 427 ++++++++---------- 2 files changed, 198 insertions(+), 231 deletions(-) diff --git a/build.gradle b/build.gradle index 83e8f941eae29..4bad308ff9d1b 100644 --- a/build.gradle +++ b/build.gradle @@ -264,7 +264,7 @@ allprojects { } project.afterEvaluate { - configurations.all { + configurations.matching { it.canBeResolved }.all { resolutionStrategy.dependencySubstitution { DependencySubstitutions subs -> projectSubstitutions.each { k,v -> subs.substitute(subs.module(k)).with(subs.project(v)) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index ebde61b84acb1..829b01c1af550 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -19,14 +19,18 @@ package org.elasticsearch.gradle import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import org.apache.commons.io.IOUtils import org.apache.tools.ant.taskdefs.condition.Os import org.eclipse.jgit.lib.Constants import org.eclipse.jgit.lib.RepositoryBuilder import org.elasticsearch.gradle.info.JavaHome +import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.elasticsearch.gradle.precommit.PrecommitTasks import org.elasticsearch.gradle.test.ErrorReportingTestListener +import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.InvalidUserDataException import org.gradle.api.JavaVersion @@ -41,22 +45,33 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.artifacts.ResolvedArtifact import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.artifacts.repositories.AuthenticationContainer +import org.gradle.api.artifacts.repositories.IvyArtifactRepository +import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout +import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.credentials.HttpHeaderCredentials import org.gradle.api.execution.TaskActionListener import org.gradle.api.execution.TaskExecutionGraph +import org.gradle.api.file.CopySpec +import org.gradle.api.plugins.BasePlugin +import org.gradle.api.plugins.BasePluginConvention import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaPlugin +import org.gradle.api.publish.PublishingExtension import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.publish.maven.plugins.MavenPublishPlugin import org.gradle.api.publish.maven.tasks.GenerateMavenPom import org.gradle.api.tasks.SourceSet +import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.compile.GroovyCompile import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.javadoc.Javadoc import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.logging.TestLoggingContainer import org.gradle.authentication.http.HttpHeaderAuthentication +import org.gradle.external.javadoc.CoreJavadocOptions import org.gradle.internal.jvm.Jvm +import org.gradle.language.base.plugins.LifecycleBasePlugin import org.gradle.process.CommandLineArgumentProvider import org.gradle.process.ExecResult import org.gradle.process.ExecSpec @@ -234,107 +249,51 @@ class BuildPlugin implements Plugin { + "or by passing -Dbuild.docker=false") } - private static String findCompilerJavaHome() { - String compilerJavaHome = System.getenv('JAVA_HOME') - final String compilerJavaProperty = System.getProperty('compiler.java') - if (compilerJavaProperty != null) { - compilerJavaHome = findJavaHome(compilerJavaProperty) - } - if (compilerJavaHome == null) { - // if JAVA_HOME does not set,so we use the JDK that Gradle was run with. - return Jvm.current().javaHome - } - return compilerJavaHome - } - - private static String findJavaHome(String version) { - String versionedVarName = getJavaHomeEnvVarName(version) - String versionedJavaHome = System.getenv(versionedVarName); - if (versionedJavaHome == null) { - throw new GradleException( - "$versionedVarName must be set to build Elasticsearch. " + - "Note that if the variable was just set you might have to run `./gradlew --stop` for " + - "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details." - ) - } - return versionedJavaHome - } - - private static String getJavaHomeEnvVarName(String version) { - return 'JAVA' + version + '_HOME' - } - /** Add a check before gradle execution phase which ensures java home for the given java version is set. */ static void requireJavaHome(Task task, int version) { - Project rootProject = task.project.rootProject // use root project for global accounting + // use root project for global accounting + Project rootProject = task.project.rootProject + ExtraPropertiesExtension ext = rootProject.extensions.getByType(ExtraPropertiesExtension) + if (rootProject.hasProperty('requiredJavaVersions') == false) { - rootProject.rootProject.ext.requiredJavaVersions = [:] - rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> + ext.set('requiredJavaVersions', [:]) + rootProject.gradle.taskGraph.whenReady({ TaskExecutionGraph taskGraph -> List messages = [] - for (entry in rootProject.requiredJavaVersions) { - if (rootProject.javaVersions.get(entry.key) != null) { + Map> requiredJavaVersions = (Map>) ext.get('requiredJavaVersions') + for (Map.Entry> entry : requiredJavaVersions) { + List javaVersions = ext.get('javaVersions') as List + if (javaVersions.find { it.version == entry.key } != null) { continue } - List tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { " ${it.path}" } + List tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { " ${it.path}".toString() } if (tasks.isEmpty() == false) { - messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}") + messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}".toString()) } } if (messages.isEmpty() == false) { throw new GradleException(messages.join('\n')) } - rootProject.rootProject.ext.requiredJavaVersions = null // reset to null to indicate the pre-execution checks have executed - } - } else if (rootProject.rootProject.requiredJavaVersions == null) { + ext.set('requiredJavaVersions', null) // reset to null to indicate the pre-execution checks have executed + }) + } else if (ext.has('requiredJavaVersions') == false || ext.get('requiredJavaVersions') == null) { // check directly if the version is present since we are already executing - if (rootProject.javaVersions.get(version) == null) { + List javaVersions = ext.get('javaVersions') as List + if (javaVersions.get(version) == null) { throw new GradleException("JAVA${version}_HOME required to run task:\n${task}") } } else { - rootProject.requiredJavaVersions.getOrDefault(version, []).add(task) + Map> requiredJavaVersions = (Map>) ext.get('requiredJavaVersions') + requiredJavaVersions.getOrDefault(version, []).add(task) } } /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ static String getJavaHome(final Task task, final int version) { requireJavaHome(task, version) - def javaVersions = task.project.javaVersions as List + def javaVersions = task.project.property('javaVersions') as List return javaVersions.find { it.version == version } } - private static String findRuntimeJavaHome(final String compilerJavaHome) { - String runtimeJavaProperty = System.getProperty("runtime.java") - if (runtimeJavaProperty != null) { - return findJavaHome(runtimeJavaProperty) - } - return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome - } - - /** Finds printable java version of the given JAVA_HOME */ - private static String findJavaVersionDetails(Project project, String javaHome) { - String versionInfoScript = 'print(' + - 'java.lang.System.getProperty("java.vendor") + " " + java.lang.System.getProperty("java.version") + ' + - '" [" + java.lang.System.getProperty("java.vm.name") + " " + java.lang.System.getProperty("java.vm.version") + "]");' - return runJavaAsScript(project, javaHome, versionInfoScript).trim() - } - - /** Finds the parsable java specification version */ - private static String findJavaSpecificationVersion(Project project, String javaHome) { - String versionScript = 'print(java.lang.System.getProperty("java.specification.version"));' - return runJavaAsScript(project, javaHome, versionScript) - } - - private static String findJavaVendor(Project project, String javaHome) { - String vendorScript = 'print(java.lang.System.getProperty("java.vendor"));' - return runJavaAsScript(project, javaHome, vendorScript) - } - - /** Finds the parsable java specification version */ - private static String findJavaVersion(Project project, String javaHome) { - String versionScript = 'print(java.lang.System.getProperty("java.version"));' - return runJavaAsScript(project, javaHome, versionScript) - } - /** Runs the given javascript using jjs from the jdk, and returns the output */ private static String runJavaAsScript(Project project, String javaHome, String script) { ByteArrayOutputStream stdout = new ByteArrayOutputStream() @@ -344,12 +303,12 @@ class BuildPlugin implements Plugin { script = script.replace('"', '\\"') } File jrunscriptPath = new File(javaHome, 'bin/jrunscript') - ExecResult result = project.exec { - executable = jrunscriptPath - args '-e', script - standardOutput = stdout - errorOutput = stderr - ignoreExitValue = true + ExecResult result = project.exec { ExecSpec spec -> + spec.executable = jrunscriptPath + spec.args '-e', script + spec.standardOutput = stdout + spec.errorOutput = stderr + spec.ignoreExitValue = true } if (result.exitValue != 0) { project.logger.error("STDOUT:") @@ -384,7 +343,7 @@ class BuildPlugin implements Plugin { */ static void configureConfigurations(Project project) { // we want to test compileOnly deps! - project.configurations.testCompile.extendsFrom(project.configurations.compileOnly) + project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)) // we are not shipping these jars, we act like dumb consumers of these things if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') { @@ -422,9 +381,9 @@ class BuildPlugin implements Plugin { } } - project.configurations.compile.dependencies.all(disableTransitiveDeps) - project.configurations.testCompile.dependencies.all(disableTransitiveDeps) - project.configurations.compileOnly.dependencies.all(disableTransitiveDeps) + project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) + project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) + project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) project.plugins.withType(ShadowPlugin).whenPluginAdded { Configuration bundle = project.configurations.create('bundle') @@ -435,38 +394,39 @@ class BuildPlugin implements Plugin { /** Adds repositories used by ES dependencies */ static void configureRepositories(Project project) { RepositoryHandler repos = project.repositories - if (System.getProperty("repos.mavenLocal") != null) { + if (System.getProperty('repos.mavenLocal') != null) { // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is // useful for development ie. bwc tests where we install stuff in the local repository // such that we don't have to pass hardcoded files to gradle repos.mavenLocal() } repos.jcenter() - repos.ivy { - url "https://artifacts.elastic.co/downloads" - patternLayout { - artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]" + repos.ivy { IvyArtifactRepository repo -> + repo.url = 'https://artifacts.elastic.co/downloads' + repo.patternLayout { IvyPatternRepositoryLayout layout -> + layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]' } // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats - credentials(HttpHeaderCredentials) { - name = "X-Elastic-No-KPI" - value = "1" - } - authentication { - header(HttpHeaderAuthentication) + repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials credentials -> + credentials.name = 'X-Elastic-No-KPI' + credentials.value = '1' + } as Action) + repo.authentication { AuthenticationContainer auth -> + auth.create('header', HttpHeaderAuthentication) } } - repos.maven { - name "elastic" - url "https://artifacts.elastic.co/maven" + repos.maven { MavenArtifactRepository repo -> + repo.name = 'elastic' + repo.url = 'https://artifacts.elastic.co/maven' } String luceneVersion = VersionProperties.lucene if (luceneVersion.contains('-snapshot')) { // extract the revision number from the version with a regex matcher - String revision = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] - repos.maven { - name 'lucene-snapshots' - url "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + List matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List + String revision = matches.get(1) + repos.maven { MavenArtifactRepository repo -> + repo.name = 'lucene-snapshots' + repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" } } } @@ -482,6 +442,7 @@ class BuildPlugin implements Plugin { *
  • Set compile time deps back to compile from runtime (known issue with maven-publish plugin)
    • * */ + @CompileDynamic private static Closure fixupDependencies(Project project) { return { XmlProvider xml -> // first find if we have dependencies at all, and grab the node @@ -542,21 +503,22 @@ class BuildPlugin implements Plugin { } /**Configuration generation of maven poms. */ - public static void configurePomGeneration(Project project) { + static void configurePomGeneration(Project project) { // Only works with `enableFeaturePreview('STABLE_PUBLISHING')` // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185 project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask -> // The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it, // just make a copy. - generatePOMTask.ext.pomFileName = null - doLast { - project.copy { - from generatePOMTask.destination - into "${project.buildDir}/distributions" - rename { - generatePOMTask.ext.pomFileName == null ? - "${project.archivesBaseName}-${project.version}.pom" : - generatePOMTask.ext.pomFileName + ExtraPropertiesExtension ext = generatePOMTask.extensions.getByType(ExtraPropertiesExtension) + ext.set('pomFileName', null) + generatePOMTask.doLast { + project.copy { CopySpec spec -> + spec.from generatePOMTask.destination + spec.into "${project.buildDir}/distributions" + spec.rename { + ext.has('pomFileName') && ext.get('pomFileName') == null ? + "${project.convention.getByType(BasePluginConvention).archivesBaseName}-${project.version}.pom" : + ext.get('pomFileName') } } } @@ -566,23 +528,16 @@ class BuildPlugin implements Plugin { assemble.dependsOn(generatePOMTask) } } - project.plugins.withType(MavenPublishPlugin.class).whenPluginAdded { - project.publishing { - publications { - all { MavenPublication publication -> // we only deal with maven - // add exclusions to the pom directly, for each of the transitive deps of this project's deps - publication.pom.withXml(fixupDependencies(project)) - } - } + project.pluginManager.withPlugin('maven-publishing') { + PublishingExtension publishing = project.extensions.getByType(PublishingExtension) + publishing.publications.all { MavenPublication publication -> // we only deal with maven + // add exclusions to the pom directly, for each of the transitive deps of this project's deps + publication.pom.withXml(fixupDependencies(project)) } project.plugins.withType(ShadowPlugin).whenPluginAdded { - project.publishing { - publications { - nebula(MavenPublication) { - artifacts = [ project.tasks.shadowJar ] - } - } - } + publishing.publications.create('nebula', MavenPublication, { MavenPublication publication -> + publication.artifacts = [ project.tasks.getByName('shadowJar') ] + } as Action) } } } @@ -593,9 +548,9 @@ class BuildPlugin implements Plugin { static void configureSourceSets(Project project) { project.plugins.withType(ShadowPlugin).whenPluginAdded { ['main', 'test'].each {name -> - SourceSet sourceSet = project.sourceSets.findByName(name) + SourceSet sourceSet = project.extensions.getByType(SourceSetContainer).findByName(name) if (sourceSet != null) { - sourceSet.compileClasspath += project.configurations.bundle + sourceSet.compileClasspath += project.configurations.getByName('bundle') } } } @@ -603,16 +558,19 @@ class BuildPlugin implements Plugin { /** Adds compiler settings to the project */ static void configureCompile(Project project) { - project.ext.compactProfile = 'full' + ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) + ext.set('compactProfile', 'full') project.afterEvaluate { - project.tasks.withType(JavaCompile) { - final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(it.targetCompatibility) + File compilerJavaHome = ext.get('compilerJavaHome') as File + + project.tasks.withType(JavaCompile) { JavaCompile compileTask -> + final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(compileTask.targetCompatibility) // we only fork if the Gradle JDK is not the same as the compiler JDK - if (project.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { - options.fork = false + if (compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { + compileTask.options.fork = false } else { - options.fork = true - options.forkOptions.javaHome = project.compilerJavaHome + compileTask.options.fork = true + compileTask.options.forkOptions.javaHome = compilerJavaHome } /* * -path because gradle will send in paths that don't always exist. @@ -621,28 +579,28 @@ class BuildPlugin implements Plugin { */ // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :) // fail on all javac warnings - options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation,-try' << '-Xdoclint:all' << '-Xdoclint:-missing' + compileTask.options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation,-try' << '-Xdoclint:all' << '-Xdoclint:-missing' // either disable annotation processor completely (default) or allow to enable them if an annotation processor is explicitly defined - if (options.compilerArgs.contains("-processor") == false) { - options.compilerArgs << '-proc:none' + if (compileTask.options.compilerArgs.contains("-processor") == false) { + compileTask.options.compilerArgs << '-proc:none' } - options.encoding = 'UTF-8' - options.incremental = true + compileTask.options.encoding = 'UTF-8' + compileTask.options.incremental = true // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510) - options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion + compileTask.options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion } // also apply release flag to groovy, which is used in build-tools - project.tasks.withType(GroovyCompile) { + project.tasks.withType(GroovyCompile) { GroovyCompile compileTask -> // we only fork if the Gradle JDK is not the same as the compiler JDK - if (project.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { - options.fork = false + if (compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { + compileTask.options.fork = false } else { - options.fork = true - options.forkOptions.javaHome = project.compilerJavaHome - options.compilerArgs << '--release' << JavaVersion.toVersion(it.targetCompatibility).majorVersion + compileTask.options.fork = true + compileTask.options.forkOptions.javaHome = compilerJavaHome + compileTask.options.compilerArgs << '--release' << JavaVersion.toVersion(compileTask.targetCompatibility).majorVersion } } } @@ -651,11 +609,12 @@ class BuildPlugin implements Plugin { static void configureJavadoc(Project project) { // remove compiled classes from the Javadoc classpath: http://mail.openjdk.java.net/pipermail/javadoc-dev/2018-January/000400.html final List classes = new ArrayList<>() - project.tasks.withType(JavaCompile) { javaCompile -> + project.tasks.withType(JavaCompile) { JavaCompile javaCompile -> classes.add(javaCompile.destinationDir) } - project.tasks.withType(Javadoc) { javadoc -> - javadoc.executable = new File(project.compilerJavaHome, 'bin/javadoc') + project.tasks.withType(Javadoc) { Javadoc javadoc -> + File compilerJavaHome = project.extensions.getByType(ExtraPropertiesExtension).get('compilerJavaHome') as File + javadoc.executable = new File(compilerJavaHome, 'bin/javadoc') javadoc.classpath = javadoc.getClasspath().filter { f -> return classes.contains(f) == false } @@ -663,34 +622,35 @@ class BuildPlugin implements Plugin { * Generate docs using html5 to suppress a warning from `javadoc` * that the default will change to html5 in the future. */ - javadoc.options.addBooleanOption('html5', true) + (javadoc.options as CoreJavadocOptions).addBooleanOption('html5', true) } configureJavadocJar(project) } /** Adds a javadocJar task to generate a jar containing javadocs. */ static void configureJavadocJar(Project project) { - Jar javadocJarTask = project.task('javadocJar', type: Jar) + Jar javadocJarTask = project.tasks.create('javadocJar', Jar) javadocJarTask.classifier = 'javadoc' javadocJarTask.group = 'build' javadocJarTask.description = 'Assembles a jar containing javadocs.' javadocJarTask.from(project.tasks.getByName(JavaPlugin.JAVADOC_TASK_NAME)) - project.assemble.dependsOn(javadocJarTask) + project.tasks.getByName(BasePlugin.ASSEMBLE_TASK_NAME).dependsOn(javadocJarTask) } static void configureSourcesJar(Project project) { - Jar sourcesJarTask = project.task('sourcesJar', type: Jar) + Jar sourcesJarTask = project.tasks.create('sourcesJar', Jar) sourcesJarTask.classifier = 'sources' sourcesJarTask.group = 'build' sourcesJarTask.description = 'Assembles a jar containing source files.' - sourcesJarTask.from(project.sourceSets.main.allSource) - project.assemble.dependsOn(sourcesJarTask) + sourcesJarTask.from(project.extensions.getByType(SourceSetContainer).getByName(SourceSet.MAIN_SOURCE_SET_NAME).allSource) + project.tasks.getByName(BasePlugin.ASSEMBLE_TASK_NAME).dependsOn(sourcesJarTask) } /** Adds additional manifest info to jars */ static void configureJars(Project project) { - project.ext.licenseFile = null - project.ext.noticeFile = null + ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) + ext.set('licenseFile', null) + ext.set('noticeFile', null) project.tasks.withType(Jar) { Jar jarTask -> // we put all our distributable files under distributions jarTask.destinationDir = new File(project.buildDir, 'distributions') @@ -698,14 +658,15 @@ class BuildPlugin implements Plugin { jarTask.doFirst { // this doFirst is added before the info plugin, therefore it will run // after the doFirst added by the info plugin, and we can override attributes + JavaVersion compilerJavaVersion = ext.get('compilerJavaVersion') as JavaVersion jarTask.manifest.attributes( 'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch, 'X-Compile-Lucene-Version': VersionProperties.lucene, 'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(), 'Build-Date': ZonedDateTime.now(ZoneOffset.UTC), - 'Build-Java-Version': project.compilerJavaVersion) + 'Build-Java-Version': compilerJavaVersion) if (jarTask.manifest.attributes.containsKey('Change') == false) { - logger.warn('Building without git revision id.') + jarTask.logger.warn('Building without git revision id.') jarTask.manifest.attributes('Change': 'Unknown') } else { /* @@ -724,19 +685,24 @@ class BuildPlugin implements Plugin { jarTask.manifest.getAttributes().clear() } } + // add license/notice files project.afterEvaluate { - if (project.licenseFile == null || project.noticeFile == null) { + if (ext.has('licenseFile') == false || ext.get('licenseFile') == null || ext.has('noticeFile') == false || ext.get('noticeFile') == null) { throw new GradleException("Must specify license and notice file for project ${project.path}") } - jarTask.metaInf { - from(project.licenseFile.parent) { - include project.licenseFile.name - rename { 'LICENSE.txt' } + + File licenseFile = ext.get('licenseFile') as File + File noticeFile = ext.get('noticeFile') as File + + jarTask.metaInf { CopySpec spec -> + spec.from(licenseFile.parent) { CopySpec from -> + from.include licenseFile.name + from.rename { 'LICENSE.txt' } } - from(project.noticeFile.parent) { - include project.noticeFile.name - rename { 'NOTICE.txt' } + spec.from(noticeFile.parent) { CopySpec from -> + from.include noticeFile.name + from.rename { 'NOTICE.txt' } } } } @@ -747,35 +713,35 @@ class BuildPlugin implements Plugin { * normal jar with the shadow jar so we no longer want to run * the jar task. */ - project.tasks.jar.enabled = false - project.tasks.shadowJar { + project.tasks.getByName(JavaPlugin.JAR_TASK_NAME).enabled = false + project.tasks.getByName('shadowJar').configure { ShadowJar shadowJar -> /* * Replace the default "shadow" classifier with null * which will leave the classifier off of the file name. */ - classifier = null + shadowJar.classifier = null /* * Not all cases need service files merged but it is * better to be safe */ - mergeServiceFiles() + shadowJar.mergeServiceFiles() /* * Bundle dependencies of the "bundled" configuration. */ - configurations = [project.configurations.bundle] + shadowJar.configurations = [project.configurations.getByName('bundle')] } // Make sure we assemble the shadow jar - project.tasks.assemble.dependsOn project.tasks.shadowJar - project.artifacts { - apiElements project.tasks.shadowJar - } + project.tasks.getByName(BasePlugin.ASSEMBLE_TASK_NAME).dependsOn project.tasks.getByName('shadowJar') + project.artifacts.add('apiElements', project.tasks.getByName('shadowJar')) } } static void configureTestTasks(Project project) { + ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) + // Default test task should run only unit tests - project.tasks.withType(Test).matching { it.name == 'test' }.all { - include '**/*Tests.class' + project.tasks.withType(Test).matching { Test task -> task.name == 'test' }.all { Test task -> + task.include '**/*Tests.class' } // none of this stuff is applicable to the `:buildSrc` project tests @@ -785,7 +751,7 @@ class BuildPlugin implements Plugin { project.tasks.withType(Test) { Test test -> File testOutputDir = new File(test.reports.junitXml.getDestination(), "output") - doFirst { + test.doFirst { project.mkdir(testOutputDir) project.mkdir(heapdumpDir) project.mkdir(test.workingDir) @@ -793,8 +759,8 @@ class BuildPlugin implements Plugin { ErrorReportingTestListener listener = new ErrorReportingTestListener(test.testLogging, testOutputDir) test.extensions.add(ErrorReportingTestListener, 'errorReportingTestListener', listener) - addTestOutputListener(listener) - addTestListener(listener) + test.addTestOutputListener(listener) + test.addTestListener(listener) /* * We use lazy-evaluated strings in order to configure system properties whose value will not be known until @@ -808,15 +774,15 @@ class BuildPlugin implements Plugin { */ SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider() test.jvmArgumentProviders.add(nonInputProperties) - test.ext.nonInputProperties = nonInputProperties + test.extensions.getByType(ExtraPropertiesExtension).set('nonInputProperties', nonInputProperties) - executable = "${project.runtimeJavaHome}/bin/java" - workingDir = project.file("${project.buildDir}/testrun/${test.name}") - maxParallelForks = project.rootProject.ext.defaultParallel + test.executable = "${ext.get('runtimeJavaHome')}/bin/java" + test.workingDir = project.file("${project.buildDir}/testrun/${test.name}") + test.maxParallelForks = project.rootProject.extensions.getByType(ExtraPropertiesExtension).get('defaultParallel') as Integer - exclude '**/*$*.class' + test.exclude '**/*$*.class' - jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}", + test.jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}", "-Xms${System.getProperty('tests.heap.size', '512m')}", '-XX:+HeapDumpOnOutOfMemoryError', "-XX:HeapDumpPath=$heapdumpDir", @@ -824,62 +790,62 @@ class BuildPlugin implements Plugin { if (System.getProperty('tests.jvm.argline')) { - jvmArgs System.getProperty('tests.jvm.argline').split(" ") + test.jvmArgs System.getProperty('tests.jvm.argline').split(" ") } if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { - jvmArgs '-ea', '-esa' + test.jvmArgs '-ea', '-esa' } // we use './temp' since this is per JVM and tests are forbidden from writing to CWD - systemProperties 'gradle.dist.lib': new File(project.class.location.toURI()).parent, + test.systemProperties 'gradle.dist.lib': new File(project.class.location.toURI()).parent, 'gradle.worker.jar': "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar", 'gradle.user.home': project.gradle.getGradleUserHomeDir(), 'java.io.tmpdir': './temp', 'java.awt.headless': 'true', 'tests.gradle': 'true', 'tests.artifact': project.name, - 'tests.task': path, + 'tests.task': test.path, 'tests.security.manager': 'true', - 'tests.seed': project.testSeed, + 'tests.seed': project.property('testSeed'), 'jna.nosys': 'true' - nonInputProperties.systemProperty('compiler.java', "${-> project.ext.compilerJavaVersion.getMajorVersion()}") + nonInputProperties.systemProperty('compiler.java', "${-> (ext.get('compilerJavaVersion') as JavaVersion).getMajorVersion()}") - if (project.ext.inFipsJvm) { - nonInputProperties.systemProperty('runtime.java', "${-> project.ext.runtimeJavaVersion.getMajorVersion()} FIPS") + if (project.property('inFipsJvm')) { + nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()} FIPS") } else { - nonInputProperties.systemProperty('runtime.java', "${-> project.ext.runtimeJavaVersion.getMajorVersion()}") + nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()}") } // TODO: remove setting logging level via system property - systemProperty 'tests.logger.level', 'WARN' + test.systemProperty 'tests.logger.level', 'WARN' System.getProperties().each { key, value -> - if ((key.startsWith('tests.') || key.startsWith('es.'))) { - systemProperty key, value + if ((key.toString().startsWith('tests.') || key.toString().startsWith('es.'))) { + test.systemProperty key.toString(), value } } // TODO: remove this once ctx isn't added to update script params in 7.0 - systemProperty 'es.scripting.update.ctx_in_params', 'false' + test.systemProperty 'es.scripting.update.ctx_in_params', 'false' // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM - if (project.inFipsJvm) { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' + if (project.property('inFipsJvm')) { + test.systemProperty 'javax.net.ssl.trustStorePassword', 'password' + test.systemProperty 'javax.net.ssl.keyStorePassword', 'password' } - testLogging { - showExceptions = true - showCauses = true - exceptionFormat = 'full' + test.testLogging { TestLoggingContainer logging -> + logging.showExceptions = true + logging.showCauses = true + logging.exceptionFormat = 'full' } project.plugins.withType(ShadowPlugin).whenPluginAdded { // Test against a shadow jar if we made one - classpath -= project.tasks.compileJava.outputs.files - classpath += project.tasks.shadowJar.outputs.files + test.classpath -= project.tasks.getByName('compileJava').outputs.files + test.classpath += project.tasks.getByName('shadowJar').outputs.files - dependsOn project.tasks.shadowJar + test.dependsOn project.tasks.getByName('shadowJar') } } } @@ -887,29 +853,29 @@ class BuildPlugin implements Plugin { private static configurePrecommit(Project project) { Task precommit = PrecommitTasks.create(project, true) - project.check.dependsOn(precommit) - project.test.mustRunAfter(precommit) + project.tasks.getByName(LifecycleBasePlugin.CHECK_TASK_NAME).dependsOn(precommit) + project.tasks.getByName(JavaPlugin.TEST_TASK_NAME).mustRunAfter(precommit) // only require dependency licenses for non-elasticsearch deps - project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection { - it.group.startsWith('org.elasticsearch') == false - } - project.configurations.compileOnly + (project.tasks.getByName('dependencyLicenses') as DependencyLicensesTask).dependencies = project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME).fileCollection { Dependency dependency -> + dependency.group.startsWith('org.elasticsearch') == false + } - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME) project.plugins.withType(ShadowPlugin).whenPluginAdded { - project.dependencyLicenses.dependencies += project.configurations.bundle.fileCollection { - it.group.startsWith('org.elasticsearch') == false + (project.tasks.getByName('dependencyLicenses') as DependencyLicensesTask).dependencies += project.configurations.getByName('bundle').fileCollection { Dependency dependency -> + dependency.group.startsWith('org.elasticsearch') == false } } } private static configureDependenciesInfo(Project project) { - Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class) - deps.runtimeConfiguration = project.configurations.runtime + DependenciesInfoTask deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask) + deps.runtimeConfiguration = project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME) project.plugins.withType(ShadowPlugin).whenPluginAdded { deps.runtimeConfiguration = project.configurations.create('infoDeps') - deps.runtimeConfiguration.extendsFrom(project.configurations.runtime, project.configurations.bundle) + deps.runtimeConfiguration.extendsFrom(project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME), project.configurations.getByName('bundle')) } - deps.compileOnlyConfiguration = project.configurations.compileOnly + deps.compileOnlyConfiguration = project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME) project.afterEvaluate { - deps.mappings = project.dependencyLicenses.mappings + deps.mappings = (project.tasks.getByName('dependencyLicenses') as DependencyLicensesTask).mappings } } @@ -921,11 +887,12 @@ class BuildPlugin implements Plugin { * the reproduction line from one run be useful on another run. */ static String setupSeed(Project project) { - if (project.rootProject.ext.has('testSeed')) { + ExtraPropertiesExtension ext = project.rootProject.extensions.getByType(ExtraPropertiesExtension) + if (ext.has('testSeed')) { /* Skip this if we've already pinned the testSeed. It is important * that this checks the rootProject so that we know we've only ever * initialized one time. */ - return project.rootProject.ext.testSeed + return ext.get('testSeed') } String testSeed = System.getProperty('tests.seed') @@ -934,7 +901,7 @@ class BuildPlugin implements Plugin { testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT) } - project.rootProject.ext.testSeed = testSeed + ext.set('testSeed', testSeed) return testSeed } From 90e4d40fb8a7ecd10e973f2ec63f012704ca6a1b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 14:23:05 -0700 Subject: [PATCH 06/32] Remove unnecessary Fips configuration --- .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 6 ------ 1 file changed, 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 829b01c1af550..9213dd1d0b5d0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -828,12 +828,6 @@ class BuildPlugin implements Plugin { // TODO: remove this once ctx isn't added to update script params in 7.0 test.systemProperty 'es.scripting.update.ctx_in_params', 'false' - // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM - if (project.property('inFipsJvm')) { - test.systemProperty 'javax.net.ssl.trustStorePassword', 'password' - test.systemProperty 'javax.net.ssl.keyStorePassword', 'password' - } - test.testLogging { TestLoggingContainer logging -> logging.showExceptions = true logging.showCauses = true From 323f2e797d099e8b4b1736f434090628de291b6a Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 14:41:47 -0700 Subject: [PATCH 07/32] Remove usage of Java 11 API to maintain Java 8 compatibility --- .../gradle/info/GenerateGlobalBuildInfoTask.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index dfcecca01f73e..656d377a815a7 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -27,6 +27,7 @@ import java.io.UncheckedIOException; import java.io.Writer; import java.nio.file.Files; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -174,9 +175,9 @@ private String runJavaAsScript(File javaHome, String script) { if (result.getExitValue() != 0) { getLogger().error("STDOUT:"); - stdout.toString(UTF_8).lines().forEach(getLogger()::error); + Arrays.stream(stdout.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error); getLogger().error("STDERR:"); - stderr.toString(UTF_8).lines().forEach(getLogger()::error); + Arrays.stream(stderr.toString(UTF_8).split(System.getProperty("line.separator"))).forEach(getLogger()::error); result.rethrowFailure(); } return stdout.toString(UTF_8).trim(); From 13416664ebedca576ba41df5dc48c705070b94c6 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 15:29:17 -0700 Subject: [PATCH 08/32] Checkstyle --- .../info/GenerateGlobalBuildInfoTask.java | 52 +++++++++++++------ .../gradle/info/GlobalBuildInfoPlugin.java | 25 +++++---- .../gradle/info/PrintGlobalBuildInfoTask.java | 33 ++++++++++-- .../gradle/precommit/ThirdPartyAuditTask.java | 2 - 4 files changed, 81 insertions(+), 31 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 656d377a815a7..35c2dac731820 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -6,7 +6,6 @@ import org.gradle.api.JavaVersion; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.model.ObjectFactory; -import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; @@ -29,7 +28,6 @@ import java.nio.file.Files; import java.util.Arrays; import java.util.List; -import java.util.Map; import static java.nio.charset.StandardCharsets.UTF_8; @@ -41,17 +39,22 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask { private File runtimeJavaHome; private List javaVersions; private final RegularFileProperty outputFile; + private final RegularFileProperty compilerVersionFile; + private final RegularFileProperty runtimeVersionFile; @Inject public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) { this.outputFile = objectFactory.fileProperty(); + this.compilerVersionFile = objectFactory.fileProperty(); + this.runtimeVersionFile = objectFactory.fileProperty(); } @TaskAction public void generate() { String javaVendor = System.getProperty("java.vendor"); String gradleJavaVersion = System.getProperty("java.version"); - String gradleJavaVersionDetails = javaVendor + " " + gradleJavaVersion + " [" + System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version") + "]"; + String gradleJavaVersionDetails = javaVendor + " " + gradleJavaVersion + " [" + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.version") + "]"; String compilerJavaVersionDetails = gradleJavaVersionDetails; JavaVersion compilerJavaVersionEnum = JavaVersion.current(); @@ -75,16 +78,20 @@ public void generate() { try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile.getAsFile().get()))) { writer.write(" Gradle Version : " + getProject().getGradle().getGradleVersion() + "\n"); - writer.write(" OS Info : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + " (" + System.getProperty("os.arch") + ")\n"); - if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) { + writer.write(" OS Info : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + + " (" + System.getProperty("os.arch") + ")\n"); + if (gradleJavaVersionDetails.equals(compilerJavaVersionDetails) == false + || gradleJavaVersionDetails.equals(runtimeJavaVersionDetails) == false) { writer.write(" Compiler JDK Version : " + compilerJavaVersionEnum + " (" + compilerJavaVersionDetails + ")\n"); writer.write(" Compiler java.home : " + compilerJavaHome + "\n"); writer.write(" Runtime JDK Version : " + runtimeJavaVersionEnum + " (" + runtimeJavaVersionDetails + ")\n"); writer.write(" Runtime java.home : " + runtimeJavaHome + "\n"); - writer.write(" Gradle JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")\n"); + writer.write(" Gradle JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + + " (" + gradleJavaVersionDetails + ")\n"); writer.write(" Gradle java.home : " + gradleJavaHome); } else { - writer.write(" JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + " (" + gradleJavaVersionDetails + ")\n"); + writer.write(" JDK Version : " + JavaVersion.toVersion(gradleJavaVersion) + + " (" + gradleJavaVersionDetails + ")\n"); writer.write(" JAVA_HOME : " + gradleJavaHome); } } catch (IOException e) { @@ -94,7 +101,7 @@ public void generate() { // enforce Java version if (compilerJavaVersionEnum.compareTo(minimumCompilerVersion) < 0) { String message = "The compiler java.home must be set to a JDK installation directory for Java " + minimumCompilerVersion + - " but is [" + compilerJavaHome + "] corresponding to [" + compilerJavaVersionEnum +"]"; + " but is [" + compilerJavaHome + "] corresponding to [" + compilerJavaVersionEnum + "]"; throw new GradleException(message); } @@ -119,19 +126,22 @@ public void generate() { } if (javaVersionEnum != expectedJavaVersionEnum) { String message = "The environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java " + - expectedJavaVersionEnum + " but is [" + javaHome + "] corresponding to [" + javaVersionEnum + "]"; + expectedJavaVersionEnum + " but is [" + javaHome + "] corresponding to [" + javaVersionEnum + "]"; throw new GradleException(message); } } - JavaVersion finalCompilerJavaVersionEnum = compilerJavaVersionEnum; - JavaVersion finalRuntimeJavaVersionEnum = runtimeJavaVersionEnum; + try (Writer writer = new FileWriter(compilerVersionFile.getAsFile().get())) { + writer.write(compilerJavaVersionEnum.name()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } - getProject().getRootProject().allprojects(p -> { - ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); - ext.set("compilerJavaVersion", finalCompilerJavaVersionEnum); - ext.set("runtimeJavaVersion", finalRuntimeJavaVersionEnum); - }); + try (Writer writer = new FileWriter(runtimeVersionFile.getAsFile().get())) { + writer.write(runtimeJavaVersionEnum.name()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } /** @@ -234,4 +244,14 @@ public void setJavaVersions(List javaVersions) { public RegularFileProperty getOutputFile() { return outputFile; } + + @OutputFile + public RegularFileProperty getCompilerVersionFile() { + return compilerVersionFile; + } + + @OutputFile + public RegularFileProperty getRuntimeVersionFile() { + return runtimeVersionFile; + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 6342a0b57bbeb..c655d4b22a74b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -44,17 +44,22 @@ public void apply(Project project) { } } - GenerateGlobalBuildInfoTask generateTask = project.getTasks().create("generateGlobalBuildInfo", GenerateGlobalBuildInfoTask.class, task -> { - task.setJavaVersions(javaVersions); - task.setMinimumCompilerVersion(minimumCompilerVersion); - task.setMinimumRuntimeVersion(minimumRuntimeVersion); - task.setCompilerJavaHome(compilerJavaHome); - task.setRuntimeJavaHome(runtimeJavaHome); - task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info")); - }); + GenerateGlobalBuildInfoTask generateTask = project.getTasks().create("generateGlobalBuildInfo", + GenerateGlobalBuildInfoTask.class, task -> { + task.setJavaVersions(javaVersions); + task.setMinimumCompilerVersion(minimumCompilerVersion); + task.setMinimumRuntimeVersion(minimumRuntimeVersion); + task.setCompilerJavaHome(compilerJavaHome); + task.setRuntimeJavaHome(runtimeJavaHome); + task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info")); + task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version")); + task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version")); + }); PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> { task.getBuildInfoFile().set(generateTask.getOutputFile()); + task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile()); + task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile()); }); project.getExtensions().getByType(ExtraPropertiesExtension.class).set("defaultParallel", findDefaultParallel(project)); @@ -120,7 +125,9 @@ private static String getJavaHomeEnvVarName(String version) { } private static String getResourceContents(String resourcePath) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))) { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)) + )) { StringBuilder b = new StringBuilder(); for (String line = reader.readLine(); line != null; line = reader.readLine()) { if (b.length() != 0) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index 240f0f136db46..7603fff500e68 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -1,8 +1,10 @@ package org.elasticsearch.gradle.info; import org.gradle.api.DefaultTask; +import org.gradle.api.JavaVersion; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.model.ObjectFactory; +import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.resources.TextResource; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.TaskAction; @@ -10,20 +12,26 @@ import javax.inject.Inject; public class PrintGlobalBuildInfoTask extends DefaultTask { - private RegularFileProperty buildInfoFile; + private final RegularFileProperty buildInfoFile; + private final RegularFileProperty compilerVersionFile; + private final RegularFileProperty runtimeVersionFile; @Inject public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) { this.buildInfoFile = objectFactory.fileProperty(); + this.compilerVersionFile = objectFactory.fileProperty(); + this.runtimeVersionFile = objectFactory.fileProperty(); } @TaskAction public void print() { getLogger().quiet("======================================="); getLogger().quiet("Elasticsearch Build Hamster says Hello!"); - getLogger().quiet(getBuildInfoText().asString()); + getLogger().quiet(getFileText(getBuildInfoFile()).asString()); getLogger().quiet(" Random Testing Seed : " + getProject().property("testSeed")); getLogger().quiet("======================================="); + + setGlobalProperties(); } @InputFile @@ -31,8 +39,25 @@ public RegularFileProperty getBuildInfoFile() { return buildInfoFile; } - private TextResource getBuildInfoText() { - return getProject().getResources().getText().fromFile(buildInfoFile.getAsFile().get()); + @InputFile + public RegularFileProperty getCompilerVersionFile() { + return compilerVersionFile; + } + + @InputFile + public RegularFileProperty getRuntimeVersionFile() { + return runtimeVersionFile; + } + + private TextResource getFileText(RegularFileProperty regularFileProperty) { + return getProject().getResources().getText().fromFile(regularFileProperty.getAsFile().get()); } + private void setGlobalProperties() { + getProject().getRootProject().allprojects(p -> { + ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); + ext.set("compilerJavaVersion", JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString())); + ext.set("runtimeJavaVersion", JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString())); + }); + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java index 563ea0761ddde..16c491fa1647d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java @@ -19,7 +19,6 @@ package org.elasticsearch.gradle.precommit; import org.apache.commons.io.output.NullOutputStream; -import org.apache.tools.ant.taskdefs.Java; import org.elasticsearch.gradle.JdkJarHellCheck; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; @@ -28,7 +27,6 @@ import org.gradle.api.artifacts.Dependency; import org.gradle.api.file.FileTree; import org.gradle.api.provider.Property; -import org.gradle.api.provider.Provider; import org.gradle.api.specs.Spec; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Input; From 305079900a798abdb62ca8af09e10cf79b70bb42 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 15:47:39 -0700 Subject: [PATCH 09/32] Correctly configure target and source compatibility --- .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 9213dd1d0b5d0..cd6fdd621efd0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -57,6 +57,7 @@ import org.gradle.api.plugins.BasePlugin import org.gradle.api.plugins.BasePluginConvention import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaPlugin +import org.gradle.api.plugins.JavaPluginExtension import org.gradle.api.publish.PublishingExtension import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.publish.maven.tasks.GenerateMavenPom @@ -560,6 +561,10 @@ class BuildPlugin implements Plugin { static void configureCompile(Project project) { ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) ext.set('compactProfile', 'full') + + project.extensions.getByType(JavaPluginExtension).sourceCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion + project.extensions.getByType(JavaPluginExtension).targetCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion + project.afterEvaluate { File compilerJavaHome = ext.get('compilerJavaHome') as File From 202541ab711a7eed798a30e76854d6ac143f0873 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 16:30:55 -0700 Subject: [PATCH 10/32] Fixes for fipsJvm-specific config --- build.gradle | 19 ++++++----- .../elasticsearch/gradle/BuildPlugin.groovy | 24 +++++++------- .../gradle/test/ClusterFormationTasks.groovy | 12 +++---- .../info/GenerateGlobalBuildInfoTask.java | 27 ++++++++++++---- .../gradle/info/GlobalBuildInfoPlugin.java | 7 +++- .../gradle/info/GlobalInfoExtension.java | 12 +++++++ .../gradle/info/PrintGlobalBuildInfoTask.java | 22 +++++++++++++ distribution/tools/plugin-cli/build.gradle | 6 ++-- modules/transport-netty4/build.gradle | 14 ++++---- plugins/ingest-attachment/build.gradle | 10 +++--- plugins/transport-nio/build.gradle | 14 ++++---- x-pack/plugin/security/cli/build.gradle | 26 ++++++++------- x-pack/qa/full-cluster-restart/build.gradle | 32 +++++++++++-------- x-pack/qa/rolling-upgrade/build.gradle | 32 +++++++++++-------- 14 files changed, 164 insertions(+), 93 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java diff --git a/build.gradle b/build.gradle index 62099d33a9f2e..5258891bc19d6 100644 --- a/build.gradle +++ b/build.gradle @@ -620,20 +620,23 @@ allprojects { } } -subprojects { +// wait until global info is populated because we don't know if we are running in a fips jvm until execution time +globalInfo.ready { + subprojects { // Common config when running with a FIPS-140 runtime JVM if (project.ext.has("inFipsJvm") && project.ext.inFipsJvm) { - tasks.withType(Test) { + tasks.withType(Test) { + systemProperty 'javax.net.ssl.trustStorePassword', 'password' + systemProperty 'javax.net.ssl.keyStorePassword', 'password' + } + project.pluginManager.withPlugin("elasticsearch.testclusters") { + project.testClusters.all { systemProperty 'javax.net.ssl.trustStorePassword', 'password' systemProperty 'javax.net.ssl.keyStorePassword', 'password' } - project.pluginManager.withPlugin("elasticsearch.testclusters") { - project.testClusters.all { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' - } - } + } } + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index cd6fdd621efd0..3f6e9d64a18bf 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -756,12 +756,6 @@ class BuildPlugin implements Plugin { project.tasks.withType(Test) { Test test -> File testOutputDir = new File(test.reports.junitXml.getDestination(), "output") - test.doFirst { - project.mkdir(testOutputDir) - project.mkdir(heapdumpDir) - project.mkdir(test.workingDir) - } - ErrorReportingTestListener listener = new ErrorReportingTestListener(test.testLogging, testOutputDir) test.extensions.add(ErrorReportingTestListener, 'errorReportingTestListener', listener) test.addTestOutputListener(listener) @@ -778,6 +772,19 @@ class BuildPlugin implements Plugin { * build cache key or up to date checking. */ SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider() + + test.doFirst { + project.mkdir(testOutputDir) + project.mkdir(heapdumpDir) + project.mkdir(test.workingDir) + + if (project.property('inFipsJvm')) { + nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()} FIPS") + } else { + nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()}") + } + } + test.jvmArgumentProviders.add(nonInputProperties) test.extensions.getByType(ExtraPropertiesExtension).set('nonInputProperties', nonInputProperties) @@ -817,11 +824,6 @@ class BuildPlugin implements Plugin { nonInputProperties.systemProperty('compiler.java', "${-> (ext.get('compilerJavaVersion') as JavaVersion).getMajorVersion()}") - if (project.property('inFipsJvm')) { - nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()} FIPS") - } else { - nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()}") - } // TODO: remove setting logging level via system property test.systemProperty 'tests.logger.level', 'WARN' System.getProperties().each { key, value -> diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 1b5eb26f9d322..01ba34092e478 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -300,12 +300,6 @@ class ClusterFormationTasks { // its run after plugins have been installed, as the extra config files may belong to plugins setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node) - // If the node runs in a FIPS 140-2 JVM, the BCFKS default keystore will be password protected - if (project.inFipsJvm){ - node.config.systemProperties.put('javax.net.ssl.trustStorePassword', 'password') - node.config.systemProperties.put('javax.net.ssl.keyStorePassword', 'password') - } - // extra setup commands for (Map.Entry command : node.config.setupCommands.entrySet()) { // the first argument is the actual script name, relative to home @@ -733,6 +727,12 @@ class ClusterFormationTasks { } start.doLast(elasticsearchRunner) start.doFirst { + // If the node runs in a FIPS 140-2 JVM, the BCFKS default keystore will be password protected + if (project.inFipsJvm){ + node.config.systemProperties.put('javax.net.ssl.trustStorePassword', 'password') + node.config.systemProperties.put('javax.net.ssl.keyStorePassword', 'password') + } + // Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc List esJavaOpts = [node.env.get('ES_JAVA_OPTS', '')] String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ") diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 35c2dac731820..1c367d6ba2f3d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -41,12 +41,14 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask { private final RegularFileProperty outputFile; private final RegularFileProperty compilerVersionFile; private final RegularFileProperty runtimeVersionFile; + private final RegularFileProperty fipsJvmFile; @Inject public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) { this.outputFile = objectFactory.fileProperty(); this.compilerVersionFile = objectFactory.fileProperty(); this.runtimeVersionFile = objectFactory.fileProperty(); + this.fipsJvmFile = objectFactory.fileProperty(); } @TaskAction @@ -61,6 +63,7 @@ public void generate() { String runtimeJavaVersionDetails = gradleJavaVersionDetails; JavaVersion runtimeJavaVersionEnum = JavaVersion.current(); File gradleJavaHome = Jvm.current().getJavaHome(); + boolean inFipsJvm = false; try { if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { @@ -72,6 +75,12 @@ public void generate() { runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome); runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome)); } + + if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { + // We don't expect Gradle to be running in a FIPS JVM + String inFipsJvmScript = "print(java.security.Security.getProviders()[0].name.toLowerCase().contains(\"fips\"));"; + inFipsJvm = Boolean.parseBoolean(runJavaAsScript(runtimeJavaHome, inFipsJvmScript)); + } } catch (IOException e) { throw new UncheckedIOException(e); } @@ -131,14 +140,14 @@ public void generate() { } } - try (Writer writer = new FileWriter(compilerVersionFile.getAsFile().get())) { - writer.write(compilerJavaVersionEnum.name()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name()); + writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name()); + writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm)); + } - try (Writer writer = new FileWriter(runtimeVersionFile.getAsFile().get())) { - writer.write(runtimeJavaVersionEnum.name()); + private void writeToFile(File file, String content) { + try (Writer writer = new FileWriter(file)) { + writer.write(content); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -254,4 +263,8 @@ public RegularFileProperty getCompilerVersionFile() { public RegularFileProperty getRuntimeVersionFile() { return runtimeVersionFile; } + + public RegularFileProperty getFipsJvmFile() { + return fipsJvmFile; + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index c655d4b22a74b..192bb71861f0f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -23,6 +23,7 @@ import java.util.stream.Collectors; public class GlobalBuildInfoPlugin implements Plugin { + private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo"; private static Integer _defaultParallel = null; @Override @@ -31,6 +32,8 @@ public void apply(Project project) { throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); } + GlobalInfoExtension extension = project.getExtensions().create(GLOBAL_INFO_EXTENSION_NAME, GlobalInfoExtension.class); + JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion")); JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion")); @@ -54,12 +57,15 @@ public void apply(Project project) { task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info")); task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version")); task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version")); + task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm")); }); PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> { task.getBuildInfoFile().set(generateTask.getOutputFile()); task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile()); task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile()); + task.getFipsJvmFile().set(generateTask.getFipsJvmFile()); + task.setGlobalInfoListeners(extension.listeners); }); project.getExtensions().getByType(ExtraPropertiesExtension.class).set("defaultParallel", findDefaultParallel(project)); @@ -80,7 +86,6 @@ public void apply(Project project) { ext.set("javaVersions", javaVersions); ext.set("minimumCompilerVersion", minimumCompilerVersion); ext.set("minimumRuntimeVersion", minimumRuntimeVersion); - ext.set("inFipsJvm", true); ext.set("gradleJavaVersion", Jvm.current().getJavaVersion()); }); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java new file mode 100644 index 0000000000000..a2daa4a5767c0 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalInfoExtension.java @@ -0,0 +1,12 @@ +package org.elasticsearch.gradle.info; + +import java.util.ArrayList; +import java.util.List; + +public class GlobalInfoExtension { + final List listeners = new ArrayList<>(); + + public void ready(Runnable block) { + listeners.add(block); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index 7603fff500e68..8e38ccb747837 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -7,20 +7,26 @@ import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.resources.TextResource; import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.TaskAction; import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; public class PrintGlobalBuildInfoTask extends DefaultTask { private final RegularFileProperty buildInfoFile; private final RegularFileProperty compilerVersionFile; private final RegularFileProperty runtimeVersionFile; + private final RegularFileProperty fipsJvmFile; + private List globalInfoListeners = new ArrayList<>(); @Inject public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) { this.buildInfoFile = objectFactory.fileProperty(); this.compilerVersionFile = objectFactory.fileProperty(); this.runtimeVersionFile = objectFactory.fileProperty(); + this.fipsJvmFile = objectFactory.fileProperty(); } @TaskAction @@ -32,6 +38,7 @@ public void print() { getLogger().quiet("======================================="); setGlobalProperties(); + globalInfoListeners.forEach(Runnable::run); } @InputFile @@ -49,6 +56,20 @@ public RegularFileProperty getRuntimeVersionFile() { return runtimeVersionFile; } + @InputFile + public RegularFileProperty getFipsJvmFile() { + return fipsJvmFile; + } + + @Internal + public List getGlobalInfoListeners() { + return globalInfoListeners; + } + + public void setGlobalInfoListeners(List globalInfoListeners) { + this.globalInfoListeners = globalInfoListeners; + } + private TextResource getFileText(RegularFileProperty regularFileProperty) { return getProject().getResources().getText().fromFile(regularFileProperty.getAsFile().get()); } @@ -58,6 +79,7 @@ private void setGlobalProperties() { ExtraPropertiesExtension ext = p.getExtensions().getByType(ExtraPropertiesExtension.class); ext.set("compilerJavaVersion", JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString())); ext.set("runtimeJavaVersion", JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString())); + ext.set("inFipsJvm", Boolean.valueOf(getFileText(getFipsJvmFile()).asString())); }); } } diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 61e3546ed8919..48bc899cd29b4 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -40,8 +40,8 @@ test { systemProperty 'tests.security.manager', 'false' } -if (project.inFipsJvm) { +thirdPartyAudit.onlyIf { // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit, // rather than provide a long list of exclusions, disable the check on FIPS. - thirdPartyAudit.enabled = false -} + project.inFipsJvm == false +} \ No newline at end of file diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index e8050c637f160..1369f00eec398 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -171,10 +171,12 @@ thirdPartyAudit { ) } -if (project.inFipsJvm == false) { - // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in - // a FIPS JVM with BouncyCastleFIPS Provider - thirdPartyAudit.ignoreMissingClasses ( - 'org.bouncycastle.asn1.x500.X500Name' - ) +rootProject.globalInfo.ready { + if (project.inFipsJvm == false) { + // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in + // a FIPS JVM with BouncyCastleFIPS Provider + thirdPartyAudit.ignoreMissingClasses( + 'org.bouncycastle.asn1.x500.X500Name' + ) + } } diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index cbe417708d778..835147c255c8b 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -84,8 +84,8 @@ thirdPartyAudit{ ignoreMissingClasses() } -if (project.inFipsJvm) { - // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit, - // rather than provide a long list of exclusions, disable the check on FIPS. - thirdPartyAudit.enabled = false -} +thirdPartyAudit.onlyIf { + // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit, + // rather than provide a long list of exclusions, disable the check on FIPS. + project.inFipsJvm == false +} \ No newline at end of file diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 9e855995c3bfa..7684e4036cc8f 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -148,10 +148,12 @@ thirdPartyAudit { 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator' ) } -if (project.inFipsJvm == false) { - // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in - // a FIPS JVM with BouncyCastleFIPS Provider - thirdPartyAudit.ignoreMissingClasses ( - 'org.bouncycastle.asn1.x500.X500Name' - ) +rootProject.globalInfo.ready { + if (project.inFipsJvm == false) { + // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in + // a FIPS JVM with BouncyCastleFIPS Provider + thirdPartyAudit.ignoreMissingClasses( + 'org.bouncycastle.asn1.x500.X500Name' + ) + } } diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle index 19a8d11dd6f0c..076aaab289465 100644 --- a/x-pack/plugin/security/cli/build.gradle +++ b/x-pack/plugin/security/cli/build.gradle @@ -22,16 +22,18 @@ dependencyLicenses { mapping from: /bc.*/, to: 'bouncycastle' } -if (project.inFipsJvm) { - test.enabled = false - testingConventions.enabled = false - // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are - // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS. - tasks.withType(CheckForbiddenApis) { - bundledSignatures -= "jdk-non-portable" - } - // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit, - // rather than provide a long list of exclusions, disable the check on FIPS. - thirdPartyAudit.enabled = false +rootProject.globalInfo.ready { + if (project.inFipsJvm) { + test.enabled = false + testingConventions.enabled = false + // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are + // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS. + tasks.withType(CheckForbiddenApis) { + bundledSignatures -= "jdk-non-portable" + } + // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit, + // rather than provide a long list of exclusions, disable the check on FIPS. + thirdPartyAudit.enabled = false -} + } +} \ No newline at end of file diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 7f0e14d2a53bf..70767faf33499 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -116,13 +116,15 @@ for (Version version : bwcVersions.indexCompatible) { setting 'xpack.security.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' - if (project.inFipsJvm) { - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - } else { - setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' - setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + rootProject.globalInfo.ready { + if (project.inFipsJvm) { + setting 'xpack.security.transport.ssl.key', 'testnode.pem' + setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' + keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + } else { + setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' + setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + } } setting 'xpack.license.self_generated.type', 'trial' dependsOn copyTestNodeKeyMaterial @@ -160,13 +162,15 @@ for (Version version : bwcVersions.indexCompatible) { // some tests rely on the translog not being flushed setting 'indices.memory.shard_inactive_time', '20m' setting 'xpack.security.enabled', 'true' - if (project.inFipsJvm) { - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - } else { - setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' - setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + rootProject.globalInfo.ready { + if (project.inFipsJvm) { + setting 'xpack.security.transport.ssl.key', 'testnode.pem' + setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' + keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + } else { + setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' + setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + } } setting 'xpack.license.self_generated.type', 'trial' dependsOn copyTestNodeKeyMaterial diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 53810b89386f1..e1d4d43eae139 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -120,13 +120,15 @@ for (Version version : bwcVersions.wireCompatible) { setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.audit.enabled', 'true' - if (project.inFipsJvm) { - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - } else { - setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' - setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + rootProject.globalInfo.ready { + if (project.inFipsJvm) { + setting 'xpack.security.transport.ssl.key', 'testnode.pem' + setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' + keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + } else { + setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' + setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + } } dependsOn copyTestNodeKeyMaterial extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') @@ -182,13 +184,15 @@ for (Version version : bwcVersions.wireCompatible) { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' - if (project.inFipsJvm) { - setting 'xpack.security.transport.ssl.key', 'testnode.pem' - setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' - keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - } else { - setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' - setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + rootProject.globalInfo.ready { + if (project.inFipsJvm) { + setting 'xpack.security.transport.ssl.key', 'testnode.pem' + setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' + keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + } else { + setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' + setting 'xpack.security.transport.ssl.keystore.password', 'testnode' + } } setting 'node.attr.upgraded', 'true' setting 'xpack.security.authc.token.enabled', 'true' From f403411a78cf49a864f91c5816e023c45f3c11bd Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 19 Apr 2019 16:59:14 -0700 Subject: [PATCH 11/32] Fix reference to BasePluginConvention --- .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3f6e9d64a18bf..12614ff563e4b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -518,7 +518,7 @@ class BuildPlugin implements Plugin { spec.into "${project.buildDir}/distributions" spec.rename { ext.has('pomFileName') && ext.get('pomFileName') == null ? - "${project.convention.getByType(BasePluginConvention).archivesBaseName}-${project.version}.pom" : + "${project.convention.getPlugin(BasePluginConvention).archivesBaseName}-${project.version}.pom" : ext.get('pomFileName') } } From fdeb65f8ac72cda161b818664f2de2d234b974f4 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 08:46:54 -0700 Subject: [PATCH 12/32] Fix publishing plugin configuration --- .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 12614ff563e4b..411e7a4616c60 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -60,6 +60,7 @@ import org.gradle.api.plugins.JavaPlugin import org.gradle.api.plugins.JavaPluginExtension import org.gradle.api.publish.PublishingExtension import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.plugins.MavenPublishPlugin import org.gradle.api.publish.maven.tasks.GenerateMavenPom import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.SourceSetContainer @@ -529,16 +530,17 @@ class BuildPlugin implements Plugin { assemble.dependsOn(generatePOMTask) } } - project.pluginManager.withPlugin('maven-publishing') { + project.plugins.withType(MavenPublishPlugin).whenPluginAdded { PublishingExtension publishing = project.extensions.getByType(PublishingExtension) publishing.publications.all { MavenPublication publication -> // we only deal with maven // add exclusions to the pom directly, for each of the transitive deps of this project's deps publication.pom.withXml(fixupDependencies(project)) } project.plugins.withType(ShadowPlugin).whenPluginAdded { - publishing.publications.create('nebula', MavenPublication, { MavenPublication publication -> - publication.artifacts = [ project.tasks.getByName('shadowJar') ] - } as Action) + MavenPublication publication = publishing.publications.maybeCreate('nebula', MavenPublication) + publication.with { + artifacts = [ project.tasks.getByName('shadowJar') ] + } } } } From e0cdc2ae06b7b72eeb254c51c5368e50b8155a29 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 12:39:37 -0700 Subject: [PATCH 13/32] Build plugin and global info plugin are coupled, let's enforce that --- build.gradle | 1 - .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5258891bc19d6..f03ee5c80c0ab 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,6 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder plugins { id 'com.gradle.build-scan' version '2.2.1' id 'base' - id 'elasticsearch.global-build-info' } if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) { buildScan { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 411e7a4616c60..c116b20f55afa 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -26,6 +26,7 @@ import org.apache.commons.io.IOUtils import org.apache.tools.ant.taskdefs.condition.Os import org.eclipse.jgit.lib.Constants import org.eclipse.jgit.lib.RepositoryBuilder +import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin import org.elasticsearch.gradle.info.JavaHome import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.elasticsearch.gradle.precommit.PrecommitTasks @@ -92,6 +93,9 @@ class BuildPlugin implements Plugin { @Override void apply(Project project) { + // make sure the global build info plugin is applied to the root project + project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin) + if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) { throw new InvalidUserDataException('elasticsearch.standalone-test, ' + 'elasticsearch.standalone-rest-test, and elasticsearch.build ' From 99f251034074b7b72e1ee6046276e15dcf4a2e47 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 12:42:57 -0700 Subject: [PATCH 14/32] We need to apply it here as well, you know, ordering and stuff --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index f03ee5c80c0ab..5258891bc19d6 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder plugins { id 'com.gradle.build-scan' version '2.2.1' id 'base' + id 'elasticsearch.global-build-info' } if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) { buildScan { From b169755af994305538b001e551ac107a00af5637 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 12:58:08 -0700 Subject: [PATCH 15/32] Make sure compile conventions are applied to QA projects --- .../elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index ba9c64cb0e945..4dbc6930cfb2e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -57,6 +57,7 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(JavaBasePlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) + BuildPlugin.configureCompile(project) BuildPlugin.configureRepositories(project) BuildPlugin.configureTestTasks(project) From e3c4be609cf04f836881fd885f6eb7210f53ff4d Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 13:14:15 -0700 Subject: [PATCH 16/32] Just get compilation to work --- .../gradle/test/StandaloneRestTestPlugin.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 4dbc6930cfb2e..6d895abaa97c7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -27,11 +27,14 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.precommit.PrecommitTasks import org.gradle.api.InvalidUserDataException +import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.Configuration +import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaBasePlugin import org.gradle.api.plugins.JavaPlugin +import org.gradle.api.plugins.JavaPluginExtension import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.SourceSetContainer import org.gradle.api.tasks.compile.JavaCompile @@ -57,10 +60,13 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(JavaBasePlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) - BuildPlugin.configureCompile(project) BuildPlugin.configureRepositories(project) BuildPlugin.configureTestTasks(project) + ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) + project.extensions.getByType(JavaPluginExtension).sourceCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion + project.extensions.getByType(JavaPluginExtension).targetCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion + // only setup tests to build SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer) SourceSet testSourceSet = sourceSets.create('test') From 60a7d97d9469b1c8a7c465250d61ffcf70ea5358 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 13:28:10 -0700 Subject: [PATCH 17/32] Datastructure is no longer a Map --- .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index c116b20f55afa..4f6c7c8aaeb43 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -284,7 +284,7 @@ class BuildPlugin implements Plugin { } else if (ext.has('requiredJavaVersions') == false || ext.get('requiredJavaVersions') == null) { // check directly if the version is present since we are already executing List javaVersions = ext.get('javaVersions') as List - if (javaVersions.get(version) == null) { + if (javaVersions.find { it.version == version } == null) { throw new GradleException("JAVA${version}_HOME required to run task:\n${task}") } } else { From 041fa3261c8029b3ffbfe466443e65e12dd9b11d Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 22 Apr 2019 13:42:18 -0700 Subject: [PATCH 18/32] Fix java home location --- .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 4f6c7c8aaeb43..e0b74db861a50 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -297,7 +297,7 @@ class BuildPlugin implements Plugin { static String getJavaHome(final Task task, final int version) { requireJavaHome(task, version) def javaVersions = task.project.property('javaVersions') as List - return javaVersions.find { it.version == version } + return javaVersions.find { it.version == version }.javaHome.absolutePath } /** Runs the given javascript using jjs from the jdk, and returns the output */ From 20064e3651e537a035607363e86bc0cd52558674 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 8 May 2019 13:51:39 -0700 Subject: [PATCH 19/32] Fix issue with UP-TO-DATE reporting --- build.gradle | 2 +- .../elasticsearch/gradle/BuildPlugin.groovy | 42 +++++++++---------- .../gradle/info/PrintGlobalBuildInfoTask.java | 10 ++--- .../reindex-tests-with-security/build.gradle | 6 ++- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/build.gradle b/build.gradle index a76d8d00cc161..0c4ea9b200097 100644 --- a/build.gradle +++ b/build.gradle @@ -338,7 +338,7 @@ gradle.projectsEvaluated { if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) { integTest.mustRunAfter test } - configurations.all { Configuration configuration -> + configurations.matching { it.canBeResolved }.all { Configuration configuration -> dependencies.all { Dependency dep -> Project upstreamProject = dependencyToProject(dep) if (upstreamProject != null) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3063c730cfb04..05b66002ad62b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -48,9 +48,6 @@ import org.gradle.api.artifacts.ResolvedArtifact import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.artifacts.repositories.ArtifactRepository import org.gradle.api.artifacts.repositories.IvyArtifactRepository -import org.gradle.api.artifacts.repositories.MavenArtifactRepository -import org.gradle.api.artifacts.repositories.AuthenticationContainer -import org.gradle.api.artifacts.repositories.IvyArtifactRepository import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.credentials.HttpHeaderCredentials @@ -406,7 +403,7 @@ class BuildPlugin implements Plugin { if (repository instanceof MavenArtifactRepository) { final MavenArtifactRepository maven = (MavenArtifactRepository) repository assertRepositoryURIUsesHttps(maven, project, maven.getUrl()) - repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(project, uri) } + repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(maven, project, uri) } } else if (repository instanceof IvyArtifactRepository) { final IvyArtifactRepository ivy = (IvyArtifactRepository) repository assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl()) @@ -420,32 +417,31 @@ class BuildPlugin implements Plugin { repos.mavenLocal() } repos.jcenter() - repos.ivy { - name "elasticsearch" - url "https://artifacts.elastic.co/downloads" - patternLayout { - artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]" + repos.ivy { IvyArtifactRepository repo -> + repo.name = 'elasticsearch' + repo.url = 'https://artifacts.elastic.co/downloads' + repo.patternLayout { IvyPatternRepositoryLayout layout -> + layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]' } // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats - credentials(HttpHeaderCredentials) { - name = "X-Elastic-No-KPI" - value = "1" - } - authentication { - header(HttpHeaderAuthentication) - } + repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds -> + creds.name = 'X-Elastic-No-KPI' + creds.value = '1' + } as Action) + repo.authentication.create('header', HttpHeaderAuthentication) } - repos.maven { - name "elastic" - url "https://artifacts.elastic.co/maven" + repos.maven { MavenArtifactRepository repo -> + repo.name = 'elastic' + repo.url = 'https://artifacts.elastic.co/maven' } String luceneVersion = VersionProperties.lucene if (luceneVersion.contains('-snapshot')) { // extract the revision number from the version with a regex matcher - String revision = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] - repos.maven { - name 'lucene-snapshots' - url "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + List matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List + String revision = matches.get(1) + repos.maven { MavenArtifactRepository repo -> + repo.name = 'lucene-snapshots' + repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index 8e38ccb747837..c0f40aa43579c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -39,6 +39,11 @@ public void print() { setGlobalProperties(); globalInfoListeners.forEach(Runnable::run); + + // Since all tasks depend on this task, and it always runs for every build, this makes sure that lifecycle tasks will still + // correctly report as UP-TO-DATE, since the convention is a lifecycle task (i.e. assemble, build, etc) will only be marked as + // UP-TO-DATE if all upstream tasks were also UP-TO-DATE. + setDidWork(false); } @InputFile @@ -61,11 +66,6 @@ public RegularFileProperty getFipsJvmFile() { return fipsJvmFile; } - @Internal - public List getGlobalInfoListeners() { - return globalInfoListeners; - } - public void setGlobalInfoListeners(List globalInfoListeners) { this.globalInfoListeners = globalInfoListeners; } diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index 7cbdfae5ed48c..b0ae65b344845 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -38,8 +38,10 @@ integTestCluster { setting 'reindex.ssl.truststore.password', 'password' // Workaround for JDK-8212885 - if (project.ext.runtimeJavaVersion.isJava12Compatible() == false) { - setting 'reindex.ssl.supported_protocols', 'TLSv1.2' + rootProject.globalInfo.ready { + if (project.ext.runtimeJavaVersion.isJava12Compatible() == false) { + setting 'reindex.ssl.supported_protocols', 'TLSv1.2' + } } extraConfigFile 'roles.yml', 'roles.yml' From 26cc00d5129e2cbe2b0af7888388f34a1b4f44ac Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 8 May 2019 14:05:14 -0700 Subject: [PATCH 20/32] Unused import --- .../org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index c0f40aa43579c..8da2ef39a0bf6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -7,7 +7,6 @@ import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.resources.TextResource; import org.gradle.api.tasks.InputFile; -import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.TaskAction; import javax.inject.Inject; From e3cb50af19a98030af0f6229418e7e7bbda6c94f Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 8 May 2019 14:08:53 -0700 Subject: [PATCH 21/32] Ensure fips jvm metadata is tracked as an output --- .../elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 1c367d6ba2f3d..0109b841bbd49 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -264,6 +264,7 @@ public RegularFileProperty getRuntimeVersionFile() { return runtimeVersionFile; } + @OutputFile public RegularFileProperty getFipsJvmFile() { return fipsJvmFile; } From 6255824c4a8f5604e4951382ed56b983ed4e1bff Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 8 May 2019 16:34:43 -0700 Subject: [PATCH 22/32] Sort out issue with cluster start/stop ordering. --- x-pack/plugin/ccr/qa/restart/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index 8501de714fae6..cace98d97b015 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -41,7 +41,7 @@ followClusterTestRunner { task followClusterRestartTest(type: RestIntegTestTask) {} followClusterRestartTestCluster { - dependsOn followClusterTestRunner + dependsOn followClusterTestRunner, 'followClusterTestCluster#stop' numNodes = 1 clusterName = 'follow-cluster' dataDir = { nodeNumber -> followClusterTest.nodes[0].dataDir } From db86526b121a59384887db2fb6dabdd5aa378109 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 9 May 2019 08:32:17 -0700 Subject: [PATCH 23/32] Read cluster settings at execution time --- .../gradle/test/ClusterFormationTasks.groovy | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 9acf3d79c1cac..6a90beb90f775 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -397,16 +397,17 @@ class ClusterFormationTasks { if (node.nodeVersion.major >= 7) { esConfig['indices.breaker.total.use_real_memory'] = false } - for (Map.Entry setting : node.config.settings) { - if (setting.value == null) { - esConfig.remove(setting.key) - } else { - esConfig.put(setting.key, setting.value) - } - } Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) writeConfig.doFirst { + for (Map.Entry setting : node.config.settings) { + if (setting.value == null) { + esConfig.remove(setting.key) + } else { + esConfig.put(setting.key, setting.value) + } + } + esConfig = configFilter.call(esConfig) File configFile = new File(node.pathConf, 'elasticsearch.yml') logger.info("Configuring ${configFile}") From acec279b1767fde8a987096ffd167d637c99b879 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 9 May 2019 08:48:44 -0700 Subject: [PATCH 24/32] Remove unnecessarily usages of runJavaAsScript() --- .../elasticsearch/gradle/BuildPlugin.groovy | 26 ------------------- .../sql/qa/security/with-ssl/build.gradle | 14 +++++----- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 05b66002ad62b..ed2e758b85be3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -300,32 +300,6 @@ class BuildPlugin implements Plugin { return javaVersions.find { it.version == version }.javaHome.absolutePath } - /** Runs the given javascript using jjs from the jdk, and returns the output */ - private static String runJavaAsScript(Project project, String javaHome, String script) { - ByteArrayOutputStream stdout = new ByteArrayOutputStream() - ByteArrayOutputStream stderr = new ByteArrayOutputStream() - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // gradle/groovy does not properly escape the double quote for windows - script = script.replace('"', '\\"') - } - File jrunscriptPath = new File(javaHome, 'bin/jrunscript') - ExecResult result = project.exec { ExecSpec spec -> - spec.executable = jrunscriptPath - spec.args '-e', script - spec.standardOutput = stdout - spec.errorOutput = stderr - spec.ignoreExitValue = true - } - if (result.exitValue != 0) { - project.logger.error("STDOUT:") - stdout.toString('UTF-8').eachLine { line -> project.logger.error(line) } - project.logger.error("STDERR:") - stderr.toString('UTF-8').eachLine { line -> project.logger.error(line) } - result.rethrowFailure() - } - return stdout.toString('UTF-8').trim() - } - /** Return the configuration name used for finding transitive deps of the given dependency. */ private static String transitiveDepConfigName(String groupId, String artifactId, String version) { return "_transitive_${groupId}_${artifactId}_${version}" diff --git a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle index de4e173463612..19459bade97a8 100644 --- a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle @@ -207,18 +207,16 @@ integTestCluster { return tmpFile.exists() } } -Closure notRunningFips = { - Boolean.parseBoolean(BuildPlugin.runJavaAsScript(project, project.runtimeJavaHome, - 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false -} // Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail. // TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores. // https://github.com/elastic/elasticsearch/issues/32306 -tasks.matching({ it.name == "integTestCluster#init" }).all { onlyIf notRunningFips } -tasks.matching({ it.name == "integTestCluster#start" }).all { onlyIf notRunningFips } -tasks.matching({ it.name == "integTestCluster#wait" }).all { onlyIf notRunningFips } -tasks.matching({ it.name == "integTestRunner" }).all { onlyIf notRunningFips } +tasks.matching { it.name in ["integTestCluster#init", "integTestCluster#start", "integTestCluster#wait", "integTestRunner"] }.all { + onlyIf { + project.inFipsJvm == false + } +} + /** A lazy evaluator to find the san to use for certificate generation. */ class SanEvaluator { From 3de9468f4105404f65d5e845fb4da9606334336b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 9 May 2019 16:13:12 -0700 Subject: [PATCH 25/32] This should be immutable --- .../main/java/org/elasticsearch/gradle/info/JavaHome.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java index 086860a25fcf6..29ca2bafc79dc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java @@ -26,18 +26,10 @@ public Integer getVersion() { return version; } - public void setVersion(Integer version) { - this.version = version; - } - @InputDirectory @Optional @PathSensitive(PathSensitivity.RELATIVE) public File getJavaHome() { return javaHome; } - - public void setJavaHome(File javaHome) { - this.javaHome = javaHome; - } } From 70e51488af44c69b1793c50fe44dff3f4e40469b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 9 May 2019 16:13:42 -0700 Subject: [PATCH 26/32] Remove unnecessary newline --- .../elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 0109b841bbd49..7acc06cc7c17b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -171,7 +171,6 @@ private String findJavaSpecificationVersion(File javaHome) { return runJavaAsScript(javaHome, versionScript); } - /** * Runs the given javascript using jjs from the jdk, and returns the output */ From 69952e24671fc8a981a20ccc2ad26fe2e1b13b48 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 21 May 2019 08:35:07 -0700 Subject: [PATCH 27/32] Deal with fallout from merging #41989 --- build.gradle | 22 ----------- .../elasticsearch/gradle/BuildPlugin.groovy | 37 +++++++++++++------ 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/build.gradle b/build.gradle index 0c4ea9b200097..7b496c857caa1 100644 --- a/build.gradle +++ b/build.gradle @@ -619,25 +619,3 @@ allprojects { } } } - -// wait until global info is populated because we don't know if we are running in a fips jvm until execution time -globalInfo.ready { - subprojects { - // Common config when running with a FIPS-140 runtime JVM - if (project.ext.has("inFipsJvm") && project.ext.inFipsJvm) { - tasks.withType(Test) { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' - } - project.pluginManager.withPlugin("elasticsearch.testclusters") { - project.testClusters.all { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' - } - } - } - } -} - - - diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3644fa06b55e6..6aa76307b1400 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -23,18 +23,20 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import org.apache.commons.io.IOUtils -import org.apache.tools.ant.taskdefs.condition.Os import org.eclipse.jgit.lib.Constants import org.eclipse.jgit.lib.RepositoryBuilder import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin +import org.elasticsearch.gradle.info.GlobalInfoExtension import org.elasticsearch.gradle.info.JavaHome import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.elasticsearch.gradle.precommit.PrecommitTasks import org.elasticsearch.gradle.test.ErrorReportingTestListener +import org.elasticsearch.gradle.testclusters.ElasticsearchCluster import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.InvalidUserDataException import org.gradle.api.JavaVersion +import org.gradle.api.NamedDomainObjectContainer import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Task @@ -137,20 +139,31 @@ class BuildPlugin implements Plugin { configureDependenciesInfo(project) // Common config when running with a FIPS-140 runtime JVM - // Need to do it here to support external plugins - if (project.ext.inFipsJvm) { - project.tasks.withType(Test) { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' - } - project.pluginManager.withPlugin("elasticsearch.testclusters") { - project.testClusters.all { - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' + // Need to do it here to support external plugins + if (project == project.rootProject) { + GlobalInfoExtension globalInfo = project.extensions.getByType(GlobalInfoExtension) + + // wait until global info is populated because we don't know if we are running in a fips jvm until execution time + globalInfo.ready { + project.subprojects { Project subproject -> + ExtraPropertiesExtension ext = subproject.extensions.getByType(ExtraPropertiesExtension) + // Common config when running with a FIPS-140 runtime JVM + if (ext.has('inFipsJvm') && ext.get('inFipsJvm')) { + subproject.tasks.withType(Test) { Test task -> + task.systemProperty 'javax.net.ssl.trustStorePassword', 'password' + task.systemProperty 'javax.net.ssl.keyStorePassword', 'password' + } + project.pluginManager.withPlugin("elasticsearch.testclusters") { + NamedDomainObjectContainer testClusters = subproject.extensions.getByName('testClusters') as NamedDomainObjectContainer + testClusters.all { ElasticsearchCluster cluster -> + cluster.systemProperty 'javax.net.ssl.trustStorePassword', 'password' + cluster.systemProperty 'javax.net.ssl.keyStorePassword', 'password' + } + } + } } } } - } static void requireDocker(final Task task) { From f7929ae83a94a542021f5348f29b83ed89d413c9 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 22 May 2019 09:10:57 -0700 Subject: [PATCH 28/32] Address review feedback --- .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 6aa76307b1400..3a058ca9310df 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -317,15 +317,14 @@ class BuildPlugin implements Plugin { throw new GradleException("JAVA${version}_HOME required to run task:\n${task}") } } else { - Map> requiredJavaVersions = (Map>) ext.get('requiredJavaVersions') - requiredJavaVersions.getOrDefault(version, []).add(task) + (ext.get('requiredJavaVersions') as Map>).getOrDefault(version, []).add(task) } } /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ static String getJavaHome(final Task task, final int version) { requireJavaHome(task, version) - def javaVersions = task.project.property('javaVersions') as List + List javaVersions = task.project.property('javaVersions') as List return javaVersions.find { it.version == version }.javaHome.absolutePath } @@ -803,7 +802,7 @@ class BuildPlugin implements Plugin { project.mkdir(test.workingDir) if (project.property('inFipsJvm')) { - nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()} FIPS") + nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()}FIPS") } else { nonInputProperties.systemProperty('runtime.java', "${-> (ext.get('runtimeJavaVersion') as JavaVersion).getMajorVersion()}") } From 1104f5228a9cf2ae7feb7be1771eb53f302be622 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 22 May 2019 09:31:34 -0700 Subject: [PATCH 29/32] Keep java 8 compatibility for build-tools --- buildSrc/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 53ffef91af745..f239427330c58 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -70,8 +70,9 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) { throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools') } -targetCompatibility = '11' -sourceCompatibility = '11' +// Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11 +targetCompatibility = '8' +sourceCompatibility = '8' /***************************************************************************** * Dependencies used by the entire build * From 1eafd8c4ecbcad88542020d6c0828f3a6fda2772 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 22 May 2019 11:16:30 -0700 Subject: [PATCH 30/32] Reorder class members --- .../info/GenerateGlobalBuildInfoTask.java | 134 +++++++++--------- .../gradle/info/PrintGlobalBuildInfoTask.java | 34 ++--- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 7acc06cc7c17b..3a2180b564e8b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -51,6 +51,73 @@ public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) { this.fipsJvmFile = objectFactory.fileProperty(); } + @Input + public JavaVersion getMinimumCompilerVersion() { + return minimumCompilerVersion; + } + + public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { + this.minimumCompilerVersion = minimumCompilerVersion; + } + + @Input + public JavaVersion getMinimumRuntimeVersion() { + return minimumRuntimeVersion; + } + + public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { + this.minimumRuntimeVersion = minimumRuntimeVersion; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getCompilerJavaHome() { + return compilerJavaHome; + } + + public void setCompilerJavaHome(File compilerJavaHome) { + this.compilerJavaHome = compilerJavaHome; + } + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + public File getRuntimeJavaHome() { + return runtimeJavaHome; + } + + public void setRuntimeJavaHome(File runtimeJavaHome) { + this.runtimeJavaHome = runtimeJavaHome; + } + + @Nested + public List getJavaVersions() { + return javaVersions; + } + + public void setJavaVersions(List javaVersions) { + this.javaVersions = javaVersions; + } + + @OutputFile + public RegularFileProperty getOutputFile() { + return outputFile; + } + + @OutputFile + public RegularFileProperty getCompilerVersionFile() { + return compilerVersionFile; + } + + @OutputFile + public RegularFileProperty getRuntimeVersionFile() { + return runtimeVersionFile; + } + + @OutputFile + public RegularFileProperty getFipsJvmFile() { + return fipsJvmFile; + } + @TaskAction public void generate() { String javaVendor = System.getProperty("java.vendor"); @@ -200,71 +267,4 @@ private String runJavaAsScript(File javaHome, String script) { } return stdout.toString(UTF_8).trim(); } - - @Input - public JavaVersion getMinimumCompilerVersion() { - return minimumCompilerVersion; - } - - public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { - this.minimumCompilerVersion = minimumCompilerVersion; - } - - @Input - public JavaVersion getMinimumRuntimeVersion() { - return minimumRuntimeVersion; - } - - public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { - this.minimumRuntimeVersion = minimumRuntimeVersion; - } - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - public File getCompilerJavaHome() { - return compilerJavaHome; - } - - public void setCompilerJavaHome(File compilerJavaHome) { - this.compilerJavaHome = compilerJavaHome; - } - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - public File getRuntimeJavaHome() { - return runtimeJavaHome; - } - - public void setRuntimeJavaHome(File runtimeJavaHome) { - this.runtimeJavaHome = runtimeJavaHome; - } - - @Nested - public List getJavaVersions() { - return javaVersions; - } - - public void setJavaVersions(List javaVersions) { - this.javaVersions = javaVersions; - } - - @OutputFile - public RegularFileProperty getOutputFile() { - return outputFile; - } - - @OutputFile - public RegularFileProperty getCompilerVersionFile() { - return compilerVersionFile; - } - - @OutputFile - public RegularFileProperty getRuntimeVersionFile() { - return runtimeVersionFile; - } - - @OutputFile - public RegularFileProperty getFipsJvmFile() { - return fipsJvmFile; - } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index 8da2ef39a0bf6..b83fe29b073a6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -28,23 +28,6 @@ public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) { this.fipsJvmFile = objectFactory.fileProperty(); } - @TaskAction - public void print() { - getLogger().quiet("======================================="); - getLogger().quiet("Elasticsearch Build Hamster says Hello!"); - getLogger().quiet(getFileText(getBuildInfoFile()).asString()); - getLogger().quiet(" Random Testing Seed : " + getProject().property("testSeed")); - getLogger().quiet("======================================="); - - setGlobalProperties(); - globalInfoListeners.forEach(Runnable::run); - - // Since all tasks depend on this task, and it always runs for every build, this makes sure that lifecycle tasks will still - // correctly report as UP-TO-DATE, since the convention is a lifecycle task (i.e. assemble, build, etc) will only be marked as - // UP-TO-DATE if all upstream tasks were also UP-TO-DATE. - setDidWork(false); - } - @InputFile public RegularFileProperty getBuildInfoFile() { return buildInfoFile; @@ -69,6 +52,23 @@ public void setGlobalInfoListeners(List globalInfoListeners) { this.globalInfoListeners = globalInfoListeners; } + @TaskAction + public void print() { + getLogger().quiet("======================================="); + getLogger().quiet("Elasticsearch Build Hamster says Hello!"); + getLogger().quiet(getFileText(getBuildInfoFile()).asString()); + getLogger().quiet(" Random Testing Seed : " + getProject().property("testSeed")); + getLogger().quiet("======================================="); + + setGlobalProperties(); + globalInfoListeners.forEach(Runnable::run); + + // Since all tasks depend on this task, and it always runs for every build, this makes sure that lifecycle tasks will still + // correctly report as UP-TO-DATE, since the convention is a lifecycle task (i.e. assemble, build, etc) will only be marked as + // UP-TO-DATE if all upstream tasks were also UP-TO-DATE. + setDidWork(false); + } + private TextResource getFileText(RegularFileProperty regularFileProperty) { return getProject().getResources().getText().fromFile(regularFileProperty.getAsFile().get()); } From c9299c15c95e671138dd8350e29f17e3f0fdd1ae Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 22 May 2019 11:17:37 -0700 Subject: [PATCH 31/32] Remove intermediary variable --- .../org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 192bb71861f0f..f0f34e84261c6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -113,8 +113,7 @@ private static File findRuntimeJavaHome(final File compilerJavaHome) { } private static String findJavaHome(String version) { - String versionedVarName = getJavaHomeEnvVarName(version); - String versionedJavaHome = System.getenv(versionedVarName); + String versionedJavaHome = System.getenv(getJavaHomeEnvVarName(version)); if (versionedJavaHome == null) { throw new GradleException( "$versionedVarName must be set to build Elasticsearch. " + From 8cc01e4af26e65156935b544f10767a3656b311c Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 22 May 2019 11:22:02 -0700 Subject: [PATCH 32/32] Check for existence of java home directories --- .../info/GenerateGlobalBuildInfoTask.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 3a2180b564e8b..8537775ee129b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -134,19 +134,25 @@ public void generate() { try { if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { - compilerJavaVersionDetails = findJavaVersionDetails(compilerJavaHome); - compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(compilerJavaHome)); + if (compilerJavaHome.exists()) { + compilerJavaVersionDetails = findJavaVersionDetails(compilerJavaHome); + compilerJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(compilerJavaHome)); + } else { + throw new RuntimeException("Compiler Java home path of '" + compilerJavaHome + "' does not exist"); + } } if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { - runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome); - runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome)); - } - - if (Files.isSameFile(runtimeJavaHome.toPath(), gradleJavaHome.toPath()) == false) { - // We don't expect Gradle to be running in a FIPS JVM - String inFipsJvmScript = "print(java.security.Security.getProviders()[0].name.toLowerCase().contains(\"fips\"));"; - inFipsJvm = Boolean.parseBoolean(runJavaAsScript(runtimeJavaHome, inFipsJvmScript)); + if (runtimeJavaHome.exists()) { + runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome); + runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome)); + + // We don't expect Gradle to be running in a FIPS JVM + String inFipsJvmScript = "print(java.security.Security.getProviders()[0].name.toLowerCase().contains(\"fips\"));"; + inFipsJvm = Boolean.parseBoolean(runJavaAsScript(runtimeJavaHome, inFipsJvmScript)); + } else { + throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist"); + } } } catch (IOException e) { throw new UncheckedIOException(e);