From 2c02e6c02d17978b18b3273530923c6f175da817 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 14 May 2018 12:00:40 -0700 Subject: [PATCH 1/4] Make xpack modules instead of a meta plugin This commit removes xpack from being a meta-plugin-as-a-module. It also fixes a couple tests which were missing task dependencies, which failed once the gradle execution order changed. --- distribution/build.gradle | 11 ++++------- x-pack/plugin/build.gradle | 10 ++-------- x-pack/plugin/core/src/main/bin/x-pack-env | 2 +- x-pack/plugin/core/src/main/bin/x-pack-env.bat | 2 +- .../plugin/security/src/main/bin/x-pack-security-env | 2 +- .../security/src/main/bin/x-pack-security-env.bat | 2 +- x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env | 2 +- .../watcher/src/main/bin/x-pack-watcher-env.bat | 2 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 10 +++------- x-pack/qa/sql/security/ssl/build.gradle | 11 +++++------ 10 files changed, 20 insertions(+), 34 deletions(-) diff --git a/distribution/build.gradle b/distribution/build.gradle index 266cb8f8b270a..d2e2810bc7eec 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -201,17 +201,14 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { // use licenses from each of the bundled xpack plugins Project xpack = project(':x-pack:plugin') -xpack.subprojects.findAll { it.name != 'bwc' }.each { Project xpackSubproject -> - File licenses = new File(xpackSubproject.projectDir, 'licenses') +xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule -> + File licenses = new File(xpackModule.projectDir, 'licenses') if (licenses.exists()) { buildDefaultNotice.licensesDir licenses } + copyModule(processDefaultOutputs, xpackModule) + copyLog4jProperties(buildDefaultLog4jConfig, xpackModule) } -// but copy just the top level meta plugin to the default modules -copyModule(processDefaultOutputs, xpack) -copyLog4jProperties(buildDefaultLog4jConfig, xpack) - -// // make sure we have a clean task since we aren't a java project, but we have tasks that // put stuff in the build dir diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index e4dc314eb72a7..4a0b29c42582a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -8,17 +8,11 @@ import java.nio.file.Path import java.nio.file.StandardCopyOption import org.elasticsearch.gradle.test.RunTask; -apply plugin: 'elasticsearch.es-meta-plugin' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' archivesBaseName = 'x-pack' -es_meta_plugin { - name = 'x-pack' - description = 'Elasticsearch Expanded Pack Plugin' - plugins = ['core', 'deprecation', 'graph', 'logstash', - 'ml', 'monitoring', 'security', 'upgrade', 'watcher', 'sql', 'rollup'] -} - dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') } diff --git a/x-pack/plugin/core/src/main/bin/x-pack-env b/x-pack/plugin/core/src/main/bin/x-pack-env index fb5489cfebc43..cc25b86f69873 100644 --- a/x-pack/plugin/core/src/main/bin/x-pack-env +++ b/x-pack/plugin/core/src/main/bin/x-pack-env @@ -5,4 +5,4 @@ # you may not use this file except in compliance with the Elastic License. # include x-pack-core jars in classpath -ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-core/*" +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-core/*" diff --git a/x-pack/plugin/core/src/main/bin/x-pack-env.bat b/x-pack/plugin/core/src/main/bin/x-pack-env.bat index de45a53c9269c..fc97721a737d1 100644 --- a/x-pack/plugin/core/src/main/bin/x-pack-env.bat +++ b/x-pack/plugin/core/src/main/bin/x-pack-env.bat @@ -2,4 +2,4 @@ rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one rem or more contributor license agreements. Licensed under the Elastic License; rem you may not use this file except in compliance with the Elastic License. -set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-core/* +set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-core/* diff --git a/x-pack/plugin/security/src/main/bin/x-pack-security-env b/x-pack/plugin/security/src/main/bin/x-pack-security-env index fd35535be8cca..3a2b15e13fa4a 100644 --- a/x-pack/plugin/security/src/main/bin/x-pack-security-env +++ b/x-pack/plugin/security/src/main/bin/x-pack-security-env @@ -7,4 +7,4 @@ source "`dirname "$0"`"/x-pack-env # include x-pack-security jars in classpath -ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-security/*" +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-security/*" diff --git a/x-pack/plugin/security/src/main/bin/x-pack-security-env.bat b/x-pack/plugin/security/src/main/bin/x-pack-security-env.bat index 610f5835d28c2..035f1c965ffb6 100644 --- a/x-pack/plugin/security/src/main/bin/x-pack-security-env.bat +++ b/x-pack/plugin/security/src/main/bin/x-pack-security-env.bat @@ -4,4 +4,4 @@ rem you may not use this file except in compliance with the Elastic License. call "%~dp0x-pack-env.bat" || exit /b 1 -set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-security/* +set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-security/* diff --git a/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env b/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env index 4abe3d8c60761..13718a01b4330 100644 --- a/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env +++ b/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env @@ -7,4 +7,4 @@ source "`dirname "$0"`"/x-pack-env # include x-pack-security jars in classpath -ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-watcher/*" +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-watcher/*" diff --git a/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env.bat b/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env.bat index 9e43ffaa0521f..010c154eb5a39 100644 --- a/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env.bat +++ b/x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env.bat @@ -4,4 +4,4 @@ rem you may not use this file except in compliance with the Elastic License. call "%~dp0x-pack-env.bat" || exit /b 1 -set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-watcher/* +set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-watcher/* diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index 0a9fc79d4836b..887430f833758 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -105,7 +105,7 @@ task exportNodeCertificate(type: LoggedExec) { // Import the node certificate in the client's keystore task importNodeCertificateInClientKeyStore(type: LoggedExec) { - dependsOn exportNodeCertificate + dependsOn createClientKeyStore, exportNodeCertificate executable = new File(project.runtimeJavaHome, 'bin/keytool') args '-import', '-alias', 'test-node', @@ -137,7 +137,7 @@ task exportClientCertificate(type: LoggedExec) { // Import the client certificate in the node's keystore task importClientCertificateInNodeKeyStore(type: LoggedExec) { - dependsOn exportClientCertificate + dependsOn exportClientCertificate, createNodeKeyStore executable = new File(project.runtimeJavaHome, 'bin/keytool') args '-import', '-alias', 'test-client', @@ -153,14 +153,10 @@ forbiddenPatterns { // Add keystores to test classpath: it expects it there sourceSets.test.resources.srcDir(keystoreDir) -processTestResources.dependsOn( - createNodeKeyStore, createClientKeyStore, - importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore -) +processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore) integTestCluster.dependsOn(importClientCertificateInNodeKeyStore, importNodeCertificateInClientKeyStore) - ext.pluginsCount = 0 project(':plugins').getChildProjects().each { pluginName, pluginProject -> // need to get a non-decorated project object, so must re-lookup the project by path diff --git a/x-pack/qa/sql/security/ssl/build.gradle b/x-pack/qa/sql/security/ssl/build.gradle index 8c19ba0303f78..fe8aaeaff2b64 100644 --- a/x-pack/qa/sql/security/ssl/build.gradle +++ b/x-pack/qa/sql/security/ssl/build.gradle @@ -74,6 +74,7 @@ task createClientKeyStore(type: LoggedExec) { // Export the node's certificate File nodeCertificate = new File(keystoreDir, 'test-node.cert') task exportNodeCertificate(type: LoggedExec) { + dependsOn createNodeKeyStore doFirst { if (nodeCertificate.parentFile.exists() == false) { nodeCertificate.parentFile.mkdirs() @@ -92,7 +93,7 @@ task exportNodeCertificate(type: LoggedExec) { // Import the node certificate in the client's keystore task importNodeCertificateInClientKeyStore(type: LoggedExec) { - dependsOn exportNodeCertificate + dependsOn createClientKeyStore, exportNodeCertificate executable = new File(project.runtimeJavaHome, 'bin/keytool') args '-import', '-alias', 'test-node', @@ -105,6 +106,7 @@ task importNodeCertificateInClientKeyStore(type: LoggedExec) { // Export the client's certificate File clientCertificate = new File(keystoreDir, 'test-client.cert') task exportClientCertificate(type: LoggedExec) { + dependsOn createClientKeyStore doFirst { if (clientCertificate.parentFile.exists() == false) { clientCertificate.parentFile.mkdirs() @@ -123,7 +125,7 @@ task exportClientCertificate(type: LoggedExec) { // Import the client certificate in the node's keystore task importClientCertificateInNodeKeyStore(type: LoggedExec) { - dependsOn exportClientCertificate + dependsOn createNodeKeyStore, exportClientCertificate executable = new File(project.runtimeJavaHome, 'bin/keytool') args '-import', '-alias', 'test-client', @@ -139,10 +141,7 @@ forbiddenPatterns { // Add keystores to test classpath: it expects it there sourceSets.test.resources.srcDir(keystoreDir) -processTestResources.dependsOn( - createNodeKeyStore, createClientKeyStore, - importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore -) +processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore) integTestCluster.dependsOn(importClientCertificateInNodeKeyStore) From 5d855015a2ae4a7bfcb80839964e776f6117fd00 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 15 May 2018 13:33:02 -0700 Subject: [PATCH 2/4] remove unnecessary meta plugin configuration for packaging tests --- x-pack/qa/vagrant/build.gradle | 35 ---------------------------------- 1 file changed, 35 deletions(-) diff --git a/x-pack/qa/vagrant/build.gradle b/x-pack/qa/vagrant/build.gradle index 0c3428f258c0e..c69214578fd16 100644 --- a/x-pack/qa/vagrant/build.gradle +++ b/x-pack/qa/vagrant/build.gradle @@ -11,41 +11,6 @@ esvagrant { } dependencies { - // Packaging tests use the x-pack meta plugin - packaging project(path: xpackProject('plugin').path, configuration: 'zip') - // Inherit Bats test utils from :qa:vagrant project packaging project(path: ':qa:vagrant', configuration: 'packaging') } - -Map> metaPlugins = [:] -for (Project metaPlugin : project.rootProject.subprojects) { - if (metaPlugin.plugins.hasPlugin(MetaPluginBuildPlugin)) { - MetaPluginPropertiesExtension extension = metaPlugin.extensions.findByName('es_meta_plugin') - if (extension != null) { - List plugins = [] - metaPlugin.subprojects.each { - if (extension.plugins.contains(it.name)) { - Project plugin = (Project) it - if (plugin.plugins.hasPlugin(PluginBuildPlugin)) { - PluginPropertiesExtension esplugin = plugin.extensions.findByName('esplugin') - if (esplugin != null) { - plugins.add(esplugin.name) - } - } - } - } - metaPlugins.put(extension.name, plugins.toSorted()) - } - } -} - -setupPackagingTest { - doLast { - metaPlugins.each{ name, plugins -> - File expectedMetaPlugins = file("build/plugins/${name}.expected") - expectedMetaPlugins.parentFile.mkdirs() - expectedMetaPlugins.setText(plugins.join('\n'), 'UTF-8') - } - } -} From ab1b2a0ac136a793289a3e14041f5eda5ab7f821 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 15 May 2018 14:29:08 -0700 Subject: [PATCH 3/4] remove meta plugin check --- .../src/test/resources/packaging/utils/xpack.bash | 9 --------- 1 file changed, 9 deletions(-) diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash b/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash index 95ab2a08d3e57..3e44ee9f83a58 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash +++ b/x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash @@ -69,15 +69,6 @@ verify_xpack_installation() { done # nocommit: decide whether to check the files added by the distribution, not part of xpack... #assert_number_of_files "$ESCONFIG/" $configFilesCount - - # Read the $name.expected file that contains all the expected - # plugins for the meta plugin - while read plugin; do - assert_module_or_plugin_directory "$ESMODULES/$name/$plugin" - assert_file_exist "$ESMODULES/$name/$plugin/$plugin"*".jar" - assert_file_exist "$ESMODULES/$name/$plugin/plugin-descriptor.properties" - assert_file_exist "$ESMODULES/$name/$plugin/plugin-security.policy" - done Date: Wed, 16 May 2018 10:51:21 -0700 Subject: [PATCH 4/4] fix ml license check --- distribution/archives/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 9fa06021236a2..5d1703399aad4 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -224,7 +224,7 @@ subprojects { doLast { // this is just a small sample from the C++ notices, the idea being that if we've added these lines we've probably added all the required lines final List expectedLines = Arrays.asList("Apache log4cxx", "Boost Software License - Version 1.0 - August 17th, 2003") - final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack/x-pack-ml/NOTICE.txt") + final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack-ml/NOTICE.txt") final List actualLines = Files.readAllLines(noticePath) for (final String expectedLine : expectedLines) { if (actualLines.contains(expectedLine) == false) {