From 5cb970a1e808ff20b60f3f1b360cd8593670cbf3 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 20 Jul 2018 14:12:25 +0300 Subject: [PATCH 1/8] RFC: Test that example plugins build stand-alone The initial goal was to write a test to make sure that build-tools can be used by plugin authors outside of our build. Example projects seemed like a good fit rather than writing dummy ones just for the test. - Add an integration test based on testKit to build the example plugins like stand alone projects - the tests injects the snapshots repo right now - an alternative would be to inject the last release version to cause dependency resolution against that. It wouldn't be 100% realistic as we would build using old dependencies and current build-tools which never happens in practice, but it would result in fewer moving parts. - the snapshots does cover more ground at the cost of some confusion as these are delayed from the current build, so any new issues will introduced will surface in the next build only. - there are also some changes for making it easier to copy these examples, but it won't quite work yet, e.x. applying the plugin only works because of testKit. Applying the project like this won't work. It might work if we add a version number and publish it to the Gradle plugin portal. - testKit can have the gradle version configured, so it would be easy to add similar tests for different Gradle versions. - The tests are failing due to 2 reasons: - elasticsearch-nio is not published. I am looking to fix this. - `painless-whitelist` references `project(':modules:lang-painless')` so this isn't really an example that will work stand-alone. Not sure what we should do about this one? Not test it or publish the plugin jar ? --- buildSrc/build.gradle | 7 + .../gradle/plugin/PluginBuildPlugin.groovy | 2 - .../plugin/PluginPropertiesExtension.groovy | 25 +++- .../gradle/plugin/PluginPropertiesTask.groovy | 1 - .../gradle/test/RestIntegTestTask.groovy | 11 +- .../gradle/BuildExamplePluginsIT.java | 124 ++++++++++++++++++ plugins/examples/custom-settings/build.gradle | 10 +- .../examples/painless-whitelist/build.gradle | 7 +- plugins/examples/rescore/build.gradle | 8 +- plugins/examples/rest-handler/build.gradle | 9 +- .../script-expert-scoring/build.gradle | 8 +- 11 files changed, 192 insertions(+), 20 deletions(-) create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 55520728812c9..7e082ba466809 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -166,6 +166,13 @@ if (project != rootProject) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath inputs.dir(file("src/testKit")) + // tell BuildExamplePluginsIT where to find the example plugins + systemProperty ( + 'test.build-tools.plugin.examples', + files( + project(':example-plugins').subprojects.collect { it.projectDir } + ).asPath + ) } check.dependsOn(integTest) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 7f6f337e8a906..42c5230bdbdae 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -25,7 +25,6 @@ import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.test.RunTask import org.gradle.api.InvalidUserDataException -import org.gradle.api.JavaVersion import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.XmlProvider @@ -39,7 +38,6 @@ import java.nio.file.Path import java.nio.file.StandardCopyOption import java.util.regex.Matcher import java.util.regex.Pattern - /** * Encapsulates build configuration for an Elasticsearch plugin. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy index 6cfe44c806833..ce9f9d2e4eb64 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -56,17 +56,38 @@ class PluginPropertiesExtension { /** A license file that should be included in the built plugin zip. */ @Input - File licenseFile = null + private File licenseFile = null /** * A notice file that should be included in the built plugin zip. This will be * extended with notices from the {@code licenses/} directory. */ @Input - File noticeFile = null + private File noticeFile = null + + Project project = null PluginPropertiesExtension(Project project) { name = project.name version = project.version + this.project = project + } + + File getLicenseFile() { + return licenseFile + } + + void setLicenseFile(File licenseFile) { + project.ext.licenseFile = licenseFile + this.licenseFile = licenseFile + } + + File getNoticeFile() { + return noticeFile + } + + void setNoticeFile(File noticeFile) { + project.ext.noticeFile = noticeFile + this.noticeFile = noticeFile } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy index 8e913153f05ad..9588f77a71db7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -23,7 +23,6 @@ import org.gradle.api.InvalidUserDataException import org.gradle.api.Task import org.gradle.api.tasks.Copy import org.gradle.api.tasks.OutputFile - /** * Creates a plugin descriptor. */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index d2101c48aabdc..2838849981a1b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -31,6 +31,7 @@ import org.gradle.api.provider.Provider import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Input import org.gradle.api.tasks.TaskState +import org.gradle.plugins.ide.idea.IdeaPlugin import java.nio.charset.StandardCharsets import java.nio.file.Files @@ -243,10 +244,12 @@ public class RestIntegTestTask extends DefaultTask { } } } - project.idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) + if (project.plugins.hasPlugin(IdeaPlugin)) { + project.idea { + module { + if (scopes.TEST != null) { + scopes.TEST.plus.add(project.configurations.restSpec) + } } } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java new file mode 100644 index 0000000000000..c27bb88dbbdc2 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle; + +import org.apache.commons.io.FileUtils; +import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.gradle.testkit.runner.GradleRunner; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BuildExamplePluginsIT extends GradleIntegrationTestCase { + + private static List EXTERNAL_PROJECTS = Arrays.stream( + Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples")) + .split(File.pathSeparator) + ).map(File::new).collect(Collectors.toList()); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @BeforeClass + public static void assertProjectsExist() { + List existing = EXTERNAL_PROJECTS.stream().filter(File::exists).collect(Collectors.toList()); + assertEquals(EXTERNAL_PROJECTS, existing); + } + + @AfterClass + public static void assertEverythingTested() { + assertEquals( + "Some example plugins are not tested: " + EXTERNAL_PROJECTS, + 0, EXTERNAL_PROJECTS.size() + ); + } + + public void testCustomSettings() throws IOException { + runAndRemove("custom-settings"); + } + + public void testPainlessWhitelist() throws IOException { + runAndRemove("painless-whitelist"); + } + + public void testRescore() throws IOException { + runAndRemove("rescore"); + } + + public void testRestHandler() throws IOException { + runAndRemove("rest-handler"); + } + + public void testScriptExpertScoring() throws IOException { + runAndRemove("script-expert-scoring"); + } + + private void runAndRemove(String name) throws IOException { + List matches = EXTERNAL_PROJECTS.stream().filter(each -> each.getAbsolutePath().contains(name)) + .collect(Collectors.toList()); + assertEquals( + "Expected a single project folder to match `" + name + "` but got: " + matches, + 1, matches.size() + ); + + EXTERNAL_PROJECTS.remove(matches.get(0)); + + FileUtils.copyDirectory(matches.get(0), tmpDir.getRoot()); + // just get rid of deprecation warnings from now + Files.write( + tmpDir.newFile("settings.gradle").toPath(), "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) + ); + // Add a repositories section to be able to resolve from snapshots. + // !NOTE! that the plugin build will use be using stale artifacts, not the ones produced in the current build + Files.write( + new File(tmpDir.getRoot(), "build.gradle").toPath(), + ("\n" + + "repositories {\n" + + " maven {\n" + + " url \"https://snapshots.elastic.co/maven\"\n" + + " mavenLocal()\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8), + StandardOpenOption.APPEND + ); + Files.write( + tmpDir.newFile("NOTICE.txt").toPath(), + "dummy test nottice".getBytes(StandardCharsets.UTF_8) + ); + + GradleRunner.create() + // Todo make a copy, write a settings file enabling stable publishing + .withProjectDir(tmpDir.getRoot()) + .withArguments("clean", "check", "-s", "-i", "--warning-mode=all") + .withPluginClasspath() + .build(); + } + +} diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index e0e728cec2427..a69d51ca9f85e 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -17,15 +17,21 @@ * under the License. */ -apply plugin: 'elasticsearch.esplugin' +plugins { + id 'elasticsearch.esplugin' +} esplugin { name 'custom-settings' description 'An example plugin showing how to register custom settings' classname 'org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } integTestCluster { // Adds a setting in the Elasticsearch keystore before running the integration tests keystoreSetting 'custom.secured', 'password' -} \ No newline at end of file +} + + diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index ef1ca7d741e9a..8a3173e09997c 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -16,14 +16,17 @@ * specific language governing permissions and limitations * under the License. */ - -apply plugin: 'elasticsearch.esplugin' +plugins { + id 'elasticsearch.esplugin' +} esplugin { name 'painless-whitelist' description 'An example whitelisting additional classes and methods in painless' classname 'org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin' extendedPlugins = ['lang-painless'] + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index 4adeb0c721baf..b6b101794c6e5 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -16,11 +16,15 @@ * specific language governing permissions and limitations * under the License. */ - -apply plugin: 'elasticsearch.esplugin' +plugins { + id 'elasticsearch.esplugin' +} esplugin { name 'example-rescore' description 'An example plugin implementing rescore and verifying that plugins *can* implement rescore' classname 'org.elasticsearch.example.rescore.ExampleRescorePlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } + diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index cfe84e6a45a93..deea35aa2c8be 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -16,13 +16,16 @@ * specific language governing permissions and limitations * under the License. */ - -apply plugin: 'elasticsearch.esplugin' +plugins { + id 'elasticsearch.esplugin' +} esplugin { name 'rest-handler' description 'An example plugin showing how to register a REST handler' classname 'org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } // No unit tests in this example @@ -40,4 +43,4 @@ integTestCluster { } integTestRunner { systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }" -} +} \ No newline at end of file diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index 7c602d9bc027d..78c7d0de2f286 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -16,13 +16,17 @@ * specific language governing permissions and limitations * under the License. */ - -apply plugin: 'elasticsearch.esplugin' +plugins { + id 'elasticsearch.esplugin' +} esplugin { name 'script-expert-scoring' description 'An example script engine to use low level Lucene internals for expert scoring' classname 'org.elasticsearch.example.expertscript.ExpertScriptPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') } test.enabled = false + From fa25aee82a60d60b5760979c3b959cc6ea230298 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Sat, 21 Jul 2018 15:21:59 +0300 Subject: [PATCH 2/8] Fix task up-to-date annotations --- .../gradle/plugin/PluginPropertiesExtension.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy index ce9f9d2e4eb64..c250d7695a832 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -20,6 +20,7 @@ package org.elasticsearch.gradle.plugin import org.gradle.api.Project import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile /** * A container for plugin properties that will be written to the plugin descriptor, for easy @@ -55,14 +56,12 @@ class PluginPropertiesExtension { boolean requiresKeystore = false /** A license file that should be included in the built plugin zip. */ - @Input private File licenseFile = null /** * A notice file that should be included in the built plugin zip. This will be * extended with notices from the {@code licenses/} directory. */ - @Input private File noticeFile = null Project project = null @@ -73,6 +72,7 @@ class PluginPropertiesExtension { this.project = project } + @InputFile File getLicenseFile() { return licenseFile } @@ -82,6 +82,7 @@ class PluginPropertiesExtension { this.licenseFile = licenseFile } + @InputFile File getNoticeFile() { return noticeFile } From b0213f4d6b51a348ea3380803aee84c7d5a01ba2 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 7 Aug 2018 10:04:01 +0300 Subject: [PATCH 3/8] Add lang-painless-spi, luceene snapshot repo --- build.gradle | 1 + .../org/elasticsearch/gradle/BuildExamplePluginsIT.java | 9 +++++++-- plugins/examples/painless-whitelist/build.gradle | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 66f34d8f445de..b39d9e170d906 100644 --- a/build.gradle +++ b/build.gradle @@ -232,6 +232,7 @@ subprojects { "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level', "org.elasticsearch.client:test:${version}": ':client:test', "org.elasticsearch.client:transport:${version}": ':client:transport', + "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip', diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index c27bb88dbbdc2..c161cecdb1b6b 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -97,14 +97,19 @@ private void runAndRemove(String name) throws IOException { ); // Add a repositories section to be able to resolve from snapshots. // !NOTE! that the plugin build will use be using stale artifacts, not the ones produced in the current build + // TODO: Add "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + // (revision = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]) Files.write( new File(tmpDir.getRoot(), "build.gradle").toPath(), ("\n" + "repositories {\n" + " maven {\n" + " url \"https://snapshots.elastic.co/maven\"\n" + - " mavenLocal()\n" + " }\n" + + " maven {\n" + + " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/608f0277b0\"\n" + // FIXME + " }\n" + + " mavenLocal()\n" + // FIXME "}\n").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND ); @@ -116,7 +121,7 @@ private void runAndRemove(String name) throws IOException { GradleRunner.create() // Todo make a copy, write a settings file enabling stable publishing .withProjectDir(tmpDir.getRoot()) - .withArguments("clean", "check", "-s", "-i", "--warning-mode=all") + .withArguments("clean", "check", "-s", "-i", "--warning-mode=all", "--scan") .withPluginClasspath() .build(); } diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 8a3173e09997c..9c50ecc816f4f 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -30,7 +30,7 @@ esplugin { } dependencies { - compileOnly project(':modules:lang-painless') + compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}" } if (System.getProperty('tests.distribution') == null) { From 1356199f29a7c027958fa688f78624bfb4c429aa Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 7 Aug 2018 15:37:52 +0300 Subject: [PATCH 4/8] Create a `publishNebulaPublicationToLocalTestRepository` task Will publish everything that has publishing configured to a local repo --- build.gradle | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build.gradle b/build.gradle index f215c21a55efb..84b0e9564053b 100644 --- a/build.gradle +++ b/build.gradle @@ -87,8 +87,23 @@ subprojects { } } } + repositories { + maven { + name = 'localTest' + url = "$rootProject.buildDir/local-test-repo" + } + } } } + // Make nebula and shadow play nice together, Gradle doesn't like it how one jar task replaces the output of another + // if both are part of published artifacts + project.afterEvaluate { + if (plugins.hasPlugin(MavenPublishPlugin) && plugins.hasPlugin(ShadowPlugin)) { + project.publishing.publications.nebula.artifacts = [project.tasks.shadowJar] + } + } + + plugins.withType(BuildPlugin).whenPluginAdded { project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt') project.noticeFile = project.rootProject.file('NOTICE.txt') From db988ef934001f409e5106d8af76c0af978046bc Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 7 Aug 2018 16:02:42 +0300 Subject: [PATCH 5/8] Connect tests to the local repo --- buildSrc/build.gradle | 8 ++++++- .../gradle/BuildExamplePluginsIT.java | 22 +++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index d2c6083842617..72d7e17fbd93c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -162,6 +162,10 @@ if (project != rootProject) { // it's fine as we run them as part of :buildSrc test.enabled = false task integTest(type: Test) { + // integration test requires the local testing repo for example plugin builds + dependsOn project.rootProject.allprojects.collect { + it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'} + } exclude "**/*Tests.class" include "**/*IT.class" testClassesDirs = sourceSets.test.output.classesDirs @@ -172,8 +176,10 @@ if (project != rootProject) { 'test.build-tools.plugin.examples', files( project(':example-plugins').subprojects.collect { it.projectDir } - ).asPath + ).asPath, ) + systemProperty 'test.local-test-repo-path', "$rootProject.buildDir/local-test-repo" + systemProperty 'test.luceene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] } check.dependsOn(integTest) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index c161cecdb1b6b..5f23d70175d44 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -95,21 +95,17 @@ private void runAndRemove(String name) throws IOException { Files.write( tmpDir.newFile("settings.gradle").toPath(), "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) ); - // Add a repositories section to be able to resolve from snapshots. - // !NOTE! that the plugin build will use be using stale artifacts, not the ones produced in the current build - // TODO: Add "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" - // (revision = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]) + // Add a repositories section to be able to resolve dependencies Files.write( new File(tmpDir.getRoot(), "build.gradle").toPath(), ("\n" + "repositories {\n" + " maven {\n" + - " url \"https://snapshots.elastic.co/maven\"\n" + + " url \"" + getLocalTestRepoPath() + "\"\n" + " }\n" + " maven {\n" + - " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/608f0277b0\"\n" + // FIXME + " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + getLuceneSnapshotRevision() + "\"\n" + " }\n" + - " mavenLocal()\n" + // FIXME "}\n").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND ); @@ -126,4 +122,16 @@ private void runAndRemove(String name) throws IOException { .build(); } + private String getLuceneSnapshotRevision() { + return System.getProperty("test.luceene-snapshot-revision", "not-a-snapshot"); + } + + private String getLocalTestRepoPath() { + String property = System.getProperty("test.local-test-repo-path"); + Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests"); + File file = new File(property); + assertTrue("Expected " + property + " to exist, but it did not!", file.exists()); + return file.getAbsolutePath(); + } + } From 34d73bc3af856d676755ce54587b01d9b8e09ee5 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 10 Aug 2018 11:37:13 +0300 Subject: [PATCH 6/8] pr review --- .../gradle/BuildExamplePluginsIT.java | 66 +++++++------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index 5f23d70175d44..fa4fff8924fb8 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -18,10 +18,10 @@ */ package org.elasticsearch.gradle; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.commons.io.FileUtils; import org.elasticsearch.gradle.test.GradleIntegrationTestCase; import org.gradle.testkit.runner.GradleRunner; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -32,65 +32,47 @@ import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; public class BuildExamplePluginsIT extends GradleIntegrationTestCase { - private static List EXTERNAL_PROJECTS = Arrays.stream( - Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples")) - .split(File.pathSeparator) - ).map(File::new).collect(Collectors.toList()); + private static List EXAMPLE_PLUGINS = Collections.unmodifiableList( + Arrays.stream( + Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples")) + .split(File.pathSeparator) + ).map(File::new).collect(Collectors.toList()) + ); @Rule public TemporaryFolder tmpDir = new TemporaryFolder(); - @BeforeClass - public static void assertProjectsExist() { - List existing = EXTERNAL_PROJECTS.stream().filter(File::exists).collect(Collectors.toList()); - assertEquals(EXTERNAL_PROJECTS, existing); + public final File examplePlugin; + + public BuildExamplePluginsIT(File examplePlugin) { + this.examplePlugin = examplePlugin; } - @AfterClass - public static void assertEverythingTested() { + @BeforeClass + public static void assertProjectsExist() { assertEquals( - "Some example plugins are not tested: " + EXTERNAL_PROJECTS, - 0, EXTERNAL_PROJECTS.size() + EXAMPLE_PLUGINS, + EXAMPLE_PLUGINS.stream().filter(File::exists).collect(Collectors.toList()) ); } - public void testCustomSettings() throws IOException { - runAndRemove("custom-settings"); - } - - public void testPainlessWhitelist() throws IOException { - runAndRemove("painless-whitelist"); - } - - public void testRescore() throws IOException { - runAndRemove("rescore"); - } - - public void testRestHandler() throws IOException { - runAndRemove("rest-handler"); - } - - public void testScriptExpertScoring() throws IOException { - runAndRemove("script-expert-scoring"); - } - - private void runAndRemove(String name) throws IOException { - List matches = EXTERNAL_PROJECTS.stream().filter(each -> each.getAbsolutePath().contains(name)) + @ParametersFactory + public static Iterable parameters() { + return EXAMPLE_PLUGINS + .stream() + .map(each -> new Object[] {each}) .collect(Collectors.toList()); - assertEquals( - "Expected a single project folder to match `" + name + "` but got: " + matches, - 1, matches.size() - ); - - EXTERNAL_PROJECTS.remove(matches.get(0)); + } - FileUtils.copyDirectory(matches.get(0), tmpDir.getRoot()); + public void testCurrentExamplePlugin() throws IOException { + FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot()); // just get rid of deprecation warnings from now Files.write( tmpDir.newFile("settings.gradle").toPath(), "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) From cdb057b8f1330d6bd124daf917988c20e86716eb Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 15 Aug 2018 16:21:20 +0300 Subject: [PATCH 7/8] useold syntax to apply plugin --- buildSrc/build.gradle | 4 +- .../gradle/BuildExamplePluginsIT.java | 85 ++++++++++++++----- plugins/examples/custom-settings/build.gradle | 9 +- .../examples/painless-whitelist/build.gradle | 4 +- plugins/examples/rescore/build.gradle | 4 +- plugins/examples/rest-handler/build.gradle | 4 +- .../script-expert-scoring/build.gradle | 4 +- 7 files changed, 73 insertions(+), 41 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 72d7e17fbd93c..967c2e27ee8df 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -178,8 +178,8 @@ if (project != rootProject) { project(':example-plugins').subprojects.collect { it.projectDir } ).asPath, ) - systemProperty 'test.local-test-repo-path', "$rootProject.buildDir/local-test-repo" - systemProperty 'test.luceene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] + systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo" + systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1] } check.dependsOn(integTest) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java index fa4fff8924fb8..9b63d6f45e06b 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collections; @@ -73,39 +74,83 @@ public static Iterable parameters() { public void testCurrentExamplePlugin() throws IOException { FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot()); - // just get rid of deprecation warnings from now + // just get rid of deprecation warnings Files.write( - tmpDir.newFile("settings.gradle").toPath(), "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) - ); - // Add a repositories section to be able to resolve dependencies - Files.write( - new File(tmpDir.getRoot(), "build.gradle").toPath(), - ("\n" + - "repositories {\n" + - " maven {\n" + - " url \"" + getLocalTestRepoPath() + "\"\n" + - " }\n" + - " maven {\n" + - " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + getLuceneSnapshotRevision() + "\"\n" + - " }\n" + - "}\n").getBytes(StandardCharsets.UTF_8), - StandardOpenOption.APPEND + getTempPath("settings.gradle"), + "enableFeaturePreview('STABLE_PUBLISHING')\n".getBytes(StandardCharsets.UTF_8) ); + + adaptBuildScriptForTest(); + Files.write( tmpDir.newFile("NOTICE.txt").toPath(), - "dummy test nottice".getBytes(StandardCharsets.UTF_8) + "dummy test notice".getBytes(StandardCharsets.UTF_8) ); GradleRunner.create() - // Todo make a copy, write a settings file enabling stable publishing .withProjectDir(tmpDir.getRoot()) .withArguments("clean", "check", "-s", "-i", "--warning-mode=all", "--scan") .withPluginClasspath() .build(); } - private String getLuceneSnapshotRevision() { - return System.getProperty("test.luceene-snapshot-revision", "not-a-snapshot"); + private void adaptBuildScriptForTest() throws IOException { + // Add the local repo as a build script URL so we can pull in build-tools and apply the plugin under test + // + is ok because we have no other repo and just want to pick up latest + writeBuildScript( + "buildscript {\n" + + " repositories {\n" + + " maven {\n" + + " url = '" + getLocalTestRepoPath() + "'\n" + + " }\n" + + " }\n" + + " dependencies {\n" + + " classpath \"org.elasticsearch.gradle:build-tools:+\"\n" + + " }\n" + + "}\n" + ); + // get the original file + Files.readAllLines(getTempPath("build.gradle"), StandardCharsets.UTF_8) + .stream() + .map(line -> line + "\n") + .forEach(this::writeBuildScript); + // Add a repositories section to be able to resolve dependencies + String luceneSnapshotRepo = ""; + String luceneSnapshotRevision = System.getProperty("test.lucene-snapshot-revision"); + if (luceneSnapshotRepo != null) { + luceneSnapshotRepo = " maven {\n" + + " url \"http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision + "\"\n" + + " }\n"; + } + writeBuildScript("\n" + + "repositories {\n" + + " maven {\n" + + " url \"" + getLocalTestRepoPath() + "\"\n" + + " }\n" + + luceneSnapshotRepo + + "}\n" + ); + Files.delete(getTempPath("build.gradle")); + Files.move(getTempPath("build.gradle.new"), getTempPath("build.gradle")); + System.err.print("Generated build script is:"); + Files.readAllLines(getTempPath("build.gradle")).forEach(System.err::println); + } + + private Path getTempPath(String fileName) { + return new File(tmpDir.getRoot(), fileName).toPath(); + } + + private Path writeBuildScript(String script) { + try { + Path path = getTempPath("build.gradle.new"); + return Files.write( + path, + script.getBytes(StandardCharsets.UTF_8), + Files.exists(path) ? StandardOpenOption.APPEND : StandardOpenOption.CREATE_NEW + ); + } catch (IOException e) { + throw new RuntimeException(e); + } } private String getLocalTestRepoPath() { diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index a69d51ca9f85e..3caf29c8513b5 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -16,10 +16,7 @@ * specific language governing permissions and limitations * under the License. */ - -plugins { - id 'elasticsearch.esplugin' -} +apply plugin: 'elasticsearch.esplugin' esplugin { name 'custom-settings' @@ -32,6 +29,4 @@ esplugin { integTestCluster { // Adds a setting in the Elasticsearch keystore before running the integration tests keystoreSetting 'custom.secured', 'password' -} - - +} \ No newline at end of file diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 4230cf93969f4..cb2aeb82e9d01 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -16,9 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -plugins { - id 'elasticsearch.esplugin' -} +apply plugin: 'elasticsearch.esplugin' esplugin { name 'painless-whitelist' diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index b6b101794c6e5..cdecd760c81e8 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -16,9 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -plugins { - id 'elasticsearch.esplugin' -} +apply plugin: 'elasticsearch.esplugin' esplugin { name 'example-rescore' diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index deea35aa2c8be..eff2fd1b6c6e4 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -16,9 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -plugins { - id 'elasticsearch.esplugin' -} +apply plugin: 'elasticsearch.esplugin' esplugin { name 'rest-handler' diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index 78c7d0de2f286..e9da62acdcff4 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -16,9 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -plugins { - id 'elasticsearch.esplugin' -} +apply plugin: 'elasticsearch.esplugin' esplugin { name 'script-expert-scoring' From 4ef6a43d3e9241ff11872eb3f9c4e3543211b4ea Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 16 Aug 2018 08:55:21 +0300 Subject: [PATCH 8/8] PR review --- build.gradle | 10 +--------- .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 3 +-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 84b0e9564053b..0df5b97ae4a26 100644 --- a/build.gradle +++ b/build.gradle @@ -90,19 +90,11 @@ subprojects { repositories { maven { name = 'localTest' - url = "$rootProject.buildDir/local-test-repo" + url = "${rootProject.buildDir}/local-test-repo" } } } } - // Make nebula and shadow play nice together, Gradle doesn't like it how one jar task replaces the output of another - // if both are part of published artifacts - project.afterEvaluate { - if (plugins.hasPlugin(MavenPublishPlugin) && plugins.hasPlugin(ShadowPlugin)) { - project.publishing.publications.nebula.artifacts = [project.tasks.shadowJar] - } - } - plugins.withType(BuildPlugin).whenPluginAdded { project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index f6c6d5d7fd7c2..4122033ace670 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -523,7 +523,7 @@ class BuildPlugin implements Plugin { project.publishing { publications { nebula(MavenPublication) { - artifact project.tasks.shadowJar + artifacts = [ project.tasks.shadowJar ] artifactId = project.archivesBaseName /* * Configure the pom to include the "shadow" as compile dependencies @@ -553,7 +553,6 @@ class BuildPlugin implements Plugin { } } } - } /** Adds compiler settings to the project */