From 86bebcbaff26f728bcced9dc05cbca9195efe2f0 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 14 Dec 2021 09:33:08 +0100 Subject: [PATCH] Leverage artifact transformations to unpack old test es distributions (#79827) (#79909) (#81636) --- modules/reindex/build.gradle | 34 +++++++++---------- .../qa/repository-old-versions/build.gradle | 34 ++++++++----------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index ec77c42477063..fedeee6d95ea4 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -11,6 +11,8 @@ import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.AntFixture +import org.elasticsearch.gradle.transform.UnzipTransform +import org.gradle.api.internal.artifacts.ArtifactAttributes apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' @@ -102,8 +104,17 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { systemProperty "tests.fromOld", "false" } } else { - /* Set up tasks to unzip and run the old versions of ES before running the - * integration tests. */ + /* Register a gradle artifact transformation to unpack resolved elasticsearch distributions. We only resolve + * zip files here. Using artifact transforms allow a better caching of the downloaded distros as the + * transformed (unpacked) distro will be cached by gradle resulting in less unpacking + * + * To avoid testing against too many old versions, always pick first and last version per major + */ + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); + transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + def versions = ['2', '1', '090'] if (Os.isFamily(Os.FAMILY_MAC)) { // 0.90 fails sometimes on mac, given that it is so old, let us disable it @@ -111,21 +122,10 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { versions = ['2', '1'] } versions.each { version -> - // TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms - TaskProvider unzip = tasks.register("unzipEs${version}", Sync) { - Configuration oldEsDependency = configurations['es' + version] - dependsOn oldEsDependency - /* Use a closure here to delay resolution of the dependency until we need - * it */ - from { - oldEsDependency.collect { zipTree(it) } - } - into temporaryDir - } - + def oldEsDependency = configurations['es' + version] + oldEsDependency.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); TaskProvider fixture = tasks.register("oldEs${version}Fixture", AntFixture) { - dependsOn project.configurations.oldesFixture, jdks.legacy - dependsOn unzip + dependsOn project.configurations.oldesFixture, jdks.legacy, oldEsDependency executable = "${BuildParams.runtimeJavaHome}/bin/java" env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" // old versions of Elasticsearch need JAVA_HOME @@ -136,7 +136,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } args 'oldes.OldElasticsearch', baseDir, - unzip.get().temporaryDir, + "${ -> oldEsDependency.singleFile.getPath()}", version == '090' waitCondition = { fixture, ant -> // the fixture writes the ports file when Elasticsearch's HTTP service diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle index fce544839e21a..d0a2d8dcd4e99 100644 --- a/x-pack/qa/repository-old-versions/build.gradle +++ b/x-pack/qa/repository-old-versions/build.gradle @@ -12,6 +12,8 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.AntFixture import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.elasticsearch.gradle.transform.UnzipTransform +import org.gradle.api.internal.artifacts.ArtifactAttributes apply plugin: 'elasticsearch.jdk-download' apply plugin: 'elasticsearch.internal-testclusters' @@ -39,9 +41,17 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows") tasks.named("testingConventions").configure { enabled = false } } else { - /* Set up tasks to unzip and run the old versions of ES before running the integration tests. + /* Register a gradle artifact transformation to unpack resolved elasticsearch distributions. We only resolve + * zip files here. Using artifact transforms allow a better caching of the downloaded distros as the + * transformed (unpacked) distro will be cached by gradle resulting in less unpacking + * * To avoid testing against too many old versions, always pick first and last version per major */ + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); + transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + for (String versionString : ['5.0.0', '5.6.16', '6.0.0', '6.8.20']) { Version version = Version.fromString(versionString) String packageName = 'org.elasticsearch.distribution.zip' @@ -49,24 +59,11 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { String versionNoDots = version.toString().replace('.', '_') String configName = "es${versionNoDots}" - configurations.create(configName) - + def config = configurations.create(configName) + config.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); dependencies.add(configName, artifact) - // TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms - TaskProvider unzip = tasks.register("unzipEs${versionNoDots}", Sync) { - Configuration oldEsDependency = configurations[configName] - dependsOn oldEsDependency - /* Use a closure here to delay resolution of the dependency until we need - * it */ - from { - oldEsDependency.collect { zipTree(it) } - } - into temporaryDir - } - String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}" - String clusterName = versionNoDots def testClusterProvider = testClusters.register(clusterName) { @@ -83,8 +80,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } TaskProvider fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) { - dependsOn project.configurations.oldesFixture, jdks.legacy - dependsOn unzip + dependsOn project.configurations.oldesFixture, jdks.legacy, config executable = "${BuildParams.runtimeJavaHome}/bin/java" env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" // old versions of Elasticsearch need JAVA_HOME @@ -95,7 +91,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } args 'oldes.OldElasticsearch', baseDir, - unzip.get().temporaryDir, + "${ -> config.getSingleFile().toPath()}", false, "path.repo: ${repoLocation}" if (version.onOrAfter('6.8.0') && Architecture.current() == Architecture.AARCH64) {