Leverage artifact transformations to unpack old test es distributions (…

…elastic#79827) (elastic#79909) (elastic#81636)
przemekwitek · Dec 14, 2021 · 86bebcb · 86bebcb
1 parent 0559dd0
commit 86bebcb
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 36 deletions.
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
@@ -11,6 +11,8 @@ import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.AntFixture
+import org.elasticsearch.gradle.transform.UnzipTransform
+import org.gradle.api.internal.artifacts.ArtifactAttributes
 
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.jdk-download'
@@ -102,30 +104,28 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     systemProperty "tests.fromOld", "false"
   }
 } else {
-  /* Set up tasks to unzip and run the old versions of ES before running the
-   * integration tests. */
+  /* Register a gradle artifact transformation to unpack resolved elasticsearch distributions. We only resolve
+   * zip files here. Using artifact transforms allow a better caching of the downloaded distros as the
+   * transformed (unpacked) distro will be cached by gradle resulting in less unpacking
+   *
+   * To avoid testing against too many old versions, always pick first and last version per major
+   */
+  project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+    transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
+    transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+  });
+
   def versions = ['2', '1', '090']
   if (Os.isFamily(Os.FAMILY_MAC)) {
     // 0.90 fails sometimes on mac, given that it is so old, let us disable it
     // see: https://github.com/elastic/elasticsearch/issues/51202
     versions = ['2', '1']
   }
   versions.each { version ->
-    // TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
-    TaskProvider<Sync> unzip = tasks.register("unzipEs${version}", Sync) {
-      Configuration oldEsDependency = configurations['es' + version]
-      dependsOn oldEsDependency
-      /* Use a closure here to delay resolution of the dependency until we need
-       * it */
-      from {
-        oldEsDependency.collect { zipTree(it) }
-      }
-      into temporaryDir
-    }
-
+    def oldEsDependency = configurations['es' + version]
+    oldEsDependency.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
     TaskProvider<AntFixture> fixture = tasks.register("oldEs${version}Fixture", AntFixture) {
-      dependsOn project.configurations.oldesFixture, jdks.legacy
-      dependsOn unzip
+      dependsOn project.configurations.oldesFixture, jdks.legacy, oldEsDependency
       executable = "${BuildParams.runtimeJavaHome}/bin/java"
       env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
       // old versions of Elasticsearch need JAVA_HOME
@@ -136,7 +136,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       }
       args 'oldes.OldElasticsearch',
         baseDir,
-        unzip.get().temporaryDir,
+        "${ -> oldEsDependency.singleFile.getPath()}",
         version == '090'
       waitCondition = { fixture, ant ->
         // the fixture writes the ports file when Elasticsearch's HTTP service

diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle
@@ -12,6 +12,8 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.elasticsearch.gradle.transform.UnzipTransform
+import org.gradle.api.internal.artifacts.ArtifactAttributes
 
 apply plugin: 'elasticsearch.jdk-download'
 apply plugin: 'elasticsearch.internal-testclusters'
@@ -39,34 +41,29 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows")
   tasks.named("testingConventions").configure { enabled = false }
 } else {
-  /* Set up tasks to unzip and run the old versions of ES before running the integration tests.
+  /* Register a gradle artifact transformation to unpack resolved elasticsearch distributions. We only resolve
+   * zip files here. Using artifact transforms allow a better caching of the downloaded distros as the
+   * transformed (unpacked) distro will be cached by gradle resulting in less unpacking
+   *
    * To avoid testing against too many old versions, always pick first and last version per major
    */
+  project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+    transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
+    transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+  });
+
   for (String versionString : ['5.0.0', '5.6.16', '6.0.0', '6.8.20']) {
     Version version = Version.fromString(versionString)
     String packageName = 'org.elasticsearch.distribution.zip'
     String artifact = "${packageName}:elasticsearch:${version}@zip"
     String versionNoDots = version.toString().replace('.', '_')
     String configName = "es${versionNoDots}"
 
-    configurations.create(configName)
-
+    def config = configurations.create(configName)
+    config.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
     dependencies.add(configName, artifact)
 
-    // TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
-    TaskProvider<Sync> unzip = tasks.register("unzipEs${versionNoDots}", Sync) {
-      Configuration oldEsDependency = configurations[configName]
-      dependsOn oldEsDependency
-      /* Use a closure here to delay resolution of the dependency until we need
-       * it */
-      from {
-        oldEsDependency.collect { zipTree(it) }
-      }
-      into temporaryDir
-    }
-
     String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}"
-
     String clusterName = versionNoDots
 
     def testClusterProvider = testClusters.register(clusterName) {
@@ -83,8 +80,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     }
 
     TaskProvider<AntFixture> fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) {
-      dependsOn project.configurations.oldesFixture, jdks.legacy
-      dependsOn unzip
+      dependsOn project.configurations.oldesFixture, jdks.legacy, config
       executable = "${BuildParams.runtimeJavaHome}/bin/java"
       env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
       // old versions of Elasticsearch need JAVA_HOME
@@ -95,7 +91,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       }
       args 'oldes.OldElasticsearch',
         baseDir,
-        unzip.get().temporaryDir,
+        "${ -> config.getSingleFile().toPath()}",
         false,
         "path.repo: ${repoLocation}"
       if (version.onOrAfter('6.8.0') && Architecture.current() == Architecture.AARCH64) {