Add cleanWorkspaceAfterBuild and cleanWorkspaceBuildOutputAfterBuild

Signed-off-by: Andrew Leonard <[email protected]>
adoptium · Feb 2, 2021 · 50bf26e · 50bf26e
1 parent a32484f
commit 50bf26e
Show file tree

Hide file tree

Showing 22 changed files with 114 additions and 35 deletions.
diff --git a/FAQ.md b/FAQ.md
@@ -125,4 +125,10 @@ For more information, see the [PR testing documentation](./pipelines/build/prTes
 
 The operating systems/distributions which we build or are documented in the
 [openjdk-build wiki](https://github.com/AdoptOpenJDK/openjdk-build/wiki/%5BWIP%5D-Minimum-OS-levels).
-Runtime platforms are in our [supported platforms page](https://adoptopenjdk.net/supported_platforms.html).
+Runtime platforms are in our [supported platforms page](https://adoptopenjdk.net/supported_platforms.html).
+
+## How to add a new build pipeline param and associated job configuration?
+
+The following PR: https://github.com/AdoptOpenJDK/openjdk-build/pull/2416
+demonstrates changes required to add a new build pipeline param, and also associated version/platform job configurations for setting the value when needed.
+
diff --git a/pipelines/build/common/build_base_file.groovy b/pipelines/build/common/build_base_file.groovy
@@ -51,6 +51,8 @@ class Builder implements Serializable {
     boolean enableInstallers
     boolean enableSigner
     boolean cleanWorkspaceBeforeBuild
+    boolean cleanWorkspaceAfterBuild
+    boolean cleanWorkspaceBuildOutputAfterBuild
     boolean propagateFailures
     boolean keepTestReportDir
     boolean keepReleaseLogs
@@ -119,12 +121,20 @@ class Builder implements Serializable {
 
         def testList = getTestList(platformConfig)
 
+        def platformCleanWorkspaceAfterBuild = getCleanWorkspaceAfterBuild(platformConfig)
+
         // Always clean on mac due to https://github.com/AdoptOpenJDK/openjdk-build/issues/1980
         def cleanWorkspace = cleanWorkspaceBeforeBuild
         if (platformConfig.os == "mac") {
             cleanWorkspace = true
         }
 
+        def cleanWsAfter = cleanWorkspaceAfterBuild
+        if (platformCleanWorkspaceAfterBuild) {
+            // Platform override specified
+            cleanWsAfter = platformCleanWorkspaceAfterBuild
+        }
+
         return new IndividualBuildConfig(
                 JAVA_TO_BUILD: javaToBuild,
                 ARCHITECTURE: platformConfig.arch as String,
@@ -151,7 +161,9 @@ class Builder implements Serializable {
                 ENABLE_TESTS: enableTests,
                 ENABLE_INSTALLERS: enableInstallers,
                 ENABLE_SIGNER: enableSigner,
-                CLEAN_WORKSPACE: cleanWorkspace
+                CLEAN_WORKSPACE: cleanWorkspace,
+                CLEAN_WORKSPACE_AFTER: cleanWsAfter,
+                CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER: cleanWorkspaceBuildOutputAfterBuild
         )
     }
 
@@ -220,6 +232,18 @@ class Builder implements Serializable {
         return testList
     }
 
+    /*
+    Get the cleanWorkspaceAfterBuild override for this platform configuration
+    */
+    Boolean getCleanWorkspaceAfterBuild(Map<String, ?> configuration) {
+        Boolean cleanWorkspaceAfterBuild = null
+        if (configuration.containsKey("cleanWorkspaceAfterBuild") && configuration.get("cleanWorkspaceAfterBuild")) {
+            cleanWorkspaceAfterBuild = configuration.cleanWorkspaceAfterBuild as Boolean
+        }
+
+        return cleanWorkspaceAfterBuild
+    }
+
     /*
     Parses and applies the dockerExcludes parameter.
     Any platforms/variants that are declared in this parameter are marked as excluded from docker building via this function. Even if they have a docker image or file declared in the build configurations!
@@ -683,6 +707,8 @@ return {
     String additionalBuildArgs,
     String overrideFileNameVersion,
     String cleanWorkspaceBeforeBuild,
+    String cleanWorkspaceAfterBuild,
+    String cleanWorkspaceBuildOutputAfterBuild,
     String adoptBuildNumber,
     String propagateFailures,
     String keepTestReportDir,
@@ -734,6 +760,8 @@ return {
             additionalBuildArgs: additionalBuildArgs,
             overrideFileNameVersion: overrideFileNameVersion,
             cleanWorkspaceBeforeBuild: Boolean.parseBoolean(cleanWorkspaceBeforeBuild),
+            cleanWorkspaceAfterBuild: Boolean.parseBoolean(cleanWorkspaceAfterBuild),
+            cleanWorkspaceBuildOutputAfterBuild: Boolean.parseBoolean(cleanWorkspaceBuildOutputAfterBuild),
             adoptBuildNumber: adoptBuildNumber,
             propagateFailures: Boolean.parseBoolean(propagateFailures),
             keepTestReportDir: Boolean.parseBoolean(keepTestReportDir),

diff --git a/pipelines/build/common/config_regeneration.groovy b/pipelines/build/common/config_regeneration.groovy
@@ -335,7 +335,9 @@ class Regeneration implements Serializable {
                 ENABLE_TESTS: true,
                 ENABLE_INSTALLERS: true,
                 ENABLE_SIGNER: true,
-                CLEAN_WORKSPACE: true
+                CLEAN_WORKSPACE: true,
+                CLEAN_WORKSPACE_AFTER: true,
+                CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER: false
             )
         } catch (Exception e) {
             // Catch invalid configurations

diff --git a/pipelines/build/common/create_job_from_template.groovy b/pipelines/build/common/create_job_from_template.groovy
@@ -87,6 +87,8 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                 <dt><strong>ENABLE_INSTALLERS</strong></dt><dd>Run installers</dd>
                 <dt><strong>ENABLE_SIGNER</strong></dt><dd>Run signer</dd>
                 <dt><strong>CLEAN_WORKSPACE</strong></dt><dd>Wipe out workspace before build</dd>
+                <dt><strong>CLEAN_WORKSPACE_AFTER</strong></dt><dd>Wipe out workspace after build</dd>
+                <dt><strong>CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER</strong></dt><dd>Wipe out workspace build output only, after build</dd>
             </dl>
         """)
     }

diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy
@@ -67,7 +67,6 @@ class Build {
         NODE_CHECKOUT_TIMEOUT : 1,
         BUILD_JDK_TIMEOUT : 8,
         BUILD_ARCHIVE_TIMEOUT : 3,
-        AIX_CLEAN_TIMEOUT : 1,
         MASTER_CLEAN_TIMEOUT : 1,
         DOCKER_CHECKOUT_TIMEOUT : 1,
         DOCKER_PULL_TIMEOUT : 2
@@ -906,27 +905,20 @@ class Build {
     Executed on a build node, the function checks out the repository and executes the build via ./make-adopt-build-farm.sh
     Once the build completes, it will calculate its version output, commit the first metadata writeout, and archive the build results.
     */
-    def buildScripts(cleanWorkspace, filename) {
+    def buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename) {
         return context.stage("build") {
-
             if (cleanWorkspace) {
                 try {
 
                     try {
                         context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: "HOURS") {
-                            if (buildConfig.TARGET_OS == "windows") {
-                                // Windows machines struggle to clean themselves, see:
-                                // https://github.com/AdoptOpenJDK/openjdk-build/issues/1855
-                                context.sh(script: "rm -rf C:/workspace/openjdk-build/workspace/build/src/build/*/jdk/gensrc")
-                                // https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1419
-                                context.sh(script: "rm -rf J:/jenkins/tmp/workspace/build/src/build/*/jdk/gensrc")
-                                // https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1662
-                                context.sh(script: "rm -rf E:/jenkins/tmp/workspace/build/src/build/*/jdk/gensrc")
-                                // https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1818
-                                context.sh(script: "rm -rf C:/Jenkins/temp/workspace/build/src/build/*/jdk/gensrc")
-                                context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true
+                            // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead
+                            // Issue: https://issues.jenkins.io/browse/JENKINS-64779
+                            if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) {
+                                context.println "Cleaning workspace files: " + context.WORKSPACE + "/*"
+                                context.sh(script: "rm -rf " + context.WORKSPACE + "/*")
                             } else {
-                                context.cleanWs notFailBuild: true
+                                context.println "Warning: Unable to clean workspace as context.WORKSPACE is null/empty"
                             }
                         }
                     } catch (FlowInterruptedException e) {
@@ -999,15 +991,27 @@ class Build {
                 }
             } finally {
                 // post-build workspace clean:
-                //   AIX due to limited ram drive space
-                //   s390x due to limited available disk space on Marist nodes 
-                if (buildConfig.TARGET_OS == "aix" || buildConfig.ARCHITECTURE == "s390x") {
+                if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) {
                     try {
-                        context.timeout(time: buildTimeouts.AIX_CLEAN_TIMEOUT, unit: "HOURS") {
-                            context.cleanWs notFailBuild: true
+                        context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: "HOURS") {
+                            // Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead
+                            // Issue: https://issues.jenkins.io/browse/JENKINS-64779
+                            if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) {
+                                if (cleanWorkspaceAfter) {
+                                    context.println "Cleaning workspace files: " + context.WORKSPACE + "/*"
+                                    context.sh(script: "rm -rf " + context.WORKSPACE + "/*")
+                                } else if (cleanWorkspaceBuildOutputAfter) {
+                                    context.println "Cleaning workspace build output files: " + context.WORKSPACE + "/workspace/build/src/build"
+                                    context.sh(script: "rm -rf " + context.WORKSPACE + "/workspace/build/src/build")
+                                    context.println "Cleaning workspace build output files: " + context.WORKSPACE + "/workspace/target"
+                                    context.sh(script: "rm -rf " + context.WORKSPACE + "/workspace/target")
+                                }
+                            } else {
+                                context.println "Warning: Unable to clean workspace as context.WORKSPACE is null/empty"
+                            }
                         }
                     } catch (FlowInterruptedException e) {
-                        context.println "[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting..."
+                        context.println "[ERROR] clean workspace timeout (${buildTimeouts.NODE_CLEAN_TIMEOUT} HOURS) has been reached. Exiting..."
                         throw new Exception()
                     }
                 }
@@ -1076,6 +1080,8 @@ class Build {
                 def enableInstallers = Boolean.valueOf(buildConfig.ENABLE_INSTALLERS)
                 def enableSigner = Boolean.valueOf(buildConfig.ENABLE_SIGNER)
                 def cleanWorkspace = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE)
+                def cleanWorkspaceAfter = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE_AFTER)
+                def cleanWorkspaceBuildOutputAfter = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER)
 
                 context.stage("queue") {
                     /* This loads the library containing two Helper classes, and causes them to be
@@ -1135,7 +1141,7 @@ class Build {
                                 }
 
                                 context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside {
-                                    buildScripts(cleanWorkspace, filename)
+                                    buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
                                 }
                             // Otherwise, pull the docker image from DockerHub
                             } else {
@@ -1149,7 +1155,7 @@ class Build {
                                 }
 
                                 context.docker.image(buildConfig.DOCKER_IMAGE).inside {
-                                    buildScripts(cleanWorkspace, filename)
+                                    buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
                                 }
                             }
                         }
@@ -1170,10 +1176,10 @@ class Build {
                                 }
                                 context.echo("changing ${workspace}")
                                 context.ws(workspace) {
-                                    buildScripts(cleanWorkspace, filename)
+                                    buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
                                 }
                             } else {
-                                buildScripts(cleanWorkspace, filename)
+                                buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
                             }
                         }
                         context.println "[NODE SHIFT] OUT OF NODE (LABELNAME ${buildConfig.NODE_LABEL}!)"

diff --git a/pipelines/build/openjdk10_pipeline.groovy b/pipelines/build/openjdk10_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk11_pipeline.groovy b/pipelines/build/openjdk11_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk12_pipeline.groovy b/pipelines/build/openjdk12_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk13_pipeline.groovy b/pipelines/build/openjdk13_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk14_pipeline.groovy b/pipelines/build/openjdk14_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk15_pipeline.groovy b/pipelines/build/openjdk15_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk16_pipeline.groovy b/pipelines/build/openjdk16_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk17_pipeline.groovy b/pipelines/build/openjdk17_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk8_pipeline.groovy b/pipelines/build/openjdk8_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/build/openjdk9_pipeline.groovy b/pipelines/build/openjdk9_pipeline.groovy
@@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
         additionalBuildArgs,
         overrideFileNameVersion,
         cleanWorkspaceBeforeBuild,
+        cleanWorkspaceAfterBuild,
+        cleanWorkspaceBuildOutputAfterBuild,
         adoptBuildNumber,
         propagateFailures,
         keepTestReportDir,

diff --git a/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk11u_pipeline_config.groovy
@@ -80,7 +80,8 @@ class Config11 {
                         hotspot: 'xlc13&&aix710',
                         openj9:  'xlc13&&aix715'
                 ],
-                test                : 'default'
+                test                : 'default',
+                cleanWorkspaceAfterBuild: true
         ],
 
         s390xLinux    : [

diff --git a/pipelines/jobs/configurations/jdk16_pipeline_config.groovy b/pipelines/jobs/configurations/jdk16_pipeline_config.groovy
@@ -110,7 +110,8 @@ class Config16 {
                         hotspot: 'xlc16&&aix710',
                         openj9:  'xlc16&&aix715'
                 ],
-                test                : 'default'
+                test                : 'default',
+                cleanWorkspaceAfterBuild: true
         ],
 
 

diff --git a/pipelines/jobs/configurations/jdk17_pipeline_config.groovy b/pipelines/jobs/configurations/jdk17_pipeline_config.groovy
@@ -122,7 +122,8 @@ class Config17 {
                 test                : [
                         nightly: [],
                         weekly : ['sanity.openjdk', 'sanity.system', 'extended.system']
-                ]
+                ],
+                cleanWorkspaceAfterBuild: true
         ],
 
 

diff --git a/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk8u_pipeline_config.groovy
@@ -77,7 +77,8 @@ class Config8 {
                         hotspot: 'xlc13&&aix710',
                         openj9:  'xlc13&&aix715'
                 ],
-                test                 : 'default'
+                test                 : 'default',
+                cleanWorkspaceAfterBuild: true
         ],
 
         s390xLinux    : [
@@ -152,4 +153,4 @@ class Config8 {
 }
 
 Config8 config = new Config8()
-return config.buildConfigurations
+return config.buildConfigurations
diff --git a/pipelines/jobs/pipeline_job_template.groovy b/pipelines/jobs/pipeline_job_template.groovy
@@ -9,6 +9,7 @@ propagateFailures = false
 runTests = true
 runInstaller = true
 runSigner = true
+cleanWsBuildOutput = true
 
 // if true means this is running in the pr builder pipeline
 if (binding.hasVariable('PR_BUILDER')) {
@@ -72,6 +73,8 @@ pipelineJob("${BUILD_FOLDER}/${JOB_NAME}") {
         stringParam('additionalBuildArgs', "", "Additional arguments to be passed to <code>makejdk-any-platform.sh</code>")
         stringParam('overrideFileNameVersion', "", "When forming the filename, ignore the part of the filename derived from the publishName or timestamp and override it.<br/>For instance if you set this to 'FOO' the final file name will be of the form: <code>OpenJDK8U-jre_ppc64le_linux_openj9_FOO.tar.gz</code>")
         booleanParam('cleanWorkspaceBeforeBuild', false, "Clean out the workspace before the build")
+        booleanParam('cleanWorkspaceAfterBuild', false, "Clean out the workspace after the build")
+        booleanParam('cleanWorkspaceBuildOutputAfterBuild', cleanWsBuildOutput, "Clean out the workspace/build/src/build and workspace/target output only, after the build")
         booleanParam('propagateFailures', propagateFailures, "If true, a failure of <b>ANY</b> downstream build (but <b>NOT</b> test) will cause the whole build to fail")
         booleanParam('keepTestReportDir', false, 'If true, test report dir (including core files where generated) will be kept even when the testcase passes, failed testcases always keep the report dir. Does not apply to JUnit jobs which are always kept, eg.openjdk.')
         booleanParam('keepReleaseLogs', true, 'If true, "Release" type pipeline Jenkins logs will be marked as "Keep this build forever".')