Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add cleanWorkspaceAfterBuild and cleanWorkspaceBuildOutputAfterBuild #2416

Merged
merged 1 commit into from
Feb 2, 2021
Merged

Add cleanWorkspaceAfterBuild and cleanWorkspaceBuildOutputAfterBuild #2416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion FAQ.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,4 +125,10 @@ For more information, see the [PR testing documentation](./pipelines/build/prTes

The operating systems/distributions which we build or are documented in the
[openjdk-build wiki](https://github.com/AdoptOpenJDK/openjdk-build/wiki/%5BWIP%5D-Minimum-OS-levels).
Runtime platforms are in our [supported platforms page](https://adoptopenjdk.net/supported_platforms.html).
Runtime platforms are in our [supported platforms page](https://adoptopenjdk.net/supported_platforms.html).

## How to add a new build pipeline param and associated job configuration?

The following PR: https://github.com/AdoptOpenJDK/openjdk-build/pull/2416
demonstrates changes required to add a new build pipeline param, and also associated version/platform job configurations for setting the value when needed.

30 changes: 29 additions & 1 deletion pipelines/build/common/build_base_file.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class Builder implements Serializable {
boolean enableInstallers
boolean enableSigner
boolean cleanWorkspaceBeforeBuild
boolean cleanWorkspaceAfterBuild
boolean cleanWorkspaceBuildOutputAfterBuild
boolean propagateFailures
boolean keepTestReportDir
boolean keepReleaseLogs
Expand Down Expand Up @@ -119,12 +121,20 @@ class Builder implements Serializable {

def testList = getTestList(platformConfig)

def platformCleanWorkspaceAfterBuild = getCleanWorkspaceAfterBuild(platformConfig)

// Always clean on mac due to https://github.com/AdoptOpenJDK/openjdk-build/issues/1980
def cleanWorkspace = cleanWorkspaceBeforeBuild
if (platformConfig.os == "mac") {
cleanWorkspace = true
}

def cleanWsAfter = cleanWorkspaceAfterBuild
if (platformCleanWorkspaceAfterBuild) {
// Platform override specified
cleanWsAfter = platformCleanWorkspaceAfterBuild
}

return new IndividualBuildConfig(
JAVA_TO_BUILD: javaToBuild,
ARCHITECTURE: platformConfig.arch as String,
Expand All @@ -151,7 +161,9 @@ class Builder implements Serializable {
ENABLE_TESTS: enableTests,
ENABLE_INSTALLERS: enableInstallers,
ENABLE_SIGNER: enableSigner,
CLEAN_WORKSPACE: cleanWorkspace
CLEAN_WORKSPACE: cleanWorkspace,
CLEAN_WORKSPACE_AFTER: cleanWsAfter,
CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER: cleanWorkspaceBuildOutputAfterBuild
)
}

Expand Down Expand Up @@ -220,6 +232,18 @@ class Builder implements Serializable {
return testList
}

/*
Get the cleanWorkspaceAfterBuild override for this platform configuration
*/
Boolean getCleanWorkspaceAfterBuild(Map<String, ?> configuration) {
Boolean cleanWorkspaceAfterBuild = null
if (configuration.containsKey("cleanWorkspaceAfterBuild") && configuration.get("cleanWorkspaceAfterBuild")) {
cleanWorkspaceAfterBuild = configuration.cleanWorkspaceAfterBuild as Boolean
}

return cleanWorkspaceAfterBuild
}

/*
Parses and applies the dockerExcludes parameter.
Any platforms/variants that are declared in this parameter are marked as excluded from docker building via this function. Even if they have a docker image or file declared in the build configurations!
Expand Down Expand Up @@ -683,6 +707,8 @@ return {
String additionalBuildArgs,
String overrideFileNameVersion,
String cleanWorkspaceBeforeBuild,
String cleanWorkspaceAfterBuild,
String cleanWorkspaceBuildOutputAfterBuild,
String adoptBuildNumber,
String propagateFailures,
String keepTestReportDir,
Expand Down Expand Up @@ -734,6 +760,8 @@ return {
additionalBuildArgs: additionalBuildArgs,
overrideFileNameVersion: overrideFileNameVersion,
cleanWorkspaceBeforeBuild: Boolean.parseBoolean(cleanWorkspaceBeforeBuild),
cleanWorkspaceAfterBuild: Boolean.parseBoolean(cleanWorkspaceAfterBuild),
cleanWorkspaceBuildOutputAfterBuild: Boolean.parseBoolean(cleanWorkspaceBuildOutputAfterBuild),
adoptBuildNumber: adoptBuildNumber,
propagateFailures: Boolean.parseBoolean(propagateFailures),
keepTestReportDir: Boolean.parseBoolean(keepTestReportDir),
Expand Down
4 changes: 3 additions & 1 deletion pipelines/build/common/config_regeneration.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,9 @@ class Regeneration implements Serializable {
ENABLE_TESTS: true,
ENABLE_INSTALLERS: true,
ENABLE_SIGNER: true,
CLEAN_WORKSPACE: true
CLEAN_WORKSPACE: true,
CLEAN_WORKSPACE_AFTER: true,
CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER: false
)
} catch (Exception e) {
// Catch invalid configurations
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/common/create_job_from_template.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ pipelineJob("$buildFolder/$JOB_NAME") {
<dt><strong>ENABLE_INSTALLERS</strong></dt><dd>Run installers</dd>
<dt><strong>ENABLE_SIGNER</strong></dt><dd>Run signer</dd>
<dt><strong>CLEAN_WORKSPACE</strong></dt><dd>Wipe out workspace before build</dd>
<dt><strong>CLEAN_WORKSPACE_AFTER</strong></dt><dd>Wipe out workspace after build</dd>
<dt><strong>CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER</strong></dt><dd>Wipe out workspace build output only, after build</dd>
</dl>
""")
}
Expand Down
56 changes: 31 additions & 25 deletions pipelines/build/common/openjdk_build_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ class Build {
NODE_CHECKOUT_TIMEOUT : 1,
BUILD_JDK_TIMEOUT : 8,
BUILD_ARCHIVE_TIMEOUT : 3,
AIX_CLEAN_TIMEOUT : 1,
MASTER_CLEAN_TIMEOUT : 1,
DOCKER_CHECKOUT_TIMEOUT : 1,
DOCKER_PULL_TIMEOUT : 2
Expand Down Expand Up @@ -906,27 +905,20 @@ class Build {
Executed on a build node, the function checks out the repository and executes the build via ./make-adopt-build-farm.sh
Once the build completes, it will calculate its version output, commit the first metadata writeout, and archive the build results.
*/
def buildScripts(cleanWorkspace, filename) {
def buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename) {
return context.stage("build") {

if (cleanWorkspace) {
try {

try {
context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: "HOURS") {
if (buildConfig.TARGET_OS == "windows") {
// Windows machines struggle to clean themselves, see:
// https://github.com/AdoptOpenJDK/openjdk-build/issues/1855
context.sh(script: "rm -rf C:/workspace/openjdk-build/workspace/build/src/build/*/jdk/gensrc")
// https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1419
context.sh(script: "rm -rf J:/jenkins/tmp/workspace/build/src/build/*/jdk/gensrc")
// https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1662
context.sh(script: "rm -rf E:/jenkins/tmp/workspace/build/src/build/*/jdk/gensrc")
// https://github.com/AdoptOpenJDK/openjdk-infrastructure/issues/1818
context.sh(script: "rm -rf C:/Jenkins/temp/workspace/build/src/build/*/jdk/gensrc")
context.cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true
// Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead
andrew-m-leonard marked this conversation as resolved.
Show resolved Hide resolved
// Issue: https://issues.jenkins.io/browse/JENKINS-64779
if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think if (context.WORKSPACE) achieves the same thing.

context.println "Cleaning workspace files: " + context.WORKSPACE + "/*"
context.sh(script: "rm -rf " + context.WORKSPACE + "/*")
Copy link
Contributor

@AdamBrousseau AdamBrousseau Feb 5, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this ignores the . hidden directories (eg. .git folder). Which is causing me issues on my win boxes.

} else {
context.cleanWs notFailBuild: true
context.println "Warning: Unable to clean workspace as context.WORKSPACE is null/empty"
}
}
} catch (FlowInterruptedException e) {
Expand Down Expand Up @@ -999,15 +991,27 @@ class Build {
}
} finally {
// post-build workspace clean:
// AIX due to limited ram drive space
// s390x due to limited available disk space on Marist nodes
if (buildConfig.TARGET_OS == "aix" || buildConfig.ARCHITECTURE == "s390x") {
if (cleanWorkspaceAfter || cleanWorkspaceBuildOutputAfter) {
try {
context.timeout(time: buildTimeouts.AIX_CLEAN_TIMEOUT, unit: "HOURS") {
context.cleanWs notFailBuild: true
context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: "HOURS") {
// Note: Underlying org.apache DirectoryScanner used by cleanWs has a bug scanning where it misses files containing ".." so use rm -rf instead
// Issue: https://issues.jenkins.io/browse/JENKINS-64779
if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) {
if (cleanWorkspaceAfter) {
context.println "Cleaning workspace files: " + context.WORKSPACE + "/*"
context.sh(script: "rm -rf " + context.WORKSPACE + "/*")
} else if (cleanWorkspaceBuildOutputAfter) {
context.println "Cleaning workspace build output files: " + context.WORKSPACE + "/workspace/build/src/build"
context.sh(script: "rm -rf " + context.WORKSPACE + "/workspace/build/src/build")
context.println "Cleaning workspace build output files: " + context.WORKSPACE + "/workspace/target"
context.sh(script: "rm -rf " + context.WORKSPACE + "/workspace/target")
}
} else {
andrew-m-leonard marked this conversation as resolved.
Show resolved Hide resolved
context.println "Warning: Unable to clean workspace as context.WORKSPACE is null/empty"
}
}
} catch (FlowInterruptedException e) {
context.println "[ERROR] AIX clean workspace timeout (${buildTimeouts.AIX_CLEAN_TIMEOUT} HOURS) has been reached. Exiting..."
context.println "[ERROR] clean workspace timeout (${buildTimeouts.NODE_CLEAN_TIMEOUT} HOURS) has been reached. Exiting..."
andrew-m-leonard marked this conversation as resolved.
Show resolved Hide resolved
throw new Exception()
}
}
Expand Down Expand Up @@ -1076,6 +1080,8 @@ class Build {
def enableInstallers = Boolean.valueOf(buildConfig.ENABLE_INSTALLERS)
def enableSigner = Boolean.valueOf(buildConfig.ENABLE_SIGNER)
def cleanWorkspace = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE)
def cleanWorkspaceAfter = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE_AFTER)
def cleanWorkspaceBuildOutputAfter = Boolean.valueOf(buildConfig.CLEAN_WORKSPACE_BUILD_OUTPUT_ONLY_AFTER)

context.stage("queue") {
/* This loads the library containing two Helper classes, and causes them to be
Expand Down Expand Up @@ -1135,7 +1141,7 @@ class Build {
}

context.docker.build("build-image", "--build-arg image=${buildConfig.DOCKER_IMAGE} -f ${buildConfig.DOCKER_FILE} .").inside {
buildScripts(cleanWorkspace, filename)
buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
}
// Otherwise, pull the docker image from DockerHub
} else {
Expand All @@ -1149,7 +1155,7 @@ class Build {
}

context.docker.image(buildConfig.DOCKER_IMAGE).inside {
buildScripts(cleanWorkspace, filename)
buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
}
}
}
Expand All @@ -1170,10 +1176,10 @@ class Build {
}
context.echo("changing ${workspace}")
context.ws(workspace) {
buildScripts(cleanWorkspace, filename)
buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
}
} else {
buildScripts(cleanWorkspace, filename)
buildScripts(cleanWorkspace, cleanWorkspaceAfter, cleanWorkspaceBuildOutputAfter, filename)
}
}
context.println "[NODE SHIFT] OUT OF NODE (LABELNAME ${buildConfig.NODE_LABEL}!)"
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk10_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk11_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk12_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk13_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk14_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk15_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk16_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk17_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk8_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
2 changes: 2 additions & 0 deletions pipelines/build/openjdk9_pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ if (scmVars != null && configureBuild != null && buildConfigurations != null) {
additionalBuildArgs,
overrideFileNameVersion,
cleanWorkspaceBeforeBuild,
cleanWorkspaceAfterBuild,
cleanWorkspaceBuildOutputAfterBuild,
adoptBuildNumber,
propagateFailures,
keepTestReportDir,
Expand Down
3 changes: 2 additions & 1 deletion pipelines/jobs/configurations/jdk11u_pipeline_config.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ class Config11 {
hotspot: 'xlc13&&aix710',
openj9: 'xlc13&&aix715'
],
test : 'default'
test : 'default',
cleanWorkspaceAfterBuild: true
andrew-m-leonard marked this conversation as resolved.
Show resolved Hide resolved
],

s390xLinux : [
Expand Down
3 changes: 2 additions & 1 deletion pipelines/jobs/configurations/jdk16_pipeline_config.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,8 @@ class Config16 {
hotspot: 'xlc16&&aix710',
openj9: 'xlc16&&aix715'
],
test : 'default'
test : 'default',
cleanWorkspaceAfterBuild: true
],


Expand Down
3 changes: 2 additions & 1 deletion pipelines/jobs/configurations/jdk17_pipeline_config.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ class Config17 {
test : [
nightly: [],
weekly : ['sanity.openjdk', 'sanity.system', 'extended.system']
]
],
cleanWorkspaceAfterBuild: true
],


Expand Down
5 changes: 3 additions & 2 deletions pipelines/jobs/configurations/jdk8u_pipeline_config.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ class Config8 {
hotspot: 'xlc13&&aix710',
openj9: 'xlc13&&aix715'
],
test : 'default'
test : 'default',
cleanWorkspaceAfterBuild: true
],

s390xLinux : [
Expand Down Expand Up @@ -152,4 +153,4 @@ class Config8 {
}

Config8 config = new Config8()
return config.buildConfigurations
return config.buildConfigurations
3 changes: 3 additions & 0 deletions pipelines/jobs/pipeline_job_template.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ propagateFailures = false
runTests = true
runInstaller = true
runSigner = true
cleanWsBuildOutput = true

// if true means this is running in the pr builder pipeline
if (binding.hasVariable('PR_BUILDER')) {
Expand Down Expand Up @@ -72,6 +73,8 @@ pipelineJob("${BUILD_FOLDER}/${JOB_NAME}") {
stringParam('additionalBuildArgs', "", "Additional arguments to be passed to <code>makejdk-any-platform.sh</code>")
stringParam('overrideFileNameVersion', "", "When forming the filename, ignore the part of the filename derived from the publishName or timestamp and override it.<br/>For instance if you set this to 'FOO' the final file name will be of the form: <code>OpenJDK8U-jre_ppc64le_linux_openj9_FOO.tar.gz</code>")
booleanParam('cleanWorkspaceBeforeBuild', false, "Clean out the workspace before the build")
booleanParam('cleanWorkspaceAfterBuild', false, "Clean out the workspace after the build")
booleanParam('cleanWorkspaceBuildOutputAfterBuild', cleanWsBuildOutput, "Clean out the workspace/build/src/build and workspace/target output only, after the build")
booleanParam('propagateFailures', propagateFailures, "If true, a failure of <b>ANY</b> downstream build (but <b>NOT</b> test) will cause the whole build to fail")
booleanParam('keepTestReportDir', false, 'If true, test report dir (including core files where generated) will be kept even when the testcase passes, failed testcases always keep the report dir. Does not apply to JUnit jobs which are always kept, eg.openjdk.')
booleanParam('keepReleaseLogs', true, 'If true, "Release" type pipeline Jenkins logs will be marked as "Keep this build forever".')
Expand Down
Loading