From e1c58834c36a834db8abfef06138fc30473bd9d9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 2 Sep 2022 19:03:13 -0400 Subject: [PATCH] Re-adding the windows snapshot with new changes (#2550) * Re-adding the windows snapshot with new changes Signed-off-by: Peter Zhu * Update manifests Signed-off-by: Peter Zhu * Add more tests Signed-off-by: Peter Zhu * Restore Pipefile.lock Signed-off-by: Peter Zhu * Tweak comments Signed-off-by: Peter Zhu Signed-off-by: Peter Zhu --- .../opensearch/distribution-build.jenkinsfile | 62 +++++++++---------- src/system/temporary_directory.py | 10 +-- .../uploadMinSnapshotsToS3_Jenkinsfile.txt | 5 +- vars/createSha512Checksums.groovy | 2 +- vars/uploadMinSnapshotsToS3.groovy | 15 +++-- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 759588763c..ad2014549a 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -199,37 +199,37 @@ pipeline { } } } - //stage('build-snapshot-windows-x64-tar') { - // agent { - // node { - // label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' - // } - // } - // tools { - // jdk dockerAgent.javaVersion - // } - // steps { - // script { - // buildManifest( - // componentName: "${COMPONENT_NAME}", - // inputManifest: "manifests/${INPUT_MANIFEST}", - // distribution: 'zip', - // platform: 'windows', - // snapshot: true - // ) - // echo("Uploading windows min snapshots to S3") - // uploadMinSnapshotsToS3( - // fileActions: [createSha512Checksums()], - // distribution: 'zip' - // ) - // } - // } - // post { - // always { - // postCleanup() - // } - // } - //} + stage('build-snapshot-windows-x64-zip') { + agent { + node { + label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' + } + } + tools { + jdk dockerAgent.javaVersion + } + steps { + script { + buildManifest( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + distribution: 'zip', + platform: 'windows', + snapshot: true + ) + echo("Uploading windows min snapshots to S3") + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()], + distribution: 'zip' + ) + } + } + post { + always { + postCleanup() + } + } + } stage('build-and-test-x64-tar') { agent { docker { diff --git a/src/system/temporary_directory.py b/src/system/temporary_directory.py index bda92e0fb3..ec775ec01f 100644 --- a/src/system/temporary_directory.py +++ b/src/system/temporary_directory.py @@ -19,17 +19,17 @@ def g__handleRemoveReadonly(func: FunctionType, path: str, exc: Any) -> Any: logging.debug(f"excvalue {excvalue}") logging.debug(f"func {func.__name__}") if func in (os.rmdir, os.remove, os.unlink): - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IWRITE | stat.S_IREAD) # 0777 nix* / +wr windows + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IREAD | stat.S_IWRITE) # 0777 nix* / +rw windows retry_total = 3 for retry_count in range(retry_total): # Re-run func to force deletion especially on windows try: - logging.debug(f'Try count: {retry_count + 1}/{retry_total}') + logging.warn(f'Removing try count: {retry_count + 1}/{retry_total} for {path}') func_result = func(path) + logging.debug(f'func_result: {func_result}') if func_result is None: break except Exception as ex: - logging.warn(func_result) - logging.warn(ex) + logging.warn(f'Exception: {ex}') else: raise @@ -58,5 +58,5 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: if self.keep: logging.info(f"Keeping {self.name}") else: - logging.debug(f"Removing {self.name}") + logging.info(f"Removing {self.name}") shutil.rmtree(self.name, ignore_errors=False, onerror=g__handleRemoveReadonly) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index 17ae992024..92cbd91308 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -30,9 +30,10 @@ ccc bbb ccc }) + uploadMinSnapshotsToS3.echo(Start copying files) uploadMinSnapshotsToS3.sh( - cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz - cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz + cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 ) uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) diff --git a/vars/createSha512Checksums.groovy b/vars/createSha512Checksums.groovy index 93dd170086..0fc2d2edc4 100644 --- a/vars/createSha512Checksums.groovy +++ b/vars/createSha512Checksums.groovy @@ -10,7 +10,7 @@ Closure call() { return { argsMap -> body: { - final foundFiles = sh(script: "find $argsMap.artifactPath -type f", returnStdout: true).split() + final foundFiles = sh(script: "find ${argsMap.artifactPath} -type f", returnStdout: true).split() for (file in foundFiles) { acceptTypeFound = false diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index be4d2a443f..a9524ff9ae 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -22,9 +22,12 @@ void call(Map args = [:]) { extension = buildManifest.build.getExtension() // Setup src & dst variables for artifacts - srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist" - dstDir = "snapshots/core/${productName}/${version}" - baseName = "${productName}-min-${version}-${platform}-${architecture}" + // Replace backslash with forward slash ('\' to '/') in path + // Compatible with both Windows as well as any nix* env + // Else the map in groovy will treat '\' as escape char on Windows + String srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist".replace("\\", "/") + String dstDir = "snapshots/core/${productName}/${version}" + String baseName = "${productName}-min-${version}-${platform}-${architecture}" // Create checksums echo('Create .sha512 for Min Snapshots Artifacts') @@ -34,9 +37,11 @@ void call(Map args = [:]) { action(argsMap) } + echo('Start copying files') + sh """ - cp ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension} - cp ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512 + cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension} + cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512 sed -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512 """ withCredentials([