Re-adding the windows snapshot with new changes (#2550)

* Re-adding the windows snapshot with new changes Signed-off-by: Peter Zhu <[email protected]> * Update manifests Signed-off-by: Peter Zhu <[email protected]> * Add more tests Signed-off-by: Peter Zhu <[email protected]> * Restore Pipefile.lock Signed-off-by: Peter Zhu <[email protected]> * Tweak comments Signed-off-by: Peter Zhu <[email protected]> Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Sep 2, 2022 · e1c5883 · e1c5883
1 parent b657fba
commit e1c5883
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 44 deletions.
diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -199,37 +199,37 @@ pipeline {
                         }
                     }
                 }
-                //stage('build-snapshot-windows-x64-tar') {
-                //    agent {
-                //        node {
-                //            label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host'
-                //        }
-                //    }
-                //    tools {
-                //        jdk dockerAgent.javaVersion
-                //    }
-                //    steps {
-                //        script {
-                //            buildManifest(
-                //                componentName: "${COMPONENT_NAME}",
-                //                inputManifest: "manifests/${INPUT_MANIFEST}",
-                //                distribution: 'zip',
-                //                platform: 'windows',
-                //                snapshot: true
-                //            )
-                //            echo("Uploading windows min snapshots to S3")
-                //            uploadMinSnapshotsToS3(
-                //                fileActions: [createSha512Checksums()],
-                //                distribution: 'zip'
-                //            )
-                //        }
-                //    }
-                //    post {
-                //        always {
-                //            postCleanup()
-                //        }
-                //    }
-                //}
+                stage('build-snapshot-windows-x64-zip') {
+                    agent {
+                        node {
+                            label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host'
+                        }
+                    }
+                    tools {
+                        jdk dockerAgent.javaVersion
+                    }
+                    steps {
+                        script {
+                            buildManifest(
+                                componentName: "${COMPONENT_NAME}",
+                                inputManifest: "manifests/${INPUT_MANIFEST}",
+                                distribution: 'zip',
+                                platform: 'windows',
+                                snapshot: true
+                            )
+                            echo("Uploading windows min snapshots to S3")
+                            uploadMinSnapshotsToS3(
+                                fileActions: [createSha512Checksums()],
+                                distribution: 'zip'
+                            )
+                        }
+                    }
+                    post {
+                        always {
+                            postCleanup()
+                        }
+                    }
+                }
                 stage('build-and-test-x64-tar') {
                     agent {
                         docker {

diff --git a/src/system/temporary_directory.py b/src/system/temporary_directory.py
@@ -19,17 +19,17 @@ def g__handleRemoveReadonly(func: FunctionType, path: str, exc: Any) -> Any:
     logging.debug(f"excvalue {excvalue}")
     logging.debug(f"func {func.__name__}")
     if func in (os.rmdir, os.remove, os.unlink):
-        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IWRITE | stat.S_IREAD)  # 0777 nix* / +wr windows
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IREAD | stat.S_IWRITE)  # 0777 nix* / +rw windows
         retry_total = 3
         for retry_count in range(retry_total):  # Re-run func to force deletion especially on windows
             try:
-                logging.debug(f'Try count: {retry_count + 1}/{retry_total}')
+                logging.warn(f'Removing try count: {retry_count + 1}/{retry_total} for {path}')
                 func_result = func(path)
+                logging.debug(f'func_result: {func_result}')
                 if func_result is None:
                     break
             except Exception as ex:
-                logging.warn(func_result)
-                logging.warn(ex)
+                logging.warn(f'Exception: {ex}')
     else:
         raise
 
@@ -58,5 +58,5 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
         if self.keep:
             logging.info(f"Keeping {self.name}")
         else:
-            logging.debug(f"Removing {self.name}")
+            logging.info(f"Removing {self.name}")
             shutil.rmtree(self.name, ignore_errors=False, onerror=g__handleRemoveReadonly)
diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
@@ -30,9 +30,10 @@ ccc
 bbb
 ccc
 })
+                  uploadMinSnapshotsToS3.echo(Start copying files)
                   uploadMinSnapshotsToS3.sh(
-        cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz
-        cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
+        cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz
+        cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
         sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
     )
                   uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})

diff --git a/vars/createSha512Checksums.groovy b/vars/createSha512Checksums.groovy
@@ -10,7 +10,7 @@ Closure call() {
 
     return { argsMap -> body: {
 
-        final foundFiles = sh(script: "find $argsMap.artifactPath -type f", returnStdout: true).split()
+        final foundFiles = sh(script: "find ${argsMap.artifactPath} -type f", returnStdout: true).split()
 
         for (file in foundFiles) {
             acceptTypeFound = false

diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy
@@ -22,9 +22,12 @@ void call(Map args = [:]) {
     extension = buildManifest.build.getExtension()
 
     // Setup src & dst variables for artifacts
-    srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist"
-    dstDir = "snapshots/core/${productName}/${version}"
-    baseName = "${productName}-min-${version}-${platform}-${architecture}"
+    // Replace backslash with forward slash ('\' to '/') in path
+    // Compatible with both Windows as well as any nix* env
+    // Else the map in groovy will treat '\' as escape char on Windows
+    String srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist".replace("\\", "/")
+    String dstDir = "snapshots/core/${productName}/${version}"
+    String baseName = "${productName}-min-${version}-${platform}-${architecture}"
 
     // Create checksums
     echo('Create .sha512 for Min Snapshots Artifacts')
@@ -34,9 +37,11 @@ void call(Map args = [:]) {
         action(argsMap)
     }
 
+    echo('Start copying files')
+
     sh """
-        cp ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
-        cp ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
+        cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
+        cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
         sed -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
     """
     withCredentials([