Add Smoke Tests

Signed-off-by: Sophia Guo <[email protected]>
adoptium · Mar 9, 2021 · e3dee46 · e3dee46
1 parent ca9d2c1
commit e3dee46
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 19 deletions.
diff --git a/docs/UsingOurScripts.md b/docs/UsingOurScripts.md
@@ -14,6 +14,8 @@ This file contains the default constants and paths used in the build scripts for
         "build_url"          : "https://github.com/AdoptOpenJDK/openjdk-build.git",
         // Git branch you wish to use when running the shell scripts inside the build_url
         "build_branch"       : "master",
+        // Smoke tests directory under the build repo
+        "test_dirs"          : "/test/functional",
         // Git Url of the current repository.
         "pipeline_url"       : "https://github.com/AdoptOpenJDK/ci-jenkins-pipelines.git",
         // Git branch you wish to use when running the groovy scripts inside the pipeline_url

diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy
@@ -132,11 +132,45 @@ class Build {
     The test jobs all follow the same name naming pattern that is defined in the openjdk-tests repository.
     E.g. Test_openjdk11_hs_sanity.system_ppc64_aix
     */
-    def getTestJobParams(testType) {
-        def jobParams = [:]
+     def getSmokeTestJobParams() {
+        def jobParams = getCommonTestJobParams()
+        jobParams.put('LEVELS', "extend")
+        jobParams.put('GROUPS', "functional")
+        jobParams.put('TEST_JOB_NAME', "${env.JOB_NAME}_SmokeTests")
+        def useAdoptShellScripts = Boolean.valueOf(buildConfig.USE_ADOPT_SHELL_SCRIPTS)
+        def vendorTestRepos = ((String)ADOPT_DEFAULTS_JSON['repository']['build_url']).minus(".git")
+        def vendorTestBranches = ADOPT_DEFAULTS_JSON['repository']['build_branch']
+        def vendorTestDirs = ADOPT_DEFAULTS_JSON['repository']['test_dirs']
+        if (!useAdoptShellScripts) {
+            vendorTestRepos = ((String)DEFAULTS_JSON['repository']['build_url']).minus(".git")
+            vendorTestBranches = DEFAULTS_JSON['repository']['build_branch']
+            vendorTestDirs = DEFAULTS_JSON['repository']['test_dirs']
+        }                     
+        jobParams.put("VENDOR_TEST_REPOS", vendorTestRepos)
+        jobParams.put("VENDOR_TEST_BRANCHES", vendorTestBranches)
+        jobParams.put("VENDOR_TEST_DIRS", vendorTestDirs)
+        return jobParams
+    }
+
+    def getAQATestJobParams(testType) {
+        def jobParams = getCommonTestJobParams()
         def (level, group) = testType.tokenize('.')
         jobParams.put('LEVELS', level)
         jobParams.put('GROUPS', group)
+        def jobName = "Test_openjdk${jobParams['JDK_VERSIONS']}_${jobParams['JDK_IMPL']}_${testType}_${jobParams['ARCH_OS_LIST']}"
+        if (buildConfig.ADDITIONAL_FILE_NAME_TAG) {
+            switch (buildConfig.ADDITIONAL_FILE_NAME_TAG) {
+                case ~/.*XL.*/: 
+                    jobName += "_xl";
+                    break
+            }
+        }
+        jobParams.put('TEST_JOB_NAME', jobName)
+        return jobParams
+    }
+
+    def getCommonTestJobParams() {
+        def commonJobParams = [:]
         def number = getJavaVersionNumber()
         jobParams.put('JDK_VERSIONS', number)
         def variant
@@ -152,22 +186,18 @@ class Build {
         if (arch == "x64") {
             arch = "x86-64"
         }
-        def arch_os = "${arch}_${buildConfig.TARGET_OS}"    
-        def jobName = "Test_openjdk${number}_${variant}_${testType}_${arch_os}"
+        def arch_os = "${arch}_${buildConfig.TARGET_OS}"
         if (buildConfig.ADDITIONAL_FILE_NAME_TAG) {
             switch (buildConfig.ADDITIONAL_FILE_NAME_TAG) {
                 case ~/.*XL.*/: 
-                    jobName += "_xl";
                     arch_os += "_xl";
                     break
             }
         }
-        jobParams.put('TEST_JOB_NAME', jobName)
         jobParams.put('ARCH_OS_LIST', arch_os)
         jobParams.put('LIGHT_WEIGHT_CHECKOUT', true)
         return jobParams
     }
-
     /*
     Retrieve the corresponding OpenJDK source code repository branch. This is used the downstream tests to determine what source code branch the tests should run against.
     */
@@ -236,13 +266,49 @@ class Build {
 
         return jdkRepo
     }
+    /*
+    Run smoke tests, which should block the running downstream test jobs should there are failures. 
+    If a test job that doesn't exist, it will be created dynamically.
+    */
+    def runSmokeTests() {
+        def additionalTestLabel = buildConfig.ADDITIONAL_TEST_LABEL
+
+        try {
+            context.println "Running smoke test"
+            context.stage("smoke test") {
+                smokeTestJobParams = getSmokeTestJobParams()
+                def jobName = smokeTestJobParams.TEST_JOB_NAME
+                def JobHelper = context.library(identifier: 'openjdk-jenkins-helper@master').JobHelper
+                if (!JobHelper.jobIsRunnable(jobName as String)) {
+                    context.sh('curl -Os https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-tests/master/buildenv/jenkins/testJobTemplate')
+                    def templatePath = 'testJobTemplate'
+                    context.println "Smoke test job doesn't exist, create test job: ${jobName}"
+                    jobDsl targets: templatePath, ignoreExisting: false, additionalParameters: smokeTestJobParams
+                }
 
+                context.catchError {
+                    context.build job: jobName,
+                            propagate: false,
+                            parameters: [
+                                    context.string(name: 'UPSTREAM_JOB_NUMBER', value: "${env.BUILD_NUMBER}"),
+                                    context.string(name: 'UPSTREAM_JOB_NAME', value: "${env.JOB_NAME}"),
+                                    context.string(name: 'RELEASE_TAG', value: "${buildConfig.SCM_REF}"),
+                                    context.string(name: 'LABEL_ADDITION', value: additionalTestLabel),
+                                    context.string(name: 'KEEP_REPORTDIR', value: "${keep_test_reportdir}"),
+                                    context.string(name: 'ACTIVE_NODE_TIMEOUT', value: "${buildConfig.ACTIVE_NODE_TIMEOUT}")]
+                }
+            }
+        } catch (Exception e) {
+            context.println "Failed to execute test: ${e.message}"
+            context.println "Failed to execute test: ${e.getLocalizedMessage()}"
+            throw new Exception("[ERROR] Smoke Tests failed. Tests Stop. ")
+        }
+    }
     /*
     Run the downstream test jobs based off the configuration passed down from the top level pipeline jobs.
-    If we try to call a test job that doesn't exist, the pipeline will not fail but it will print out a warning.
-    If you need more test jobs added, please request so in #testing on Slack.
+    If a test job doesn't exist, it will be created dynamically. 
     */
-    def runTests() {
+    def runAQATests() {
         def testStages = [:]
         List testList = []
         def jdkBranch = getJDKBranch()
@@ -266,7 +332,7 @@ class Build {
                             keep_test_reportdir = "true"
                         }
 
-                        def jobParams = getTestJobParams(testType)
+                        def jobParams = getAQATestJobParams(testType)
                         def jobName = jobParams.TEST_JOB_NAME
                         def JobHelper = context.library(identifier: 'openjdk-jenkins-helper@master').JobHelper
 
@@ -1324,15 +1390,21 @@ class Build {
                         throw new Exception("[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...")
                     }
                 }
-
-                if (enableTests && buildConfig.TEST_LIST.size() > 0) {
-                    try {
-                        // Run tests if we have a test list, don't use timeouts as the jobs have their own
-                        def testStages = runTests()
-                        context.parallel testStages
-                    } catch (Exception e) {
-                        context.println "Failed test: ${e}"
+
+                // Run Smoke Tests and AQA Tests
+                try { 
+                    runSmokeTests()
+                    if (enableTests && buildConfig.TEST_LIST.size() > 0) {
+                        try {
+                            // Run tests if we have a test list, don't use timeouts as the jobs have their own
+                            def testStages = runAQATests()
+                            context.parallel testStages
+                        } catch (Exception e) {
+                        context.println (e.message)
+                        }
                     }
+                } catch (Exception e) {
+                    context.println(e.message)
                 }
 
                 //buildInstaller if needed

diff --git a/pipelines/defaults.json b/pipelines/defaults.json
@@ -2,6 +2,7 @@
     "repository"             : {
         "build_url"          : "https://github.com/AdoptOpenJDK/openjdk-build.git",
         "build_branch"       : "master",
+        "test_dirs"          : "/test/functional",
         "pipeline_url"       : "https://github.com/AdoptOpenJDK/ci-jenkins-pipelines.git",
         "pipeline_branch"    : "master"
     },