Skip to content

Commit

Permalink
[Build][Test] OS and OSD improvements plus support for Playground
Browse files Browse the repository at this point in the history 
While using a child folder 'Playground' in Jenkins I had to do a number of
changes to support a successful test run. I also included the ability to
skip publishing the notification not to spam channels from builds from
for example `Playground`. Also ability to skip building docker because
from what I could tell it was publishing docker images from `Playground`
pipelines as well.

Finally, some cleanup to integ tests to use the input manifest to container image.
Also, make integ tests run in docker container for OpenSearch.

Issues resolved:
opensearch-project#1688
opensearch-project#1687
opensearch-project#1686

Signed-off-by: Kawika Avilla <[email protected]>
  • Loading branch information
@kavilla
kavilla committed Mar 16, 2022
1 parent 947ab86 commit 10adc5f
Show file tree
Hide file tree
Showing 9 changed files with 313 additions and 81 deletions.
104 changes: 66 additions & 38 deletions jenkins/opensearch-dashboards/distribution-build.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ pipeline {
agent none
environment {
AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
INTEG_TEST_JOB_NAME = 'integ-test-opensearch-dashboards'
AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
DEFAULT_INTEG_TEST_JOB_NAME = 'integ-test-opensearch-dashboards'
}
parameters {
string(
Expand All @@ -23,8 +23,33 @@ pipeline {
description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.',
trim: true
)
string(
name: 'INTEG_TEST_JOB_NAME',
description: "Name of integration test job that will be triggered, e.g. Playground/integ-test-opensearch-dashboards.",
defaultValue: "integ-test-opensearch-dashboards",
trim: true
)
booleanParam(
name: 'BUILD_DOCKER',
description: 'Build docker image or not.',
defaultValue: true
)
booleanParam(
name: 'PUBLISH_NOTIFICATION',
description: 'Publish the status of this build job or not.',
defaultValue: true
)
}
stages {
stage('verify-parameters') {
steps {
script {
env.INTEG_TEST_JOB_NAME = INTEG_TEST_JOB_NAME != '' ?
INTEG_TEST_JOB_NAME :
DEFAULT_INTEG_TEST_JOB_NAME
}
}
}
stage('detect docker image + args') {
agent {
docker {
Expand Down Expand Up @@ -65,22 +90,20 @@ pipeline {
echo "artifactUrl (x64): ${artifactUrl}"

def integTestResults =
build job: INTEG_TEST_JOB_NAME,
build job: env.INTEG_TEST_JOB_NAME,
propagate: false,
wait: true,
parameters: [
string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST),
string(name: 'TEST_MANIFEST', value: TEST_MANIFEST),
string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl),
string(name: 'AGENT_LABEL', value: AGENT_X64),
string(name: 'CONTAINER_IMAGE', value: getTestDocker(testManifest: "manifests/${TEST_MANIFEST}")?.image ?: dockerAgent.image)
string(name: 'AGENT_LABEL', value: AGENT_X64)
]

String status = integTestResults.getResult()
String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:'
lib.jenkins.Messages.new(this).add(
STAGE_NAME,
lib.jenkins.Messages.new(this).get([STAGE_NAME]) +
"\nInteg Tests (x64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}"
createTestResultsMessage(
testType: "Integ Tests (x64)",
status: integTestResults.getResult(),
absoluteUrl: integTestResults.getAbsoluteUrl()
)
}
}
Expand Down Expand Up @@ -141,22 +164,20 @@ pipeline {
echo "artifactUrl (arm64): ${artifactUrl}"

def integTestResults =
build job: INTEG_TEST_JOB_NAME,
build job: env.INTEG_TEST_JOB_NAME,
propagate: false,
wait: true,
parameters: [
string(name: 'INPUT_MANIFEST', value: INPUT_MANIFEST),
string(name: 'TEST_MANIFEST', value: TEST_MANIFEST),
string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl),
string(name: 'AGENT_LABEL', value: AGENT_ARM64),
string(name: 'CONTAINER_IMAGE', value: getTestDocker(testManifest: "manifests/${TEST_MANIFEST}")?.image ?: dockerAgent.image)
string(name: 'AGENT_LABEL', value: AGENT_ARM64)
]

String status = integTestResults.getResult()
String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:'
lib.jenkins.Messages.new(this).add(
STAGE_NAME,
lib.jenkins.Messages.new(this).get([STAGE_NAME]) +
"\nInteg Tests (arm64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}"
createTestResultsMessage(
testType: "Integ Tests (arm64)",
status: integTestResults.getResult(),
absoluteUrl: integTestResults.getAbsoluteUrl()
)
}
}
Expand All @@ -171,6 +192,10 @@ pipeline {
}
}
stage('docker build') {
when {
beforeAgent true
equals expected: true, actual: BUILD_DOCKER
}
steps {
node(AGENT_X64) {
script {
Expand All @@ -191,19 +216,20 @@ pipeline {
success {
node(AGENT_X64) {
script {
def stashed = lib.jenkins.Messages.new(this).get([
'build-and-test-linux-x64',
'build-archive-linux-arm64',
'assemble-archive-and-test-linux-arm64'
])
if (params.PUBLISH_NOTIFICATION) {
def stashed = lib.jenkins.Messages.new(this).get([
'build-and-test-linux-x64',
'assemble-archive-and-test-linux-arm64'
])

publishNotification(
icon: ':white_check_mark:',
message: 'Successful Build',
extra: stashed,
credentialsId: 'BUILD_NOTICE_WEBHOOK',
manifest: "${INPUT_MANIFEST}"
)
publishNotification(
icon: ':white_check_mark:',
message: 'Successful Build',
extra: stashed,
credentialsId: 'BUILD_NOTICE_WEBHOOK',
manifest: "${INPUT_MANIFEST}"
)
}

postCleanup()
}
Expand All @@ -212,12 +238,14 @@ pipeline {
failure {
node(AGENT_X64) {
script {
publishNotification(
icon: ':warning:',
message: 'Failed Build',
credentialsId: 'BUILD_NOTICE_WEBHOOK',
manifest: "${INPUT_MANIFEST}"
)
if (params.PUBLISH_NOTIFICATION) {
publishNotification(
icon: ':warning:',
message: 'Failed Build',
credentialsId: 'BUILD_NOTICE_WEBHOOK',
manifest: "${INPUT_MANIFEST}"
)
}

postCleanup()
}
Expand Down
39 changes: 27 additions & 12 deletions jenkins/opensearch-dashboards/integ-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@ pipeline {
agent none
environment {
BUILD_MANIFEST = "build-manifest.yml"
BUILD_JOB_NAME = 'distribution-build-opensearch-dashboards'
DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch-dashboards"
}
parameters {
string(
name: 'INPUT_MANIFEST',
description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0.yml.',
trim: true
)
string(
name: 'TEST_MANIFEST',
description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.',
Expand All @@ -25,11 +30,6 @@ pipeline {
description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.',
trim: true
)
string(
name: 'CONTAINER_IMAGE',
description: 'The container image running on the agent where the tests should be executed, e.g. opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028.',
trim: true
)
}
stages {
stage('verify-parameters') {
Expand All @@ -39,18 +39,33 @@ pipeline {
currentBuild.result = 'ABORTED'
error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.")
}
if (CONTAINER_IMAGE == '') {
currentBuild.result = 'ABORTED'
error("Integration Tests failed to start. Missing parameter: CONTAINER_IMAGE.")
}
env.BUILD_JOB_NAME = currentBuild.upstreamBuilds ?
currentBuild.upstreamBuilds[0].fullProjectName :
env.DEFAULT_BUILD_JOB_NAME
}
}
}
stage('detect docker image + args') {
agent {
docker {
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
alwaysPull true
}
}
steps {
script {
currentBuild.description = "$INPUT_MANIFEST"
dockerAgent = detectDockerAgent()
}
}
}
stage('integ-test') {
agent {
docker {
label AGENT_LABEL
image CONTAINER_IMAGE
image dockerAgent.image
args dockerAgent.args
alwaysPull true
}
}
Expand All @@ -66,7 +81,7 @@ pipeline {
echo "BUILD_ID: ${BUILD_ID}"

runIntegTestScript(
jobName: BUILD_JOB_NAME,
jobName: env.BUILD_JOB_NAME,
buildManifest: BUILD_MANIFEST,
testManifest: "manifests/${TEST_MANIFEST}",
buildId: BUILD_ID
Expand Down
Loading

0 comments on commit 10adc5f

Please sign in to comment.