Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow user to specify branch of systemtests being triggered by trigger_systemtests.py #143

Draft
wants to merge 3 commits into
base: develop
Choose a base branch
from
Draft

Allow user to specify branch of systemtests being triggered by trigger_systemtests.py #143

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb83675
add --branch argument and reformat code
BenjaminRodenberg Dec 18, 2019
55ffcc0
Add missing branch argument
BenjaminRodenberg Dec 18, 2019
779eba9
Provide branch information to mock request
BenjaminRodenberg Jan 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 106 additions & 104 deletions trigger_systemtests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,28 @@

adapter_info = namedtuple('adapter_info', 'repo tests base')

adapters_info = {"openfoam": adapter_info('openfoam-adapter', ['of-of', 'of-ccx'], 'Ubuntu1604.home'),
"calculix": adapter_info('calculix-adapter', ['of-ccx','su2-ccx'], 'Ubuntu1604.home'),
"su2": adapter_info('su2-adapter', ['su2-ccx'], 'Ubuntu1604.home'),
"dealii": adapter_info('dealii-adapter', ['dealii-of'], 'Ubuntu1604.home'),
"fenics": adapter_info('fenics-adapter', ['fe-fe'], 'Ubuntu1804.home')}
adapters_info = {"openfoam": adapter_info('openfoam-adapter', ['of-of', 'of-ccx'], 'Ubuntu1604.home'),
"calculix": adapter_info('calculix-adapter', ['of-ccx', 'su2-ccx'], 'Ubuntu1604.home'),
"su2": adapter_info('su2-adapter', ['su2-ccx'], 'Ubuntu1604.home'),
"dealii": adapter_info('dealii-adapter', ['dealii-of'], 'Ubuntu1604.home'),
"fenics": adapter_info('fenics-adapter', ['fe-fe'], 'Ubuntu1804.home')}

def get_json_response(url, **kwargs):

def get_json_response(url, **kwargs):
headers = {
"Content-Type" : "application/json",
"Accept": "application/json",
"Travis-API-Version": "3",
"Authorization": "token {}".format(os.environ['TRAVIS_ACCESS_TOKEN'])
"Content-Type": "application/json",
"Accept": "application/json",
"Travis-API-Version": "3",
"Authorization": "token {}".format(os.environ['TRAVIS_ACCESS_TOKEN'])
}

req = Request(url, headers = headers, **kwargs )
response = urlopen( req ).read().decode()
json_response = json.loads( response )
req = Request(url, headers=headers, **kwargs)
response = urlopen(req).read().decode()
json_response = json.loads(response)

return json_response


def adjust_travis_script(script, user, adapter):
""" Patches travis job in case we are running on fork or a different branch """

Expand All @@ -64,16 +65,15 @@ def adjust_travis_script(script, user, adapter):
if branch or not pull_req in [None, "false"]:
preprocess_cmd = "grep -rl --include=\*Dockerfile* github.com/{user}/{adapter}.git |\
xargs sed -i 's|\(github.com/{user}/{adapter}.git\)|\\1 \&\& cd \
{adapter} \&\& {post_clone_cmd} \&\& cd .. |g'".format(user = user, adapter =
adapters_info[adapter].repo, post_clone_cmd = post_clone_cmd)
{adapter} \&\& {post_clone_cmd} \&\& cd .. |g'".format(user=user, adapter=
adapters_info[adapter].repo, post_clone_cmd=post_clone_cmd)

main_script = " && ".join(filter(None, ([ preprocess_cmd, script ])))
main_script = " && ".join(filter(None, ([preprocess_cmd, script])))

return main_script


def determine_image_tag():

""" Generates tag information based on the branch and pull request names """

branch = os.environ.get("TRAVIS_BRANCH", "latest")
Expand All @@ -83,78 +83,81 @@ def determine_image_tag():
else:
return branch

def generate_travis_job(adapter, user, trigger_failure = True):

triggered_by = os.environ["TRAVIS_JOB_WEB_URL"] if "TRAVIS_JOB_WEB_URL" in\
os.environ else "manual script call"
def generate_travis_job(adapter, user, trigger_failure=True, branch="master"):
triggered_by = os.environ["TRAVIS_JOB_WEB_URL"] if "TRAVIS_JOB_WEB_URL" in \
os.environ else "manual script call"

base = adapters_info[adapter].base

after_failure_action = "python push.py -t {TEST} --base {BASE} ;"
main_test_script = "python system_testing.py -s {TEST} --base {BASE}"

base_remote = "precice/precice-{base}-develop".format(base = base.lower())
base_remote = "precice/precice-{base}-develop".format(base=base.lower())
main_build_script = "docker build -f adapters/Dockerfile.{adapter} -t \
{user}/{adapter}:{tag} --build-arg from={base_remote} .".format(adapter =
adapters_info[adapter].repo, user = user, base_remote =
base_remote, tag = determine_image_tag())
{user}/{adapter}:{tag} --build-arg from={base_remote} .".format(adapter=
adapters_info[
adapter].repo,
user=user, base_remote=
base_remote,
tag=determine_image_tag())

if trigger_failure:
after_failure_action += " python trigger_systemtests.py --failure --owner {USER} --adapter {ADAPTER}"

after_failure_action += " python trigger_systemtests.py --failure --owner {USER} --adapter {ADAPTER}" + "--branch {}".format(os.environ["TRAVIS_BRANCH"])

# template for building this particular adapter
build_template = {
"stage": "Building adapter",
"name": adapters_info[adapter].repo,
"script": adjust_travis_script(main_build_script, user, adapter),
"after_success":
[ 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin',
"docker push {user}/{adapter}:{tag}".format(adapter =
adapters_info[adapter].repo, user = user,tag = determine_image_tag()) ]
}
['echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin',
"docker push {user}/{adapter}:{tag}".format(adapter=
adapters_info[adapter].repo, user=user,
tag=determine_image_tag())]
}

# template for actually runnig an adapter in combination with other
# adapters
# template for actually running an adapter in combination with other adapters
systest_templates = {
"stage": "Running tests",
"name": "[{BASE}] {TESTNAME} <-> {TESTNAME}",
"name": "[{BASE}] {TESTNAME} <-> {TESTNAME}",
# force docker-compose to consider an image with a particular tag
"script": "export ${adapter_tag}={tag}; ".format(adapter_tag = adapter.upper() + "_TAG", tag = determine_image_tag()) \
+ main_test_script,
"script": "export ${adapter_tag}={tag}; ".format(adapter_tag=adapter.upper() + "_TAG",
tag=determine_image_tag()) \
+ main_test_script,
"after_success": "python push.py -s -t {TEST}",
"after_failure": after_failure_action
};
}

job_body={
job_body = {
"request": {
"message": "{} systemtests".format(adapter),
"branch": "master",
"config": {
# we need to use 'replace' to replace .travis.yml,
# that is originally present in the repo
"merge_mode": "replace",
"sudo": "true",
"dist": "trusty",
"language": "python",
"services": "docker",
"python": "3.5",
"jobs": {
"include":[
]
}
"message": "{} systemtests".format(adapter),
"branch": "{}".format(branch),
"config": {
# we need to use 'replace' to replace .travis.yml,
# that is originally present in the repo
"merge_mode": "replace",
"sudo": "true",
"dist": "trusty",
"language": "python",
"services": "docker",
"python": "3.5",
"jobs": {
"include": [
]
}
}
}
}
}

# generate jobs body for the request, build of an adapter
# should be first stage
jobs = [build_template]
for tests in adapters_info[adapter].tests:
job = {}
for key,systest_template in systest_templates.items():
for key, systest_template in systest_templates.items():
job[key] = systest_template.format(TESTNAME=tests, USER=user, TEST=tests,
ADAPTER=adapter, BASE=base)
ADAPTER=adapter, BASE=base)
jobs.append(job)

job_body["request"]["message"] = "{} systemtest. Triggered by:{}".format(adapter, triggered_by)
Expand All @@ -169,44 +172,46 @@ def trigger_travis_build(job_body, user, repo):
"""

url = "https://api.{TRAVIS_URL}/repo/{USER}%2F{REPO}/requests".format(TRAVIS_URL="travis-ci.org",
USER=user,REPO=repo)
USER=user, REPO=repo)

data = json.dumps(job_body).encode('utf8')

return get_json_response(url, data = data)
return get_json_response(url, data=data)


def check_job_status(job_id):
""" Checks status of the travis job"""
url = "https://api.{TRAVIS_URL}/build/{JOB_ID}".format(TRAVIS_URL="travis-ci.org", JOB_ID=job_id)

resp = get_json_response(url)
resp = get_json_response(url)
return resp['state']

def get_requests(user, repo):

def get_requests(user, repo):
url = "https://api.{TRAVIS_URL}/repo/{USER}%2F{REPO}/requests".format(TRAVIS_URL="travis-ci.org",
USER=user, REPO=repo )
USER=user, REPO=repo)
return get_json_response(url)

def query_request_info(user, repo, req_id):

def query_request_info(user, repo, req_id):
url = "https://api.{TRAVIS_URL}/repo/{USER}%2F{REPO}/request/{REQUEST_ID}".format(TRAVIS_URL="travis-ci.org",
USER=user, REPO=repo, REQUEST_ID=req_id)
USER=user, REPO=repo,
REQUEST_ID=req_id)
return get_json_response(url)

def trigger_travis_and_wait_and_respond(job_body, user, repo):

def trigger_travis_and_wait_and_respond(job_body, user, repo):
json_response = trigger_travis_build(job_body, user, repo)
request_id = json_response['request']['id']

request_info = query_request_info(user, repo, request_id)
# it case request is being processes slow for whatever reasons
# (e.g) multiple requests were triggered, we don't yet have
# info about job_id, so we'll wait until we have it
print ("Request pending..")
print("Request pending..")
while request_info.get('state') == 'pending':
# for ther reference in case of failures
print ("Current request status is {}".format(request_info['state']))
# for their reference in case of failures
print("Current request status is {}".format(request_info['state']))
request_info = query_request_info(user, repo, request_id)
time.sleep(60)

Expand All @@ -221,10 +226,10 @@ def trigger_travis_and_wait_and_respond(job_body, user, repo):
success_status = ["passed", "canceled"]
failed_status = ["errored", "failed"]

print ("Job started..")
while not job_status in (success_status + failed_status):
print("Job started..")
while job_status not in (success_status + failed_status):
job_status = check_job_status(build_id)
print ("Current job status is {}. Be patient...".format(job_status))
print("Current job status is {}. Be patient...".format(job_status))
time.sleep(60)

if job_status in success_status:
Expand All @@ -233,59 +238,56 @@ def trigger_travis_and_wait_and_respond(job_body, user, repo):
exit(1)


def generate_failure_callback():
def generate_failure_callback(branch="master"):
""" Generates travis job body, that simply fails without
running anything
"""

triggered_by = os.environ["TRAVIS_JOB_WEB_URL"];

callback_body={
"request": {
"message": "Systemtests failed. Build url:{}".format(triggered_by),
"branch": "master",
"config": {
"merge_mode": "replace",
"sudo": "true",
"dist": "trusty",
"jobs": {
"include":{
"name": "Systemtest failure callback",
"script": "false"
triggered_by = os.environ["TRAVIS_JOB_WEB_URL"]

callback_body = {
"request": {
"message": "Systemtests failed. Build url:{}".format(triggered_by),
"branch": "{}".format(branch),
"config": {
"merge_mode": "replace",
"sudo": "true",
"dist": "trusty",
"jobs": {
"include": {
"name": "Systemtest failure callback",
"script": "false"
}
}
}
}
}
}}
}}

return callback_body


if __name__ == "__main__":

parser = argparse.ArgumentParser(description="Generate and trigger job for systemtests")
parser.add_argument('--owner', type=str, help="Owner of repository", default='precice' )
parser.add_argument('--adapter', type=str, help="Adapter for which you want to trigger systemtests",
required=True, choices = adapters_info.keys() )
parser.add_argument('--failure', help="Whether to trigger normal or failure build",
action="store_true")
parser.add_argument('--wait', help='Whether exit only when the triggered build succeeds',
action='store_true')
parser.add_argument('--test', help='Only print generated job, do not send the request',
action='store_true')
parser.add_argument('--owner', type=str, help="Owner of repository", default='precice')
parser.add_argument('--adapter', type=str, help="Adapter for which you want to trigger systemtests", required=True,
choices=adapters_info.keys())
parser.add_argument('--failure', help="Whether to trigger normal or failure build", action="store_true")
parser.add_argument('--wait', help='Whether exit only when the triggered build succeeds', action='store_true')
parser.add_argument('--test', help='Only print generated job, do not send the request', action='store_true')
parser.add_argument('--branch', help='Branch of systemtests being used for testing', default='master')
args = parser.parse_args()

if args.failure:
repo = adapters_info[ args.adapter ].repo
trigger_travis_build( generate_failure_callback(), args.owner,repo)
repo = adapters_info[args.adapter].repo
trigger_travis_build(generate_failure_callback(branch=args.branch), args.owner, repo)
Copy link
Contributor

@Eder-K Eder-K Dec 30, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This conditional branch that contains this line is only ever reached in the following case:

  1. Some external repository, let's say fenics-adapter, commits something that triggers Travis to execute the .travis.yml file. This file contains the crucial line python trigger_systemtests.py --adapter fenics, which triggers a build on systemtests (and with this PR merged, on a specific branch of it). Note that the --wait argument is omitted, meaning that the fenics-adapter build on Travis will just send the request and instantly return as success.

  2. A systemtests build is launched to run the test that was requested. If this test now fails we reach the after_failure_action

    systemtests/trigger_systemtests.py

    Lines 105 to 106 in 55ffcc0

    if trigger_failure:
    after_failure_action += " python trigger_systemtests.py --failure --owner {USER} --adapter {ADAPTER}"

    So, the systemtests build will send a request back to fenics-adapter which originally triggered the whole sequence.

At this point we have reached the highlighted line 282.

  1. On fenics-adapter, a mock build is launched. This build will just fail and do nothing else, so that the actual feedback for a failed test is negative on the original repository as well.

systemtests/trigger_systemtests.py

Line 282 in 55ffcc0

trigger_travis_build(generate_failure_callback(branch=args.branch), args.owner, repo)

Because of this, we have to be careful with which branch we choose here, as this must not be a branch of systemtests. Currently, this mock build would always run on the master branch of the fenics-adapter. It would be nice to have this actually run on the specific branch that was originally commited to in point 1. Note that this is again a different branch than the systemtests branch that will be adjustable with this PR.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the mock request is generated on systemtests, we would need it to have received information in some way about which branch of the outside repository triggered it. One way might be to have the original request contain the following as after failure action

if trigger_failure:
    after_failure_action += " python trigger_systemtests.py --failure --owner {USER} --adapter {ADAPTER}" + "--branch {}".format(os.environ["TRAVIS_BRANCH"])

This way, when systemtests is executing trigger_systemtests.py as a result of a failed test, it will have received an args.branch value that correctly contains the branch of the outside repo that originally issued the test. Then we could actually call generate_failure_callback(branch=args.branch)

This way of formatting seems rather ugly, but it doesn't interfere with the formatting step done later on in

systemtests/trigger_systemtests.py

Lines 158 to 160 in 55ffcc0

for key, systest_template in systest_templates.items():
job[key] = systest_template.format(TESTNAME=tests, USER=user, TEST=tests,
ADAPTER=adapter, BASE=base)

We also don't require any further variables this way.

Suggestions?

else:
if args.test:
job = generate_travis_job(args.adapter, args.owner, trigger_failure
= False)
job = generate_travis_job(args.adapter, args.owner, trigger_failure=False, branch=args.branch)
pprint.pprint(job)
else:
if args.wait:
trigger_travis_and_wait_and_respond(generate_travis_job(args.adapter, args.owner, trigger_failure
= False), args.owner, 'systemtests' )
trigger_travis_and_wait_and_respond(generate_travis_job(args.adapter, args.owner, trigger_failure=False,
branch=args.branch), args.owner, 'systemtests')
else:
trigger_travis_build( generate_travis_job(args.adapter, args.owner),
args.owner, 'systemtests' )
trigger_travis_build(generate_travis_job(args.adapter, args.owner, branch=args.branch), args.owner,
'systemtests')