From 64d4c9e02ff59596a7c91155e03dd09bad2f9385 Mon Sep 17 00:00:00 2001 From: floweisshardt Date: Fri, 15 Jan 2021 14:15:20 +0100 Subject: [PATCH 1/4] add run_github_actions as equivalent to run_travis --- industrial_ci/scripts/run_github_actions | 217 +++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100755 industrial_ci/scripts/run_github_actions diff --git a/industrial_ci/scripts/run_github_actions b/industrial_ci/scripts/run_github_actions new file mode 100755 index 000000000..be92e575d --- /dev/null +++ b/industrial_ci/scripts/run_github_actions @@ -0,0 +1,217 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2017, Mathias Lüdtke +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import difflib +import os +import os.path +import re +import subprocess +import sys + +import yaml + +# find config file either in first argument or curret working directory +def find_config_file(args, names): + def test_file(dir): # test if one of the file names is present + for n in names: + p = os.path.join(dir, n) + if os.path.isfile(p): + return os.path.abspath(p) + return None + + if len(args) > 0: # test first argument if available + arg0=args[0] + if os.path.isfile(arg0): + return os.path.abspath(arg0),args[1:] + p = test_file(arg0) + if p is not None: + return p, args[1:] + return test_file(os.getcwd()), args + +# parse range tuples from arguments +def parse_ranges(args, offset=0): + r = [] + def apply_offset(i): # adjust for user offsets + ao = int(i) + offset + if ao < 0: + raise ValueError("%s is not in supported range" % str(i)) + return ao + + for a in args: + if '-' in a: # range string + p1, p2 = a.split('-', 2) + if p1 == '' and len(r) == 0: # -X + p1 = 0 + else: # X-[Y] + p1 = apply_offset(p1) + if p2 == '': # [X]- + r.append((p1, -1)) + break + r.append((p1, apply_offset(p2)+1)) # X-Y + else: + i = apply_offset(a) + r.append((i, i+1)) + return r + +def apply_ranges(ranges, num): + for start,end in ranges: + if end == -1: + end = num + for i in range(start,end): + yield i + + +def read_yaml(p): + with open(p) as f: + return yaml.safe_load(f) + +# read global and job-specific envs from +def read_job(config): + job_envs = [] + global_env = '' + if isinstance(config, dict): + for l in config['strategy']['matrix']['env']: + job_env = '' + for key, value in l.items(): + job_env += key+"='"+str(value) + "' " # adds a space to every element + job_env = job_env[:-1] # remove space from last element + job_envs.append(job_env) + for key, value in config['env'].items(): + global_env += " "+key+"='"+str(value)+"'" + return global_env, job_envs + +def read_allow_failures(config): + try: + af = config['matrix']['allow_failures'] + except: + return list() + return list(x['env'] for x in af) + +def read_num_include(config): + try: + return len(config['matrix']['include']) + except: + return 0 + +def parse_extra_args(args): + try: + extra = args.index('--') + return args[0:extra], args[extra+1:] + except ValueError: + return args, [] + +env_assigment = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*=") +def gen_env(e): + if env_assigment.match(e): + return e + return '%s=%s' % (e, os.getenv(e, '')) + +# from https://github.com/travis-ci/travis-build/blob/73bf69a439bb546520a5e5b6b6847fb5424a7c9f/lib/travis/build/env/var.rb#L5 +travis_env = re.compile(r"([\w]+)=((?:[^\"'`\ ]?(\"|'|`).*?((? 0): + print('Globals: %s' % str(highlight_diff(global_env))) + jobs = len(job_envs) + digits = len(str(jobs)) + for i in range(jobs): + print('Job %s%s: %s' % ( str(i+1).rjust(digits), + ' (allow_failures)' if job_envs[i] in allow_failures else '', + highlight_diff(job_envs[i]) if job_envs[i] is not None else "")) + print("run all with %s -" % sys.argv[0]) + sys.exit(0) + + ranges = parse_ranges(args, -1) + + run_ci = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), filter_env(global_env)] + run_ci_diff = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), highlight_diff(global_env, 44)] + + bash = ['env', '-i'] +list(map(gen_env, ['DOCKER_PORT', 'HOME', 'PATH', 'SSH_AUTH_SOCK', 'TERM'])) + ['bash','-e'] + + selection = set(apply_ranges(ranges, len(job_envs))) + + for i in selection: + if job_envs[i] is None: + print("\033[1;43mSkipped job %d, because jobs from 'include' section are not supported\033[1;m" %(i+1,)) + continue + cmd = ' '.join(run_ci + [filter_env(job_envs[i])] + list(map(gen_env, extra_env))) + cmd_diff = ' '.join(run_ci_diff + [highlight_diff(job_envs[i], 44)] + list(map(gen_env, extra_env))) + print('\033[1;44mRunning job %d%s: %s\033[1;m' %(i+1, ' (allow_failures)' if job_envs[i] in allow_failures else '', cmd_diff)) + + proc = subprocess.Popen(bash, stdin=subprocess.PIPE) + proc.communicate(cmd.encode()) + if proc.returncode: + print('\033[1;41mFailed job %d: %s\033[1;m' %(i+1, cmd)) + if job_envs[i] not in allow_failures: + sys.exit(proc.returncode) + +if __name__ == "__main__": + main(os.path.abspath(os.path.dirname(__file__)), sys.argv) From abc6437a2669cab752b9b763d8dd235cfb317392 Mon Sep 17 00:00:00 2001 From: floweisshardt Date: Fri, 15 Jan 2021 17:29:54 +0100 Subject: [PATCH 2/4] accept capital True and False --- industrial_ci/src/util.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/industrial_ci/src/util.sh b/industrial_ci/src/util.sh index 069801093..341848eac 100644 --- a/industrial_ci/src/util.sh +++ b/industrial_ci/src/util.sh @@ -492,8 +492,12 @@ function ici_parse_jobs { _ici_parse_jobs_res="$3";; "true") _ici_parse_jobs_res="0";; + "True") + _ici_parse_jobs_res="0";; "false") _ici_parse_jobs_res="1";; + "False") + _ici_parse_jobs_res="1";; *) if ! [[ "$_ici_parse_jobs_res" =~ ^[0-9]+$ ]]; then ici_error "cannot parse $2=$_ici_parse_jobs_res as a number" From b710728a80a9ce2d3f34cd541b1816934a2a0db8 Mon Sep 17 00:00:00 2001 From: fmessmer Date: Thu, 7 Oct 2021 17:01:29 +0200 Subject: [PATCH 3/4] use python module structure for run_gha --- industrial_ci/CMakeLists.txt | 2 +- .../industrial_ci/common.py} | 69 ---------- industrial_ci/python/industrial_ci/gha.py | 97 ++++++++++++++ industrial_ci/python/industrial_ci/travis.py | 120 +----------------- industrial_ci/scripts/run_gha | 29 +++++ 5 files changed, 128 insertions(+), 189 deletions(-) rename industrial_ci/{scripts/run_github_actions => python/industrial_ci/common.py} (58%) mode change 100755 => 100644 create mode 100644 industrial_ci/python/industrial_ci/gha.py create mode 100755 industrial_ci/scripts/run_gha diff --git a/industrial_ci/CMakeLists.txt b/industrial_ci/CMakeLists.txt index dbca39770..b9b4b6d72 100644 --- a/industrial_ci/CMakeLists.txt +++ b/industrial_ci/CMakeLists.txt @@ -22,7 +22,7 @@ else() message(FATAL_ERROR "ROS_VERSION is neither 1 nor 2") endif() -install(PROGRAMS scripts/run_ci scripts/rerun_ci scripts/run_travis +install(PROGRAMS scripts/run_ci scripts/rerun_ci scripts/run_travis scripts/run_gha DESTINATION ${${PROJECT_NAME}_BIN_DESTINATION} ) diff --git a/industrial_ci/scripts/run_github_actions b/industrial_ci/python/industrial_ci/common.py old mode 100755 new mode 100644 similarity index 58% rename from industrial_ci/scripts/run_github_actions rename to industrial_ci/python/industrial_ci/common.py index be92e575d..90adac226 --- a/industrial_ci/scripts/run_github_actions +++ b/industrial_ci/python/industrial_ci/common.py @@ -82,21 +82,6 @@ def read_yaml(p): with open(p) as f: return yaml.safe_load(f) -# read global and job-specific envs from -def read_job(config): - job_envs = [] - global_env = '' - if isinstance(config, dict): - for l in config['strategy']['matrix']['env']: - job_env = '' - for key, value in l.items(): - job_env += key+"='"+str(value) + "' " # adds a space to every element - job_env = job_env[:-1] # remove space from last element - job_envs.append(job_env) - for key, value in config['env'].items(): - global_env += " "+key+"='"+str(value)+"'" - return global_env, job_envs - def read_allow_failures(config): try: af = config['matrix']['allow_failures'] @@ -161,57 +146,3 @@ def print_help(cmd): Additional variable names can be passed at the end. """ % cmd) - -def main(scripts_dir, argv): - if '--help' in argv: - print_help(argv[0]) - sys.exit(0) - - args, extra_env = parse_extra_args(argv[1:]) - - path, args = find_config_file(args, ['.github/workflows/industrial_ci_action.yml','.github/workflows/industrial_ci_action.yaml']) - config = read_yaml(path) - global_env, job_envs = read_job(config['jobs']['industrial_ci']) - allow_failures = read_allow_failures(config) - job_envs = [ x for x in job_envs if x not in allow_failures ] \ - + [None] * read_num_include(config) \ - + [ x for x in job_envs if x in allow_failures ] - - if len(args) == 0: - if(len(global_env) > 0): - print('Globals: %s' % str(highlight_diff(global_env))) - jobs = len(job_envs) - digits = len(str(jobs)) - for i in range(jobs): - print('Job %s%s: %s' % ( str(i+1).rjust(digits), - ' (allow_failures)' if job_envs[i] in allow_failures else '', - highlight_diff(job_envs[i]) if job_envs[i] is not None else "")) - print("run all with %s -" % sys.argv[0]) - sys.exit(0) - - ranges = parse_ranges(args, -1) - - run_ci = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), filter_env(global_env)] - run_ci_diff = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), highlight_diff(global_env, 44)] - - bash = ['env', '-i'] +list(map(gen_env, ['DOCKER_PORT', 'HOME', 'PATH', 'SSH_AUTH_SOCK', 'TERM'])) + ['bash','-e'] - - selection = set(apply_ranges(ranges, len(job_envs))) - - for i in selection: - if job_envs[i] is None: - print("\033[1;43mSkipped job %d, because jobs from 'include' section are not supported\033[1;m" %(i+1,)) - continue - cmd = ' '.join(run_ci + [filter_env(job_envs[i])] + list(map(gen_env, extra_env))) - cmd_diff = ' '.join(run_ci_diff + [highlight_diff(job_envs[i], 44)] + list(map(gen_env, extra_env))) - print('\033[1;44mRunning job %d%s: %s\033[1;m' %(i+1, ' (allow_failures)' if job_envs[i] in allow_failures else '', cmd_diff)) - - proc = subprocess.Popen(bash, stdin=subprocess.PIPE) - proc.communicate(cmd.encode()) - if proc.returncode: - print('\033[1;41mFailed job %d: %s\033[1;m' %(i+1, cmd)) - if job_envs[i] not in allow_failures: - sys.exit(proc.returncode) - -if __name__ == "__main__": - main(os.path.abspath(os.path.dirname(__file__)), sys.argv) diff --git a/industrial_ci/python/industrial_ci/gha.py b/industrial_ci/python/industrial_ci/gha.py new file mode 100644 index 000000000..1dc6b659b --- /dev/null +++ b/industrial_ci/python/industrial_ci/gha.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2017, Mathias Lüdtke +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import difflib +import os +import os.path +import re +import subprocess +import sys + +import yaml + +from industrial_ci.common import * + +# read global and job-specific envs from +def read_env(env): + m = [] + g = '' + if isinstance(env, dict): + for l in env['strategy']['matrix']['include']: + job_env = '' + for key, value in l.items(): + job_env += key+"='"+str(value) + "' " # adds a space to every element + job_env = job_env[:-1] # remove space from last element + m.append(job_env) + for key, value in env['env'].items(): + g += " "+key+"='"+str(value)+"'" + return g, m + +def main(scripts_dir, argv): + if '--help' in argv: + print_help(argv[0]) + sys.exit(0) + + args, extra_env = parse_extra_args(argv[1:]) + + path, args = find_config_file(args, ['.github/workflows/main.yml','.github/workflows/main.yaml']) + config = read_yaml(path) + global_env, job_envs = read_env(config['jobs']['industrial_ci']) + allow_failures = read_allow_failures(config) + job_envs = [ x for x in job_envs if x not in allow_failures ] \ + + [None] * read_num_include(config) \ + + [ x for x in job_envs if x in allow_failures ] + + if len(args) == 0: + if(len(global_env) > 0): + print('Globals: %s' % str(highlight_diff(global_env))) + jobs = len(job_envs) + digits = len(str(jobs)) + for i in range(jobs): + print('Job %s%s: %s' % ( str(i+1).rjust(digits), + ' (allow_failures)' if job_envs[i] in allow_failures else '', + highlight_diff(job_envs[i]) if job_envs[i] is not None else "")) + print("run all with %s -" % sys.argv[0]) + sys.exit(0) + + ranges = parse_ranges(args, -1) + + run_ci = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), filter_env(global_env)] + run_ci_diff = [os.path.join(scripts_dir, "run_ci"), os.path.dirname(path), highlight_diff(global_env, 44)] + + bash = ['env', '-i'] +list(map(gen_env, ['DOCKER_PORT', 'HOME', 'PATH', 'SSH_AUTH_SOCK', 'TERM'])) + ['bash','-e'] + + selection = set(apply_ranges(ranges, len(job_envs))) + + for i in selection: + if job_envs[i] is None: + print("\033[1;43mSkipped job %d, because jobs from 'include' section are not supported\033[1;m" %(i+1,)) + continue + cmd = ' '.join(run_ci + [filter_env(job_envs[i])] + list(map(gen_env, extra_env))) + cmd_diff = ' '.join(run_ci_diff + [highlight_diff(job_envs[i], 44)] + list(map(gen_env, extra_env))) + print('\033[1;44mRunning job %d%s: %s\033[1;m' %(i+1, ' (allow_failures)' if job_envs[i] in allow_failures else '', cmd_diff)) + + proc = subprocess.Popen(bash, stdin=subprocess.PIPE) + proc.communicate(cmd.encode()) + if proc.returncode: + print('\033[1;41mFailed job %d: %s\033[1;m' %(i+1, cmd)) + if job_envs[i] not in allow_failures: + sys.exit(proc.returncode) + diff --git a/industrial_ci/python/industrial_ci/travis.py b/industrial_ci/python/industrial_ci/travis.py index 8494bb51e..e8ff3daa9 100644 --- a/industrial_ci/python/industrial_ci/travis.py +++ b/industrial_ci/python/industrial_ci/travis.py @@ -27,60 +27,7 @@ import yaml -# find config file either in first argument or curret working directory -def find_config_file(args, names): - def test_file(dir): # test if one of the file names is present - for n in names: - p = os.path.join(dir, n) - if os.path.isfile(p): - return os.path.abspath(p) - return None - - if len(args) > 0: # test first argument if available - arg0=args[0] - if os.path.isfile(arg0): - return os.path.abspath(arg0),args[1:] - p = test_file(arg0) - if p is not None: - return p, args[1:] - return test_file(os.getcwd()), args - -# parse range tuples from arguments -def parse_ranges(args, offset=0): - r = [] - def apply_offset(i): # adjust for user offsets - ao = int(i) + offset - if ao < 0: - raise ValueError("%s is not in supported range" % str(i)) - return ao - - for a in args: - if '-' in a: # range string - p1, p2 = a.split('-', 2) - if p1 == '' and len(r) == 0: # -X - p1 = 0 - else: # X-[Y] - p1 = apply_offset(p1) - if p2 == '': # [X]- - r.append((p1, -1)) - break - r.append((p1, apply_offset(p2)+1)) # X-Y - else: - i = apply_offset(a) - r.append((i, i+1)) - return r - -def apply_ranges(ranges, num): - for start,end in ranges: - if end == -1: - end = num - for i in range(start,end): - yield i - - -def read_yaml(p): - with open(p) as f: - return yaml.safe_load(f) +from industrial_ci.common import * # read global and job-specific envs from def read_env(env): @@ -92,71 +39,6 @@ def read_env(env): g = ' '.join(env['global']) return g, m -def read_allow_failures(config): - try: - af = config['matrix']['allow_failures'] - except: - return list() - return list(x['env'] for x in af) - -def read_num_include(config): - try: - return len(config['matrix']['include']) - except: - return 0 - -def parse_extra_args(args): - try: - extra = args.index('--') - return args[0:extra], args[extra+1:] - except ValueError: - return args, [] - -env_assigment = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*=") -def gen_env(e): - if env_assigment.match(e): - return e - return '%s=%s' % (e, os.getenv(e, '')) - -# from https://github.com/travis-ci/travis-build/blob/73bf69a439bb546520a5e5b6b6847fb5424a7c9f/lib/travis/build/env/var.rb#L5 -travis_env = re.compile(r"([\w]+)=((?:[^\"'`\ ]?(\"|'|`).*?((? Date: Thu, 7 Oct 2021 17:32:25 +0200 Subject: [PATCH 4/4] Revert "accept capital True and False" This reverts commit 2dc2ac56774bea4eded1095b2bb08df869aa27fa. --- industrial_ci/src/util.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/industrial_ci/src/util.sh b/industrial_ci/src/util.sh index 341848eac..069801093 100644 --- a/industrial_ci/src/util.sh +++ b/industrial_ci/src/util.sh @@ -492,12 +492,8 @@ function ici_parse_jobs { _ici_parse_jobs_res="$3";; "true") _ici_parse_jobs_res="0";; - "True") - _ici_parse_jobs_res="0";; "false") _ici_parse_jobs_res="1";; - "False") - _ici_parse_jobs_res="1";; *) if ! [[ "$_ici_parse_jobs_res" =~ ^[0-9]+$ ]]; then ici_error "cannot parse $2=$_ici_parse_jobs_res as a number"