diff --git a/Makefile b/Makefile index e7ceec1..a52c7c5 100644 --- a/Makefile +++ b/Makefile @@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE) PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION) -all: requirements install-provisioner secrets init-terraform +all: roles install-provisioner secrets init-terraform @echo "Success!" -requirements-install: - ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml +roles-install: + ansible/roles.py --install -requirements-check: - ansible/versioncheck.py +roles-check: + ansible/roles.py --check -requirements: requirements-install requirements-check +roles-update: + ansible/roles.py --update + +roles: roles-install roles-check $(PROVISIONER_PATH): @mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \ diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 31e8b72..b80e1c3 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,24 +1,24 @@ ---- - name: infra-role-bootstrap-linux - src: git@github.com:status-im/infra-role-bootstrap-linux.git scm: git + src: git@github.com:status-im/infra-role-bootstrap-linux.git - name: infra-role-wireguard - src: git@github.com:status-im/infra-role-wireguard.git scm: git + src: git@github.com:status-im/infra-role-wireguard.git - name: infra-role-open-ports - src: git@github.com:status-im/infra-role-open-ports.git scm: git + src: git@github.com:status-im/infra-role-open-ports.git - name: infra-role-swap-file - src: git@github.com:status-im/infra-role-swap-file.git scm: git + src: git@github.com:status-im/infra-role-swap-file.git - name: infra-role-consul-service - src: git@github.com:status-im/infra-role-consul-service.git scm: git + src: git@github.com:status-im/infra-role-consul-service.git - name: infra-role-systemd-timer - src: git@github.com:status-im/infra-role-systemd-timer.git scm: git + src: git@github.com:status-im/infra-role-systemd-timer.git + version: dasdasd diff --git a/ansible/roles.py b/ansible/roles.py new file mode 100755 index 0000000..e9e3c61 --- /dev/null +++ b/ansible/roles.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# WARNING: If importing this fails set PYTHONPATH. +import pickle +import yaml +import logging +import ansible +import argparse +import subprocess +import functools +from enum import Enum +from os import path, environ +from collections import OrderedDict +from packaging.version import parse as version_parse +from concurrent import futures + +HELP_DESCRIPTION=''' +This tool managed Ansible roles as Git repositories. + +It is both faster and simpler than Ansible Galaxy. +''' +HELP_EXAMPLE='''Examples: +./roles.py --install +./roles.py --check +./roles.py --update +''' + + +SCRIPT_DIR = path.dirname(path.realpath(__file__)) +# Where Ansible looks for installed roles. +ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles') +REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml') +WORK_DIR = path.join(environ['HOME'], 'work') + +# Setup logging. +log_format = '[%(levelname)s] %(message)s' +logging.basicConfig(level=logging.INFO, format=log_format) +LOG = logging.getLogger(__name__) + +# Colors +rst = '\033[0m' +PURPLE = lambda x: f'\033[35m{x}{rst}' +YELLOW = lambda x: f'\033[33m{x}{rst}' +RED = lambda x: f'\033[31m{x}{rst}' +GREEN = lambda x: f'\033[32m{x}{rst}' +BLUE = lambda x: f'\033[36m{x}{rst}' +BOLD = lambda x: f'\033[1m{x}{rst}' + +class State(Enum): + # Order is priority. Higher status trumps lower. + UNKNOWN = 0 + EXISTS = 1 + WRONG_VERSION = 2 + DIRTY = 3 + NO_VERSION = 4 + CLONE_FAILURE = 5 + MISSING = 6 + CLONED = 7 + UPDATED = 8 + VALID = 9 + SKIPPED = 10 + + def __str__(self): + match self: + case State.CLONED: return GREEN(self.name) + case State.UPDATED: return GREEN(self.name) + case State.VALID: return GREEN(self.name) + case State.DIRTY: return YELLOW(self.name) + case State.NO_VERSION: return PURPLE(self.name) + case State.CLONE_FAILURE: return RED(self.name) + case State.WRONG_VERSION: return RED(self.name) + case State.MISSING: return RED(self.name) + case _: return self.name + + # Allow calling max() to compare with previous state. + def __gt__(self, other): + if other is None: + return True + if self.__class__ is other.__class__: + return self.value > other.value + return NotImplemented + + # Decorator to manage Role state based on function return value. + def update(success=None, failure=None): + def decorator(func): + @functools.wraps(func) + def wrapper_decorator(self, *args, **kwargs): + # Set state to failure one on exception. + try: + rval = func(self, *args, **kwargs) + except: + self.state = max(failure, self.state) + raise + # Set state based on truthiness of result, higher one wins. + if rval: + self.state = max(success, self.state) + else: + self.state = max(failure, self.state) + LOG.debug('[%-27s] - %s%s: state = %s', + self.name, func.__name__, args, self.state) + return rval + return wrapper_decorator + return decorator + +class Role: + + def __init__(self, name, src, required): + self.state = State.UNKNOWN + self.name = name + self.src = src + self.required = required + + @classmethod + def from_requirement(cls, obj): + return cls(obj['name'], obj.get('src'), obj.get('version')) + + def __repr__(self): + return 'Role(name=%s, src=%s, required=%s, state=%s)' % ( + self.name, self.src, self.required, self.state, + ) + + def to_dict(self): + obj = { + 'name': self.name, + 'src': self.src, + 'scm': 'git', + } + if self.required: + obj['version'] = self.required + return obj + + def _git(self, *args, cwd=None): + cmd = ['git'] + list(args) + LOG.debug('[%s]: COMMAND: %s', self.name, ' '.join(cmd)) + rval = subprocess.run( + cmd, + capture_output=True, + cwd=cwd or self.path + ) + LOG.debug('[%s]: RETURN: %d', self.name, rval.returncode) + if rval.stdout: + LOG.debug('[%s]: STDOUT: %s', self.name, rval.stdout.decode().strip()) + if rval.stderr: + LOG.debug('[%s]: STDERR: %s', self.name, rval.stderr.decode().strip()) + rval.check_returncode() + return str(rval.stdout.strip(), 'utf-8') + + @property + def repo_parent_dir(self): + return self.path.removesuffix(self.name) + + @property + def branch(self): + return self._git('rev-parse', '--abbrev-ref', 'HEAD') + + @property + def current_commit(self): + return self._git('rev-parse', 'HEAD') + + @State.update(success=State.DIRTY) + def is_dirty(self): + try: + self._git('diff-files', '--quiet') + except: + return True + else: + return False + + @property + @State.update(failure=State.NO_VERSION) + def version(self): + return self.required + + @version.setter + @State.update(success=State.UPDATED, failure=State.SKIPPED) + def version(self, version): + if self.required is not None: + self.required = version + return self.required + + @State.update(success=State.VALID, failure=State.WRONG_VERSION) + def valid_version(self): + return self.required == self.current_commit + + @State.update(success=State.UPDATED) + def pull(self): + return self._git('pull') + + @State.update(success=State.CLONED, failure=State.CLONE_FAILURE) + def clone(self): + LOG.debug('Clogning: %s', self.src) + try: + self._git( + 'clone', + self.src, self.name, + cwd=self.repo_parent_dir + ) + except Exception as ex: + LOG.error('Clone failed: %s', ex.stderr.decode()) + return False + return True + + @property + def path(self): + return path.join(ANSIBLE_ROLES_PATH, self.name) + + @State.update(success=State.EXISTS, failure=State.MISSING) + def exists(self): + return path.isdir(self.path) + + +def handle_role(role, check=False, update=False): + LOG.debug('[%s]: Processing role...', role.name) + if not role.exists(): + if not check and not update: + role.clone() + return role + + if role.valid_version(): + return role + + if role.is_dirty(): + return role + + # No need to fail if no version is set. + if not role.version and check: + return role + + if update: + role.version = role.current_commit + elif not check: + role.pull() + return role + + +# Special function to preserve order and separating newlines. +def roles_to_yaml(old_reqs, roles): + return '\n'.join([ + yaml.dump([roles[role['name']].to_dict()]) + for role in old_reqs + ]) + +def commit_or_any(commit): + return 'ANY' if commit is None else commit[:8] + +def parse_args(): + parser = argparse.ArgumentParser( + description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE + ) + + parser.add_argument('-f', '--filter', default='', + help='Filter role repo names.') + parser.add_argument('-w', '--workers', default=20, type=int, + help='Max workers to run in parallel.') + parser.add_argument('-l', '--log-level', default='INFO', + help='Logging level.') + parser.add_argument('-d', '--fail-dirty', action='store_true', + help='Fail if repo is dirty.') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-i', '--install', action='store_true', + help='Clone and update required roles.') + group.add_argument('-c', '--check', action='store_true', + help='Only check roles, no installing.') + group.add_argument('-u', '--update', action='store_true', + help='Update requirements with current commits.') + + args = parser.parse_args() + + assert args.install or args.check or args.update, \ + parser.error('Pick one: --install, --check, --update') + + return args + + +def main(): + args = parse_args() + + LOG.setLevel(args.log_level.upper()) + + # Verify Ansible version is 2.8 or newer. + if version_parse(ansible.__version__) < version_parse("2.8"): + LOG.error('Your Ansible version is lower than 2.8. Upgrade it.') + exit(1) + + # Read Ansible requirements file. + with open(REQUIREMENTS_PATH, 'r') as f: + requirements = yaml.load(f, Loader=yaml.FullLoader) + + roles = [ + Role.from_requirement(req) + for req in requirements + if args.filter in req['name'] + ] + + # Check if each Ansible role is installed and has correct version. + with futures.ProcessPoolExecutor(max_workers=args.workers) as executor: + these_futures = [ + executor.submit(handle_role, role, args.check, args.update) + for role in roles + ] + + roles = { + r.name: r for r in + [r.result() for r in futures.as_completed(these_futures)] + } + + for req in requirements: + role = roles[req['name']] + print('%-38s --- %22s (Git: %s | Req: %s)' % + (BOLD(role.name), role.state, + BLUE(role.current_commit[:8]), + commit_or_any(role.required))) + + if args.update: + with open(REQUIREMENTS_PATH, 'w') as f: + f.write(roles_to_yaml(requirements, roles)) + + fail_states = set([ + State.MISSING, + State.WRONG_VERSION + ]) + if args.fail_dirty: + fail_states.append(State.DIRTY) + if fail_states.intersection([r.state for r in roles.values()]): + exit(1) + +if __name__ == "__main__": + main() diff --git a/ansible/versioncheck.py b/ansible/versioncheck.py deleted file mode 100755 index d88a47e..0000000 --- a/ansible/versioncheck.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# WARNING: If importing this fails set PYTHONPATH. -import yaml -import ansible -import subprocess -from os import path, environ -from packaging import version - -SCRIPT_DIR = path.dirname(path.realpath(__file__)) -# Where Ansible looks for installed roles. -ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles') - - -class Role: - def __init__(self, name, version): - self.name = name - self.version = version - - @property - def path(self): - return path.join(ANSIBLE_ROLES_PATH, self.name) - - def exists(self): - return path.isdir(self.path) - - def local_version(self): - cmd = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - capture_output=True, - cwd=self.path - ) - cmd.check_returncode() - return str(cmd.stdout.strip(), 'utf-8') - - -# Verify Ansible version is 2.8 or newer. -if version.parse(ansible.__version__) < version.parse("2.8"): - print('Your Ansible version is lower than 2.8. Upgrade it.') - exit(1) - -# Read Ansible requirements file. -with open(path.join(SCRIPT_DIR, 'requirements.yml'), 'r') as f: - requirements = yaml.load(f, Loader=yaml.FullLoader) - -# Check if each Ansible role is installed and has correct version. -errors = 0 -for req in requirements: - role = Role(req['name'], req.get('version')) - - if not role.exists(): - print('%25s - MISSING!' % role.name) - errors += 1 - continue - - # For now we allow not specifying versions for everyhing. - if role.version is None: - print('%25s - No version!' % role.name) - continue - - local_version = role.local_version() - if role.version != local_version: - print('%25s - MISMATCH: %s != %s' % - (role.name, role.version[:8], local_version[:8])) - errors += 1 - continue - - print('%25s - VALID' % role.name) - -# Any issue with any role should cause failure. -if errors > 0: - exit(1)