roles.py: script to replace Ansible Galaxy

Usage:
```
 > ansible/roles.py --help
usage: roles.py [-h] [-f FILTER] [-w WORKERS] [-l LOG_LEVEL] [-d] [-i | -c | -u]

This tool managed Ansible roles as Git repositories. It is both faster and simpler than Ansible Galaxy.

options:
  -h, --help            show this help message and exit
  -f FILTER, --filter FILTER
                        Filter role repo names.
  -w WORKERS, --workers WORKERS
                        Max workers to run in parallel.
  -l LOG_LEVEL, --log-level LOG_LEVEL
                        Logging level.
  -d, --fail-dirty      Fail if repo is dirty.
  -i, --install         Clone and update required roles.
  -c, --check           Only check roles, no installing.
  -u, --update          Update requirements with current commits.

Examples: ./roles.py --install ./roles.py --check ./roles.py --update
```

Signed-off-by: Jakub Sokołowski <[email protected]>

Makefile

@@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER
 PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE)
 PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION)
 
-all: requirements install-provisioner secrets init-terraform
+all: roles install-provisioner secrets init-terraform
 	@echo "Success!"
 
-requirements-install:
-	ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml
+roles-install:
+	ansible/roles.py --install
 
-requirements-check:
-	ansible/versioncheck.py
+roles-check:
+	ansible/roles.py --check
 
-requirements: requirements-install requirements-check
+roles-update:
+	ansible/roles.py --update
+
+roles: roles-install roles-check
 
 $(PROVISIONER_PATH):
 	@mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \

ansible/requirements.yml

@@ -1,24 +1,24 @@
----
 - name: infra-role-bootstrap-linux
-  src: [email protected]:status-im/infra-role-bootstrap-linux.git
   scm: git
+  src: [email protected]:status-im/infra-role-bootstrap-linux.git
 
 - name: infra-role-wireguard
-  src: [email protected]:status-im/infra-role-wireguard.git
   scm: git
+  src: [email protected]:status-im/infra-role-wireguard.git
 
 - name: infra-role-open-ports
-  src: [email protected]:status-im/infra-role-open-ports.git
   scm: git
+  src: [email protected]:status-im/infra-role-open-ports.git
 
 - name: infra-role-swap-file
-  src: [email protected]:status-im/infra-role-swap-file.git
   scm: git
+  src: [email protected]:status-im/infra-role-swap-file.git
 
 - name: infra-role-consul-service
-  src: [email protected]:status-im/infra-role-consul-service.git
   scm: git
+  src: [email protected]:status-im/infra-role-consul-service.git
 
 - name: infra-role-systemd-timer
-  src: [email protected]:status-im/infra-role-systemd-timer.git
   scm: git
+  src: [email protected]:status-im/infra-role-systemd-timer.git
+  version: 50575b23eb71b6d838a87eec92e4094b0f34dccd

ansible/roles.py

@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+# WARNING: If importing this fails set PYTHONPATH.
+import pickle
+import yaml
+import logging
+import ansible
+import argparse
+import subprocess
+import functools
+from enum import Enum
+from os import path, environ
+from collections import OrderedDict
+from packaging.version import parse as version_parse
+from concurrent import futures
+
+HELP_DESCRIPTION='''
+This tool managed Ansible roles as Git repositories.
+
+It is both faster and simpler than Ansible Galaxy.
+'''
+HELP_EXAMPLE='''Examples:
+./roles.py --install
+./roles.py --check
+./roles.py --update
+'''
+
+SCRIPT_DIR = path.dirname(path.realpath(__file__))
+# Where Ansible looks for installed roles.
+ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles')
+REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml')
+WORK_DIR = path.join(environ['HOME'], 'work')
+
+# Setup logging.
+log_format = '[%(levelname)s] %(message)s'
+logging.basicConfig(level=logging.INFO, format=log_format)
+LOG = logging.getLogger(__name__)
+
+# Colors
+RST = '\033[0m'
+NORMAL = lambda x: f'\033[00m{x}{RST}'
+PURPLE = lambda x: f'\033[35m{x}{RST}'
+YELLOW = lambda x: f'\033[33m{x}{RST}'
+RED    = lambda x: f'\033[31m{x}{RST}'
+GREEN  = lambda x: f'\033[32m{x}{RST}'
+BLUE   = lambda x: f'\033[36m{x}{RST}'
+BOLD   = lambda x: f'\033[1m{x}{RST}'
+
+class State(Enum):
+    # Order is priority. Higher status trumps lower.
+    UNKNOWN       = 0
+    EXISTS        = 1
+    WRONG_VERSION = 2
+    DIRTY         = 3
+    NO_VERSION    = 4
+    CLONE_FAILURE = 5
+    MISSING       = 6
+    CLONED        = 7
+    UPDATED       = 8
+    PULLED        = 9
+    VALID         = 10
+    SKIPPED       = 11
+
+    def __str__(self):
+        match self:
+            case State.CLONED:        return GREEN(self.name)
+            case State.UPDATED:       return GREEN(self.name)
+            case State.VALID:         return GREEN(self.name)
+            case State.DIRTY:         return YELLOW(self.name)
+            case State.NO_VERSION:    return PURPLE(self.name)
+            case State.CLONE_FAILURE: return RED(self.name)
+            case State.WRONG_VERSION: return RED(self.name)
+            case State.MISSING:       return RED(self.name)
+            case _:                   return NORMAL(self.name)
+
+    # Allow calling max() to compare with previous state.
+    def __gt__(self, other):
+        if other is None:
+            return True
+        if self.__class__ is other.__class__:
+            return self.value > other.value
+        return NotImplemented
+
+    # Decorator to manage Role state based on function return value.
+    def update(success=None, failure=None):
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper_decorator(self, *args, **kwargs):
+                # Set state to failure one on exception.
+                try:
+                    rval = func(self, *args, **kwargs)
+                except:
+                    self.state = max(failure, self.state)
+                    raise
+                # Set state based on truthiness of result, higher one wins.
+                if rval:
+                    self.state = max(success, self.state)
+                else:
+                    self.state = max(failure, self.state)
+                LOG.debug('[%-27s] - %s%s: state = %s',
+                          self.name, func.__name__, args, self.state)
+                return rval
+            return wrapper_decorator
+        return decorator
+
+class Role:
+
+    def __init__(self, name, src, required):
+        self.state = State.UNKNOWN
+        self.name = name
+        self.src = src
+        self.required = required
+
+    @classmethod
+    def from_requirement(cls, obj):
+        return cls(obj['name'], obj.get('src'), obj.get('version'))
+
+    def __repr__(self):
+        return 'Role(name=%s, src=%s, required=%s, state=%s)' % (
+            self.name, self.src, self.required, self.state,
+        )
+
+    def to_dict(self):
+        obj = {
+            'name': self.name,
+            'src':  self.src,
+            'scm':  'git',
+        }
+        if self.required:
+            obj['version'] = self.required
+        return obj
+
+    def _git(self, *args, cwd=None):
+        cmd = ['git'] + list(args)
+        LOG.debug('[%s]: COMMAND: %s', self.name, ' '.join(cmd))
+        rval = subprocess.run(
+            cmd,
+            capture_output=True,
+            cwd=cwd or self.path
+        )
+        LOG.debug('[%s]: RETURN: %d', self.name, rval.returncode)
+        if rval.stdout:
+            LOG.debug('[%s]: STDOUT: %s', self.name, rval.stdout.decode().strip())
+        if rval.stderr:
+            LOG.debug('[%s]: STDERR: %s', self.name, rval.stderr.decode().strip())
+        rval.check_returncode()
+        return str(rval.stdout.strip(), 'utf-8')
+
+    @property
+    def repo_parent_dir(self):
+        return self.path.removesuffix(self.name)
+
+    @property
+    def branch(self):
+        return self._git('rev-parse', '--abbrev-ref', 'HEAD')
+
+    @property
+    def current_commit(self):
+        return self._git('rev-parse', 'HEAD')
+
+    @State.update(success=State.DIRTY)
+    def is_dirty(self):
+        try:
+            self._git('diff-files', '--quiet')
+        except:
+            return True
+        else:
+            return False
+
+    @property
+    @State.update(failure=State.NO_VERSION)
+    def version(self):
+        return self.required
+
+    @version.setter
+    @State.update(success=State.UPDATED, failure=State.SKIPPED)
+    def version(self, version):
+        if self.required is not None:
+            self.required = version
+        return self.required
+
+    @State.update(success=State.VALID, failure=State.WRONG_VERSION)
+    def valid_version(self):
+        return self.required == self.current_commit
+
+    @State.update(success=State.PULLED, failure=State.VALID)
+    def pull(self):
+        status = self._git('remote', 'update')
+        status = self._git('status', '--untracked-files=no')
+        if 'branch is behind' in status:
+            return self._git('pull')
+        else:
+            return None
+
+    @State.update(success=State.CLONED, failure=State.CLONE_FAILURE)
+    def clone(self):
+        LOG.debug('Clogning: %s', self.src)
+        try:
+            self._git(
+                'clone',
+                self.src, self.name,
+                cwd=self.repo_parent_dir
+            )
+        except Exception as ex:
+            LOG.error('Clone failed: %s', ex.stderr.decode())
+            return False
+        return True
+
+    @property
+    def path(self):
+        return path.join(ANSIBLE_ROLES_PATH, self.name)
+
+    @State.update(success=State.EXISTS, failure=State.MISSING)
+    def exists(self):
+        return path.isdir(self.path)
+
+
+def handle_role(role, check=False, update=False):
+    LOG.debug('[%s]: Processing role...', role.name)
+    if not role.exists():
+        if not check and not update:
+            role.clone()
+        return role
+
+    # Check if current version matches required.
+    if role.valid_version():
+        return role
+
+    # Verify if git repo is not dirty.
+    if role.is_dirty():
+        return role
+
+    # No need to fail if no version is set.
+    if not role.version and check:
+        return role
+
+    # Update config version or pull new changes.
+    if update:
+        role.version = role.current_commit
+    elif not check:
+        # If version is not specified we just want the newest.
+        role.pull()
+    return role
+
+
+# Special function to preserve order and separating newlines.
+def roles_to_yaml(old_reqs, roles):
+    return '\n'.join([
+        yaml.dump([roles[role['name']].to_dict()])
+        for role in old_reqs
+    ])
+
+def commit_or_any(commit):
+    return 'ANY' if commit is None else commit[:8]
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=HELP_DESCRIPTION, epilog=HELP_EXAMPLE
+    )
+
+    parser.add_argument('-f', '--filter', default='',
+                        help='Filter role repo names.')
+    parser.add_argument('-w', '--workers', default=20, type=int,
+                        help='Max workers to run in parallel.')
+    parser.add_argument('-l', '--log-level', default='INFO',
+                        help='Logging level.')
+    parser.add_argument('-d', '--fail-dirty', action='store_true',
+                       help='Fail if repo is dirty.')
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('-i', '--install', action='store_true',
+                       help='Clone and update required roles.')
+    group.add_argument('-c', '--check', action='store_true',
+                       help='Only check roles, no installing.')
+    group.add_argument('-u', '--update', action='store_true',
+                       help='Update requirements with current commits.')
+
+    args = parser.parse_args()
+
+    assert args.install or args.check or args.update, \
+        parser.error('Pick one: --install, --check, --update')
+
+    return args
+
+
+def main():
+    args = parse_args()
+
+    LOG.setLevel(args.log_level.upper())
+
+    # Verify Ansible version is 2.8 or newer.
+    if version_parse(ansible.__version__) < version_parse("2.8"):
+        LOG.error('Your Ansible version is lower than 2.8. Upgrade it.')
+        exit(1)
+
+    # Read Ansible requirements file.
+    with open(REQUIREMENTS_PATH, 'r') as f:
+        requirements = yaml.load(f, Loader=yaml.FullLoader)
+
+    roles = [
+        Role.from_requirement(req)
+        for req in requirements
+        if args.filter in req['name']
+    ]
+
+    # Check if each Ansible role is installed and has correct version.
+    with futures.ProcessPoolExecutor(max_workers=args.workers) as executor:
+        these_futures = [
+            executor.submit(handle_role, role, args.check, args.update)
+            for role in roles
+        ]
+        # Wait for all the workers to finishe and return their role.
+        roles = {
+            r.name: r for r in
+            [r.result() for r in futures.as_completed(these_futures)]
+        }
+
+    # Use the same order as requirements.yml file.
+    for req in requirements:
+        role = roles[req['name']]
+        print('%-38s --- %22s (Git: %s | Req: %s)' %
+              (BOLD(role.name), role.state,
+               BLUE(role.current_commit[:8]),
+               commit_or_any(role.required)))
+
+    if args.update:
+        with open(REQUIREMENTS_PATH, 'w') as f:
+            f.write(roles_to_yaml(requirements, roles))
+
+    fail_states = set([State.MISSING, State.WRONG_VERSION])
+    if args.fail_dirty:
+        fail_states.append(State.DIRTY)
+    if fail_states.intersection([r.state for r in roles.values()]):
+        exit(1)
+
+if __name__ == "__main__":
+    main()