From 211d1d67fbcb1a7beb6027b6e92fa5d6bd0581dd Mon Sep 17 00:00:00 2001 From: Chris Houseknecht Date: Fri, 19 May 2017 06:05:16 -0400 Subject: [PATCH] Deploy bug fixes and clean up --- .travis.yml | 2 +- conductor-requirements.txt | 4 +- container/cli.py | 25 ++-- container/config.py | 18 ++- container/core.py | 109 +++++++++--------- container/docker/engine.py | 46 +++++--- container/k8s/base_deploy.py | 5 +- container/k8s/base_engine.py | 43 ++++--- container/k8s/engine.py | 23 ---- container/utils/__init__.py | 17 ++- .../validate-build-and-import/tasks/main.yml | 7 ++ test/tests/validate_build.py | 2 +- test/tests/validate_config.py | 2 +- 13 files changed, 163 insertions(+), 140 deletions(-) diff --git a/.travis.yml b/.travis.yml index c62f6979..2a4e40ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ services: before_install: - sudo apt-add-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe' - sudo apt-get update -qq - - sudo apt-get install -y shellcheck + # - sudo apt-get install -y shellcheck - pip install --upgrade setuptools script: diff --git a/conductor-requirements.txt b/conductor-requirements.txt index 11813d17..58dc657f 100644 --- a/conductor-requirements.txt +++ b/conductor-requirements.txt @@ -1,5 +1,5 @@ -#https://github.com/ansible/ansible/archive/devel.tar.gz -ansible>=2.3.0 +https://github.com/ansible/ansible/archive/devel.tar.gz +# ansible>=2.3.0 https://github.com/openshift/openshift-restclient-python/archive/master.tar.gz#egg=openshift-1.0.0 PyYAML>=3.12 docker-compose>=1.7 diff --git a/container/cli.py b/container/cli.py index fedb77e9..8d1a665d 100644 --- a/container/cli.py +++ b/container/cli.py @@ -19,7 +19,6 @@ from container.config import AnsibleContainerConductorConfig from container.utils import list_to_ordereddict -from collections import OrderedDict from logging import config LOGGING = { 'version': 1, @@ -91,29 +90,26 @@ def subcmd_common_parsers(self, parser, subparser, cmd): help=u'If authentication with the registry is required, provide a valid password.', dest='password', default=None) subparser.add_argument('--push-to', action='store', - help=(u'Name of a registry defined in container.yml or the actual URL of the registry, ' - u'including the namespace. If passing a URL, an example would be: ' - u'"https://registry.example.com:5000/myproject"'), + help=(u'Name of a registry defined in container.yml, or a registry URL. When ' + u'providing a URL, include the repository or project namespace.'), dest='push_to', default=None) subparser.add_argument('--tag', action='store', help=u'Tag the images before pushing.', dest='tag', default=None) - def subcmd_init_parser(self, parser, subparser): subparser.add_argument('--server', '-s', action='store', default='https://galaxy.ansible.com/', help=u'Use a different Galaxy server URL') subparser.add_argument('project', nargs='?', action='store', - help=u'Use a project template instead of making a ' - u'blank project from an Ansible Container project ' - u'from Ansible Galaxy.') + help=(u'Rather than starting with a blank project, use a project template ' + u'from an Ansible Container project downloaded from the Ansible Galaxy ' + u'web site.')) subparser.add_argument('--force', '-f', action='store_true', help=u'Overrides the requirement that init be run' u'in an empty directory, for example' u'if a virtualenv exists in the directory.') - def subcmd_build_parser(self, parser, subparser): subparser.add_argument('--flatten', action='store_true', help=u'By default, Ansible Container will add a single ' @@ -264,14 +260,19 @@ def __call__(self): LOGGING['loggers']['container']['level'] = 'DEBUG' config.dictConfig(LOGGING) + vargs = vars(args) + if not vargs['project_name']: + # Set the default project_name + vargs['project_name'] = os.path.basename(vargs['base_path']) + try: - getattr(core, u'hostcmd_{}'.format(args.subcommand))(**vars(args)) + getattr(core, u'hostcmd_{}'.format(args.subcommand))(**vargs) except exceptions.AnsibleContainerAlreadyInitializedException as e: - logger.error('{0}'.format(e), exc_info=False) + logger.error("Project already initialized. Use the --force option.") sys.exit(1) except exceptions.AnsibleContainerNotInitializedException: logger.error('No Ansible Container project data found - do you need to ' - 'run "ansible-container init"?', exc_info=False) + 'run "ansible-container init"?', exc_info=False) sys.exit(1) except exceptions.AnsibleContainerNoAuthenticationProvidedException: logger.error('No authentication provided, unable to continue', exc_info=False) diff --git a/container/config.py b/container/config.py index ce63d271..1c1d1734 100644 --- a/container/config.py +++ b/container/config.py @@ -44,18 +44,28 @@ class AnsibleContainerConfig(Mapping): base_path = None @container.host_only - def __init__(self, base_path, var_file=None, engine_name=None): + def __init__(self, base_path, var_file=None, engine_name=None, project_name=None): self.base_path = base_path self.var_file = var_file self.engine_name = engine_name self.config_path = path.join(self.base_path, 'container.yml') + self.project_name = project_name self.set_env('prod') @property def deployment_path(self): dep_path = self.get('settings', yaml.compat.ordereddict()).get('deployment_output_path', path.join(self.base_path, 'ansible-deployment/')) - return path.normpath(path.abspath(path.expanduser(dep_path))) + return path.normpath(path.abspath(path.expanduser(path.expandvars(dep_path)))) + + @property + def image_namespace(self): + # When pushing images or deploying, we need to know the registry namespace + namespace = self.project_name + if self.engine_name in ('k8s', 'openshift'): + if self._config.get('settings', {}).get('k8s_namespace', {}).get('name'): + namespace = self._config['settings']['k8s_namespace']['name'] + return namespace def set_env(self, env): """ @@ -106,7 +116,7 @@ def set_env(self, env): config['volumes'][vol_key] = docker_settings elif self.engine_name in ('openshift', 'k8s'): # For openshift/k8s remove unrelated attributes - for vol_key in config['volumes'].keys(): + for vol_key in list(config['volumes'].keys()): for engine_key in list(config['volumes'][vol_key].keys()): if engine_key != self.engine_name: del config['volumes'][vol_key][engine_key] @@ -273,7 +283,7 @@ def _process_top_level_sections(self): self._config['settings'] = self._config.get('settings', yaml.compat.ordereddict()) for section in ['volumes', 'registries']: logger.debug('Processing section...', section=section) - setattr(self, section, self._process_section(self._config.get(section, yaml.compat.ordereddict()))) + setattr(self, section, dict(self._process_section(self._config.get(section, yaml.compat.ordereddict())))) def _process_services(self): services = yaml.compat.ordereddict() diff --git a/container/core.py b/container/core.py index ef9229aa..469426bd 100644 --- a/container/core.py +++ b/container/core.py @@ -46,11 +46,12 @@ @host_only def hostcmd_init(base_path, project=None, **kwargs): + force = kwargs.pop('force') + container_cfg = os.path.join(base_path, 'container.yml') + if os.path.exists(container_cfg) and not force: + raise AnsibleContainerAlreadyInitializedException() + if project: - force = kwargs.pop('force') - if os.listdir(base_path) and not force: - raise AnsibleContainerAlreadyInitializedException( - u'The init command can only be run in an empty directory.') try: namespace, name = project.split('.', 1) except ValueError: @@ -107,11 +108,6 @@ def hostcmd_init(base_path, project=None, **kwargs): tar_obj.extract(member, base_path) logger.info(u'Ansible Container initialized from Galaxy container app %r', project) else: - container_cfg = os.path.join(base_path, 'container.yml') - if os.path.exists(container_cfg): - raise AnsibleContainerAlreadyInitializedException() - if not os.path.exists(base_path): - os.mkdir(base_path) template_dir = os.path.join(jinja_template_path(), 'init') context = { u'ansible_container_version': __version__, @@ -131,9 +127,9 @@ def hostcmd_init(base_path, project=None, **kwargs): @host_only def hostcmd_build(base_path, project_name, engine_name, var_file=None, **kwargs): - config = get_config(base_path, var_file=var_file, engine_name=engine_name) + config = get_config(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) engine_obj = load_engine(['BUILD', 'RUN'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) conductor_container_id = engine_obj.get_container_id_for_service('conductor') @@ -169,27 +165,29 @@ def hostcmd_build(base_path, project_name, engine_name, var_file=None, def hostcmd_deploy(base_path, project_name, engine_name, var_file=None, cache=True, **kwargs): assert_initialized(base_path) - logger.debug('Got extra args to `deploy` command', arguments=kwargs) - config = get_config(base_path, var_file=var_file, engine_name=engine_name) + config = get_config(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) local_images = kwargs.get('local_images') output_path = kwargs.pop('deployment_output_path', None) or config.deployment_path engine_obj = load_engine(['LOGIN', 'PUSH', 'DEPLOY'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) params = { - 'deployment_output_path': os.path.normpath(os.path.expanduser(output_path)), + 'deployment_output_path': os.path.normpath(os.path.abspath(os.path.expanduser(output_path))), 'host_user_uid': os.getuid(), 'host_user_gid': os.getgid(), } if config.get('settings', {}).get('k8s_auth'): params['k8s_auth'] = config['settings']['k8s_auth'] + if config.get('settings', {}).get('k8s_namespace'): + params['k8s_namespace'] = config['settings']['k8s_namespace'] if kwargs: params.update(kwargs) if not local_images: - url, namespace = push_images(base_path, engine_obj, config, save_conductor=False, **params) + url, namespace = push_images(base_path, config.image_namespace, engine_obj, config, + save_conductor=False, **params) params['url'] = url params['namespace'] = namespace @@ -202,14 +200,14 @@ def hostcmd_run(base_path, project_name, engine_name, var_file=None, cache=True, **kwargs): assert_initialized(base_path) logger.debug('Got extra args to `run` command', arguments=kwargs) - config = get_config(base_path, var_file=var_file, engine_name=engine_name) + config = get_config(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) if not kwargs['production']: config.set_env('dev') logger.debug('hostcmd_run configuration', config=config.__dict__) engine_obj = load_engine(['RUN'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) remove_existing_container(engine_obj, 'conductor', remove_volumes=True) @@ -235,10 +233,10 @@ def hostcmd_destroy(base_path, project_name, engine_name, var_file=None, cache=T **kwargs): assert_initialized(base_path) logger.debug('Got extra args to `destroy` command', arguments=kwargs) - config = get_config(base_path, var_file=var_file, engine_name=engine_name) + config = get_config(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) engine_obj = load_engine(['RUN'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) remove_existing_container(engine_obj, 'conductor', remove_volumes=True) @@ -261,9 +259,9 @@ def hostcmd_destroy(base_path, project_name, engine_name, var_file=None, cache=T @host_only def hostcmd_stop(base_path, project_name, engine_name, force=False, services=[], **kwargs): - config = get_config(base_path, engine_name=engine_name) + config = get_config(base_path, engine_name=engine_name, project_name=project_name) engine_obj = load_engine(['RUN'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) params = { @@ -285,9 +283,9 @@ def hostcmd_stop(base_path, project_name, engine_name, force=False, services=[], @host_only def hostcmd_restart(base_path, project_name, engine_name, force=False, services=[], **kwargs): - config = get_config(base_path, engine_name=engine_name) + config = get_config(base_path, engine_name=engine_name, project_name=project_name) engine_obj = load_engine(['RUN'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) params = { 'deployment_output_path': config.deployment_path, @@ -313,12 +311,14 @@ def hostcmd_push(base_path, project_name, engine_name, var_file=None, **kwargs): registry, pass username and/or password. If you exclude password, you will be prompted. """ assert_initialized(base_path) - config = get_config(base_path, var_file=var_file, engine_name=engine_name) + config = get_config(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) engine_obj = load_engine(['LOGIN', 'PUSH'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) + logger.debug('PROJECT NAME', project_name=project_name) push_images(base_path, + config.image_namespace, engine_obj, config, save_conductor=config.get('settings', {}).get('save_conductor_container', False), @@ -326,7 +326,7 @@ def hostcmd_push(base_path, project_name, engine_name, var_file=None, **kwargs): @host_only -def push_images(base_path, engine_obj, config, **kwargs): +def push_images(base_path, image_namespace, engine_obj, config, **kwargs): """ Pushes images to a Docker registry. Returns (url, namespace) used to push images. """ config_path = kwargs.get('config_path', engine_obj.auth_config_path) username = kwargs.get('username') @@ -334,19 +334,19 @@ def push_images(base_path, engine_obj, config, **kwargs): push_to = kwargs.get('push_to') url = engine_obj.default_registry_url registry_name = engine_obj.default_registry_name - namespace = None + namespace = image_namespace save_conductor = config.get('settings', {}).get('save_conductor_container', False) if push_to: if config.get('registries', dict()).get(push_to): url = config['registries'][push_to].get('url') - namespace = config['registries'][push_to].get('namespace') + namespace = config['registries'][push_to].get('namespace', namespace) if not url: raise AnsibleContainerRegistryAttributeException( u"Registry {} missing required attribute 'url'".format(push_to) ) else: - url, namespace = resolve_push_to(push_to, engine_obj.default_registry_url) + url, namespace = resolve_push_to(push_to, engine_obj.default_registry_url, namespace) if username and not password: # If a username was supplied without a password, prompt for it @@ -360,16 +360,17 @@ def push_images(base_path, engine_obj, config, **kwargs): config_path = os.path.normpath(os.path.expanduser(config_path)) if os.path.exists(config_path) and os.path.isdir(config_path): raise AnsibleContainerException( - u"Expecting --config-path to be a path to a file and not a directory" + u"Expecting --config-path to be a path to a file, not a directory" ) elif not os.path.exists(config_path): # Make sure the directory path exists - try: - os.makedirs(os.path.dirname(config_path), 0o750) - except OSError: - raise AnsibleContainerException( - u"Failed to create the requested the path {}".format(os.path.dirname(config_path)) - ) + if not os.path.exists(os.path.dirname(config_path)): + try: + os.makedirs(os.path.dirname(config_path), 0o750) + except OSError: + raise AnsibleContainerException( + u"Failed to create the requested the path {}".format(os.path.dirname(config_path)) + ) # Touch the file open(config_path, 'w').close() @@ -390,10 +391,10 @@ def push_images(base_path, engine_obj, config, **kwargs): @host_only def hostcmd_install(base_path, project_name, engine_name, **kwargs): assert_initialized(base_path) - config = get_config(base_path, engine_name=engine_name) + config = get_config(base_path, engine_name=engine_name, project_name=project_name) save_conductor = config.get('settings', {}).get('save_conductor_container', False) engine_obj = load_engine(['INSTALL'], - engine_name, project_name or os.path.basename(base_path), + engine_name, project_name, config['services'], **kwargs) engine_obj.await_conductor_command('install', dict(config), @@ -410,7 +411,7 @@ def hostcmd_version(base_path, project_name, engine_name, **kwargs): assert_initialized(base_path) engine_obj = load_engine(['VERSION'], engine_name, - project_name or os.path.basename(base_path), + project_name, {}, **kwargs) engine_obj.print_version_info() @@ -419,7 +420,7 @@ def hostcmd_version(base_path, project_name, engine_name, **kwargs): def hostcmd_import(base_path, project_name, engine_name, **kwargs): engine_obj = load_engine(['IMPORT'], engine_name, - project_name or os.path.basename(base_path), + project_name, {}, **kwargs) engine_obj.import_project(base_path, **kwargs) @@ -439,7 +440,7 @@ def remove_existing_container(engine_obj, service_name, remove_volumes=False): @host_only -def resolve_push_to(push_to, default_url): +def resolve_push_to(push_to, default_url, default_namespace): ''' Given a push-to value, return the registry and namespace. @@ -449,7 +450,7 @@ def resolve_push_to(push_to, default_url): ''' protocol = 'http://' if push_to.startswith('http://') else 'https://' url = push_to = REMOVE_HTTP.sub('', push_to) - namespace = None + namespace = default_namespace parts = url.split('/', 1) special_set = {'.', ':'} char_set = set([c for c in parts[0]]) @@ -463,7 +464,6 @@ def resolve_push_to(push_to, default_url): else: registry_url = protocol + parts[0] namespace = parts[1] - return registry_url, namespace @@ -802,17 +802,14 @@ def conductorcmd_deploy(engine_name, project_name, services, **kwargs): engine=engine.display_name) # Verify all images are built - for service_name in services: - if not services[service_name].get('roles'): - continue - logger.info(u'Verifying image for %s', service_name) - image_id = engine.get_latest_image_id_for_service(service_name) - if not image_id: - logger.error(u'Missing image. Run "ansible-container build" ' - u'to (re)create it.', service=service_name) - raise RuntimeError(u'Run failed.') - - logger.debug("conductorcmd_deploy", kwargs=kwargs) + for service_name, service_defn in services.items(): + if service_defn.get('roles'): + logger.info(u'Verifying image for %s', service_name) + image_id = engine.get_latest_image_id_for_service(service_name) + if not image_id: + msg = u'Missing image for {}. Run "ansible-container build" to (re)create it.'.format(service_name) + logger.error(msg, service=service_name) + raise RuntimeError(msg) deployment_output_path = kwargs.get('deployment_output_path') playbook = engine.generate_orchestration_playbook(**kwargs) @@ -854,7 +851,7 @@ def conductorcmd_push(engine_name, project_name, services, **kwargs): repo_data = { 'url': url, - 'namespace': namespace or username, + 'namespace': namespace, 'tag': tag, 'username': username, 'password': password diff --git a/container/docker/engine.py b/container/docker/engine.py index 5ad70f66..809c12db 100644 --- a/container/docker/engine.py +++ b/container/docker/engine.py @@ -15,6 +15,7 @@ import json import os import re +import shutil import six import sys import tarfile @@ -326,10 +327,20 @@ def await_conductor_command(self, command, config, base_path, params, save_conta conductor_id=conductor_id, command_rc=exit_code) if not save_container: self.delete_container(conductor_id, remove_volumes=True) + if exit_code: raise exceptions.AnsibleContainerConductorException( u'Conductor exited with status %s' % exit_code ) + elif command in ('run', 'destroy', 'stop', 'restart') and params.get('deployment_output_path'): + # Remove any ansible-playbook residue + output_path = params['deployment_output_path'] + for path in ('files', 'templates'): + shutil.rmtree(os.path.join(output_path, path), ignore_errors=True) + if not self.debug: + for filename in ('playbook.retry', 'playbook.yml', 'hosts'): + if os.path.exists(os.path.join(output_path, filename)): + os.remove(os.path.join(output_path, filename)) def service_is_running(self, service): try: @@ -514,38 +525,43 @@ def tag_image_as_latest(self, service_name, image_id): image_obj.tag(self.image_name_for_service(service_name), 'latest') @conductor_only - def generate_orchestration_playbook(self, url=None, namespace=None, local_images=True, **kwargs): + def generate_orchestration_playbook(self, url=None, namespace=None, **kwargs): """ Generate an Ansible playbook to orchestrate services. :param url: registry URL where images will be pulled from :param namespace: registry namespace - :param local_images: bypass pulling images, and use local copies :return: playbook dict """ states = ['start', 'restart', 'stop', 'destroy'] - + logger.debug("GENERATE DEPLOYMENT", url=url, namespace=namespace) service_def = {} for service_name, service in self.services.items(): service_definition = {} if service.get('roles'): - image = self.get_latest_image_for_service(service_name) - if image is None: - raise exceptions.AnsibleContainerConductorException( - u"No image found for service {}, make sure you've run `ansible-container " - u"build`".format(service_name) - ) - service_definition[u'image'] = image.tags[0] + if url and namespace: + # Reference previously pushed image + service_definition[u'image'] = '{}/{}/{}'.format(re.sub(r'/$', '', url), namespace, + self.image_name_for_service(service_name)) + else: + # Check that the image was built + image = self.get_latest_image_for_service(service_name) + if image is None: + raise exceptions.AnsibleContainerConductorException( + u"No image found for service {}, make sure you've run `ansible-container " + u"build`".format(service_name) + ) + service_definition[u'image'] = image.tags[0] else: try: + # Check if the image is already local image = self.client.images.get(service['from']) + image_from = image.tags[0] except docker.errors.ImageNotFound: - image = None + image_from = service['from'] logger.warning(u"Image {} for service {} not found. " u"An attempt will be made to pull it.".format(service['from'], service_name)) - if image: - service_definition[u'image'] = image.tags[0] - else: - service_definition[u'image'] = service['from'] + service_definition[u'image'] = image_from + for extra in self.COMPOSE_WHITELIST: if extra in service: service_definition[extra] = service[extra] diff --git a/container/k8s/base_deploy.py b/container/k8s/base_deploy.py index 8a174e51..11467aa3 100644 --- a/container/k8s/base_deploy.py +++ b/container/k8s/base_deploy.py @@ -209,6 +209,7 @@ def get_service_tasks(self, tags=[]): 'options', 'volume_driver', 'volumes_from', # TODO: figure out how to map? + 'from', 'roles', 'k8s', 'openshift', @@ -286,7 +287,7 @@ def _service_to_container(name, service): if isinstance(value, string_types): container['args'] = shlex.split(value) else: - container['args'] = value + container['args'] = copy.copy(value) elif key == 'container_name': container['name'] = value elif key == 'entrypoint': @@ -521,7 +522,7 @@ def _append_port(host, container, protocol): protocol = 'TCP' if isinstance(port, string_types) and '/' in port: port, protocol = port.split('/') - _append_port(port, protocol) + _append_port(port, port, protocol) return ports diff --git a/container/k8s/base_engine.py b/container/k8s/base_engine.py index 14d50624..09a2f606 100644 --- a/container/k8s/base_engine.py +++ b/container/k8s/base_engine.py @@ -2,15 +2,15 @@ from __future__ import absolute_import import os - +import re from ruamel.yaml.comments import CommentedMap, CommentedSeq from six import add_metaclass -from six.moves.urllib.parse import urljoin from abc import ABCMeta, abstractproperty, abstractmethod from container import conductor_only, host_only +from container import exceptions from container.docker.engine import Engine as DockerEngine, log_runs from container.utils.visibility import getLogger @@ -37,10 +37,13 @@ class K8sBaseEngine(DockerEngine): _deploy = None def __init__(self, project_name, services, debug=False, selinux=True, **kwargs): - self.namespace_name = kwargs.pop('namespace_name', project_name) - self.namespace_display_name = kwargs.pop('namespace_display_name', None) - self.namespace_description = kwargs.pop('namespace_description', None) + k8s_namespace = kwargs.pop('k8s_namespace', {}) + self.namespace_name = k8s_namespace.get('name', None) or project_name + self.namespace_display_name = k8s_namespace.get('display_name') + self.namespace_description = k8s_namespace.get('description') super(K8sBaseEngine, self).__init__(project_name, services, debug, selinux=selinux, **kwargs) + logger.debug("k8s namespace", namspace=self.namespace_name, display_name=self.namespace_display_name, + description=self.namespace_description) logger.debug("Volume for k8s", volumes=self.volumes) @property @@ -78,24 +81,31 @@ def run_conductor(self, command, config, base_path, params, engine_name=None, vo volumes=volumes) @conductor_only - def generate_orchestration_playbook(self, url=None, namespace=None, local_images=True, **kwargs): + def generate_orchestration_playbook(self, url=None, namespace=None, **kwargs): """ Generate an Ansible playbook to orchestrate services. :param url: registry URL where images will be pulled from :param namespace: registry namespace - :param local_images: bypass pulling images, and use local copies :return: playbook dict """ - for service_name in self.services: - image = self.get_latest_image_for_service(service_name) - if local_images: - self.services[service_name]['image'] = image.tags[0] - else: - if namespace is not None: - image_url = urljoin('{}/'.format(urljoin(url, namespace)), image.tags[0]) + for service_name, service_config in self.services.iteritems(): + if service_config.get('roles'): + if url and namespace: + # Reference previously pushed image + self.services[service_name][u'image'] = '{}/{}/{}'.format(re.sub(r'/$', '', url), namespace, + self.image_name_for_service(service_name)) else: - image_url = urljoin(url, image.tags[0]) - self.services[service_name]['image'] = image_url + # We're using a local image, so check that the image was built + image = self.get_latest_image_for_service(service_name) + if image is None: + raise exceptions.AnsibleContainerConductorException( + u"No image found for service {}, make sure you've run `ansible-container " + u"build`".format(service_name) + ) + self.services[service_name][u'image'] = image.tags[0] + else: + # Not a built image + self.services[service_name][u'image'] = service_config['from'] if kwargs.get('k8s_auth'): self.k8s_client.set_authorization(kwargs['auth']) @@ -127,4 +137,3 @@ def generate_orchestration_playbook(self, url=None, namespace=None, local_images logger.debug(u'Created playbook to run project', playbook=playbook) return playbook - diff --git a/container/k8s/engine.py b/container/k8s/engine.py index 7ab6b4b6..4fb7b050 100644 --- a/container/k8s/engine.py +++ b/container/k8s/engine.py @@ -38,26 +38,3 @@ def run_conductor(self, command, config, base_path, params, engine_name=None, vo engine_name = __name__.rsplit('.', 2)[-2] return super(Engine, self).run_conductor(command, config, base_path, params, engine_name=engine_name) - # def _create_ks8_object(self, api_version, kind, request): - # name = request.get('metadata', {}).get('name') - # namespace = request.get('metadata', {}).get('namespace') - # - # self.client.set_model(api_version, kind) - # k8s_obj = self.client.get_object(name, namespace) - # if not k8s_obj: - # # create when it doesn't exist - # try: - # k8s_obj = self.client.create_object(namespace, body=request) - # except KubernetesException as exc: - # raise exceptions.AnsibleContainerConductorException( - # u"Error creating {0}: {1}".format(request['kind'], exc) - # ) - # else: - # # otherwise, replace it. not idempotent, but good enough for now - # try: - # k8s_obj = self.client.replace_object(name, namespace, body=request) - # except KubernetesException as exc: - # raise exceptions.AnsibleContainerConductorException( - # u"Error creating {0}: {1}".format(request['kind'], exc) - # ) - # return k8s_obj diff --git a/container/utils/__init__.py b/container/utils/__init__.py index b3e7a397..45565089 100644 --- a/container/utils/__init__.py +++ b/container/utils/__init__.py @@ -40,11 +40,11 @@ make_temp_dir = MakeTempDir -def get_config(base_path, var_file=None, engine_name=None): +def get_config(base_path, var_file=None, engine_name=None, project_name=None): # To avoid circular import from ..config import AnsibleContainerConfig - return AnsibleContainerConfig(base_path, var_file=var_file, engine_name=engine_name) + return AnsibleContainerConfig(base_path, var_file=var_file, engine_name=engine_name, project_name=project_name) def assert_initialized(base_path): @@ -197,7 +197,12 @@ def create_role_from_templates(role_name=None, role_path=None, @container.conductor_only def resolve_role_to_path(role_name): - loader, variable_manager = DataLoader(), VariableManager() + loader = DataLoader() + try: + variable_manager = VariableManager(loader=loader) + except TypeError: + # If Ansible prior to ansible/ansible@8f97aef1a365 + variable_manager = VariableManager() role_obj = RoleInclude.load(data=role_name, play=None, variable_manager=variable_manager, loader=loader) @@ -274,9 +279,9 @@ def ordereddict_to_list(config): # If configuration top-level key is an orderedict, convert to list of tuples, providing a # means to preserve key order. Call prior to encoding a config dict. result = {} - for key, value in config.iteritems(): + for key, value in iteritems(config): if isinstance(value, yaml.compat.ordereddict): - result[key] = value.items() + result[key] = list(value.items()) else: result[key] = value return result @@ -286,7 +291,7 @@ def list_to_ordereddict(config): # If configuration top-level key is a list, convert it to an ordereddict. # Call post decoding of a config dict. result = {} - for key, value in config.iteritems(): + for key, value in iteritems(config): if isinstance(value, list): result[key] = yaml.compat.ordereddict(value) else: diff --git a/test/roles/validate-build-and-import/tasks/main.yml b/test/roles/validate-build-and-import/tasks/main.yml index f33583f5..cd8a15f1 100644 --- a/test/roles/validate-build-and-import/tasks/main.yml +++ b/test/roles/validate-build-and-import/tasks/main.yml @@ -95,6 +95,13 @@ - include: includes/show-output.yml output_file=./task.output registered_output="{{ output }}" +- name: Get docker images + command: docker images + register: docker_images + +- name: Show docker images + debug: var=docker_images.stdout_lines + - name: Validate build shell: pytest tests/validate_build.py >>task.output 2>&1 environment: diff --git a/test/tests/validate_build.py b/test/tests/validate_build.py index 60eeb72d..7c8f7747 100644 --- a/test/tests/validate_build.py +++ b/test/tests/validate_build.py @@ -8,7 +8,7 @@ import os import json -from ansible.vars import Templar +from ansible.template import Templar from ruamel import yaml import docker diff --git a/test/tests/validate_config.py b/test/tests/validate_config.py index 1282e096..5c6311af 100644 --- a/test/tests/validate_config.py +++ b/test/tests/validate_config.py @@ -9,7 +9,7 @@ import container from container.config import AnsibleContainerConfig, AnsibleContainerConductorConfig from container.exceptions import AnsibleContainerConfigException -from ansible.vars import Templar +from ansible.template import Templar from ansible.playbook.role.include import RoleInclude try: from ansible.vars import VariableManager