changed logging to local for ansible configurator

bibigrid/core/actions/create.py

@@ -179,8 +179,10 @@ def start_vpn_or_master_instance(self, configuration, provider):
         network = configuration["network"]
 
         # create a server and block until it is up and running
+        print("Is it here")
         server = provider.create_server(name=name, flavor=flavor, key_name=self.key_name, image=image, network=network,
                                         volumes=volumes, security_groups=configuration["security_groups"], wait=True)
+        print("Or not")
         configuration["private_v4"] = server["private_v4"]
 
         # get mac address for given private address
@@ -302,7 +304,7 @@ def upload_data(self):
                 hosts_file.write("# placeholder file for worker DNS entries (see 003-dns)")
 
         ansible_configurator.configure_ansible_yaml(providers=self.providers, configurations=self.configurations,
-                                                    cluster_id=self.cluster_id)
+                                                    cluster_id=self.cluster_id, log=self.log)
         commands = [ssh_handler.get_ac_command(self.providers,
                                                AC_NAME.format(cluster_id=self.cluster_id))] + ssh_handler.ANSIBLE_START
         ssh_handler.execute_ssh(floating_ip=self.master_ip, private_key=KEY_FOLDER + self.key_name,

bibigrid/core/utility/ansible_configurator.py

@@ -2,7 +2,6 @@
 Prepares ansible files (vars, common_configuration, ...)
 """
 
-import logging
 import os
 
 import mergedeep
@@ -30,20 +29,20 @@
 SLURM_CONF = {"db": "slurm", "db_user": "slurm", "db_password": "changeme",
               "munge_key": id_generation.generate_munge_key(),
               "elastic_scheduling": {"SuspendTime": 3600, "ResumeTimeout": 900, "TreeWidth": 128}}
-LOG = logging.getLogger("bibigrid")
 
 
-def delete_old_vars():
+def delete_old_vars(log):
     """
     Deletes host_vars and group_vars
+    @param log:
     @return:
     """
     for folder in [aRP.GROUP_VARS_FOLDER, aRP.HOST_VARS_FOLDER]:
         for file_name in os.listdir(folder):
             # construct full file path
             file = os.path.join(folder, file_name)
             if os.path.isfile(file):
-                logging.debug('Deleting file: %s', file)
+                log.debug('Deleting file: %s', file)
                 os.remove(file)
 
 
@@ -67,7 +66,7 @@ def generate_site_file_yaml(custom_roles):
     return site_yaml
 
 
-def write_host_and_group_vars(configurations, providers, cluster_id):  # pylint: disable=too-many-locals
+def write_host_and_group_vars(configurations, providers, cluster_id, log):  # pylint: disable=too-many-locals
     """
     ToDo filter what information really is necessary. Determined by further development
     Filters unnecessary information
@@ -76,7 +75,7 @@ def write_host_and_group_vars(configurations, providers, cluster_id):  # pylint:
     :param cluster_id: To get proper naming
     :return: filtered information (dict)
     """
-    LOG.info("Generating instances file...")
+    log.info("Generating instances file...")
     flavor_keys = ["name", "ram", "vcpus", "disk", "ephemeral"]
     worker_count = 0
     vpn_count = 0
@@ -100,10 +99,10 @@ def write_host_and_group_vars(configurations, providers, cluster_id):  # pylint:
             worker_features = worker.get("features", [])
             if isinstance(worker_features, str):
                 worker_features = [worker_features]
-            features = set(configuration_features+worker_features)
+            features = set(configuration_features + worker_features)
             if features:
                 worker_dict["features"] = features
-            write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, group_name), worker_dict)
+            write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, group_name), worker_dict, log)
         vpngtw = configuration.get("vpnInstance")
         if vpngtw:
             name = create.VPN_WORKER_IDENTIFIER(cluster_id=cluster_id, additional=f"{vpn_count}")
@@ -113,32 +112,25 @@ def write_host_and_group_vars(configurations, providers, cluster_id):  # pylint:
             flavor_dict = {key: flavor[key] for key in flavor_keys}
             regexp = create.WORKER_IDENTIFIER(cluster_id=cluster_id, additional=r"\d+")
             vpngtw_dict = {"name": name, "regexp": regexp, "image": vpngtw["image"],
-                           "network": configuration["network"],
-                           "network_cidr": configuration["subnet_cidrs"],
-                           "floating_ip": configuration["floating_ip"],
-                           "private_v4": configuration["private_v4"],
-                           "flavor": flavor_dict,
-                           "wireguard_ip": wireguard_ip,
-                           "cloud_identifier": configuration[
-                               "cloud_identifier"]}
+                           "network": configuration["network"], "network_cidr": configuration["subnet_cidrs"],
+                           "floating_ip": configuration["floating_ip"], "private_v4": configuration["private_v4"],
+                           "flavor": flavor_dict, "wireguard_ip": wireguard_ip,
+                           "cloud_identifier": configuration["cloud_identifier"]}
             if configuration.get("wireguard_peer"):
-                vpngtw_dict["wireguard"] = {"ip": wireguard_ip,
-                                            "peer": configuration.get(
-                                                "wireguard_peer")}
-            write_yaml(os.path.join(aRP.HOST_VARS_FOLDER, name), vpngtw_dict)
+                vpngtw_dict["wireguard"] = {"ip": wireguard_ip, "peer": configuration.get("wireguard_peer")}
+            write_yaml(os.path.join(aRP.HOST_VARS_FOLDER, name), vpngtw_dict, log)
         else:
             master = configuration["masterInstance"]
             name = create.MASTER_IDENTIFIER(cluster_id=cluster_id)
             flavor = provider.get_flavor(master["type"])
             flavor_dict = {key: flavor[key] for key in flavor_keys}
             master_dict = {"name": name, "image": master["image"], "network": configuration["network"],
-                           "network_cidr": configuration["subnet_cidrs"],
-                           "floating_ip": configuration["floating_ip"], "flavor": flavor_dict,
-                           "private_v4": configuration["private_v4"],
+                           "network_cidr": configuration["subnet_cidrs"], "floating_ip": configuration["floating_ip"],
+                           "flavor": flavor_dict, "private_v4": configuration["private_v4"],
                            "cloud_identifier": configuration["cloud_identifier"]}
             if configuration.get("wireguard_peer"):
                 master_dict["wireguard"] = {"ip": "10.0.0.1", "peer": configuration.get("wireguard_peer")}
-            write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, "master.yml"), master_dict)
+            write_yaml(os.path.join(aRP.GROUP_VARS_FOLDER, "master.yml"), master_dict, log)
 
 
 def pass_through(dict_from, dict_to, key_from, key_to=None):
@@ -156,18 +148,19 @@ def pass_through(dict_from, dict_to, key_from, key_to=None):
         dict_to[key_to] = dict_from[key_from]
 
 
-def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_user, default_user):
+def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_user, default_user, log):
     """
     Generates common_configuration yaml (dict)
     :param cidrs: str subnet cidrs (provider generated)
     :param configurations: master configuration (first in file)
     :param cluster_id: id of cluster
     :param ssh_user: user for ssh connections
     :param default_user: Given default user
+    :param log:
     :return: common_configuration_yaml (dict)
     """
     master_configuration = configurations[0]
-    LOG.info("Generating common configuration file...")
+    log.info("Generating common configuration file...")
     # print(configuration.get("slurmConf", {}))
     common_configuration_yaml = {"cluster_id": cluster_id, "cluster_cidrs": cidrs, "default_user": default_user,
                                  "local_fs": master_configuration.get("localFS", False),
@@ -209,15 +202,16 @@ def generate_common_configuration_yaml(cidrs, configurations, cluster_id, ssh_us
     return common_configuration_yaml
 
 
-def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id):
+def generate_ansible_hosts_yaml(ssh_user, configurations, cluster_id, log):  # pylint: disable-msg=too-many-locals
     """
     Generates ansible_hosts_yaml (inventory file).
     :param ssh_user: str global SSH-username
     :param configurations: dict
     :param cluster_id: id of cluster
+    :param log:
     :return: ansible_hosts yaml (dict)
     """
-    LOG.info("Generating ansible hosts file...")
+    log.info("Generating ansible hosts file...")
     ansible_hosts_yaml = {"vpn": {"hosts": {},
                                   "children": {"master": {"hosts": {"localhost": to_instance_host_dict(ssh_user)}},
                                                "vpngtw": {"hosts": {}}}}, "workers": {"hosts": {}, "children": {}}}
@@ -274,7 +268,7 @@ def get_cidrs(configurations):
     return all_cidrs
 
 
-def get_ansible_roles(ansible_roles):
+def get_ansible_roles(ansible_roles, log):
     """
     Checks if ansible_roles have all necessary values and returns True if so.
     :param ansible_roles: ansible_roles from master configuration (first configuration)
@@ -289,14 +283,15 @@ def get_ansible_roles(ansible_roles):
                     ansible_role_dict[key] = ansible_role[key]
             ansible_roles_yaml.append(ansible_role_dict)
         else:
-            LOG.warning("Ansible role %s had neither galaxy,git nor url. Not added.", ansible_role)
+            log.warning("Ansible role %s had neither galaxy,git nor url. Not added.", ansible_role)
     return ansible_roles_yaml
 
 
-def get_ansible_galaxy_roles(ansible_galaxy_roles):
+def get_ansible_galaxy_roles(ansible_galaxy_roles, log):
     """
     Checks if ansible_galaxy_role have all necessary values and adds it to the return list if so.
     :param ansible_galaxy_roles:
+    :param log:
     :return: list of valid ansible_galaxy_roles
     """
     ansible_galaxy_roles_yaml = []
@@ -308,17 +303,18 @@ def get_ansible_galaxy_roles(ansible_galaxy_roles):
                     ansible_galaxy_role_dict[key] = ansible_galaxy_role[key]
             ansible_galaxy_roles_yaml.append(ansible_galaxy_role_dict)
         else:
-            LOG.warning("Galaxy role %s had neither galaxy,git nor url. Not added.", ansible_galaxy_role)
+            log.warning("Galaxy role %s had neither galaxy,git nor url. Not added.", ansible_galaxy_role)
     return ansible_galaxy_roles_yaml
 
 
-def generate_worker_specification_file_yaml(configurations):
+def generate_worker_specification_file_yaml(configurations, log):
     """
     Generates worker_specification_file_yaml
     :param configurations: list of configurations (dict)
+    :param log:
     :return: worker_specification_yaml
     """
-    LOG.info("Generating worker specification file...")
+    log.info("Generating worker specification file...")
     worker_groups_list = configuration_handler.get_list_by_key(configurations, "workerInstances", False)
     # create.prepare_configuration guarantees that key is set
     network_list = configuration_handler.get_list_by_key(configurations, "network", False)
@@ -330,15 +326,16 @@ def generate_worker_specification_file_yaml(configurations):
     return worker_specification_yaml
 
 
-def write_yaml(path, generated_yaml, alias=False):
+def write_yaml(path, generated_yaml, log, alias=False):
     """
     Writes generated_yaml to file path with or without alias
     @param path:
     @param generated_yaml:
+    @param log:
     @param alias:
     @return:
     """
-    LOG.debug("Writing yaml %s", path)
+    log.debug("Writing yaml %s", path)
     with open(path, mode="w+", encoding="UTF-8") as file:
         if alias:
             yaml.safe_dump(data=generated_yaml, stream=file)
@@ -360,27 +357,32 @@ def add_wireguard_peers(configurations):
                                                "subnet": configuration["subnet_cidrs"]}
 
 
-def configure_ansible_yaml(providers, configurations, cluster_id):
+def configure_ansible_yaml(providers, configurations, cluster_id, log):
     """
     Generates and writes all ansible-configuration-yaml files.
     :param providers: list of providers
     :param configurations: list of configurations (dict)
     :param cluster_id: id of cluster to create
+    :param log:
     :return:
     """
-    delete_old_vars()
-    LOG.info("Writing ansible files...")
+    delete_old_vars(log)
+    log.info("Writing ansible files...")
     alias = configurations[0].get("aliasDumper", False)
-    ansible_roles = get_ansible_roles(configurations[0].get("ansibleRoles"))
+    ansible_roles = get_ansible_roles(configurations[0].get("ansibleRoles"), log)
     default_user = providers[0].cloud_specification["auth"].get("username", configurations[0].get("sshUser", "Ubuntu"))
     add_wireguard_peers(configurations)
     for path, generated_yaml in [
-        (aRP.WORKER_SPECIFICATION_FILE, generate_worker_specification_file_yaml(configurations)), (
+        (aRP.WORKER_SPECIFICATION_FILE, generate_worker_specification_file_yaml(configurations, log)), (
                 aRP.COMMONS_CONFIG_FILE,
                 generate_common_configuration_yaml(cidrs=get_cidrs(configurations), configurations=configurations,
                                                    cluster_id=cluster_id, ssh_user=configurations[0]["sshUser"],
-                                                   default_user=default_user)),
-        (aRP.HOSTS_CONFIG_FILE, generate_ansible_hosts_yaml(configurations[0]["sshUser"], configurations, cluster_id)),
+                                                   default_user=default_user, log=log)), (aRP.HOSTS_CONFIG_FILE,
+                                                                                          generate_ansible_hosts_yaml(
+                                                                                              configurations[0][
+                                                                                                  "sshUser"],
+                                                                                              configurations,
+                                                                                              cluster_id, log)),
         (aRP.SITE_CONFIG_FILE, generate_site_file_yaml(ansible_roles))]:
-        write_yaml(path, generated_yaml, alias)
-    write_host_and_group_vars(configurations, providers, cluster_id)  # writing included in method
+        write_yaml(path, generated_yaml, log, alias)
+    write_host_and_group_vars(configurations, providers, cluster_id, log)  # writing included in method