Skip to content

Commit

Permalink
Reverted logging to global logging because using redirect might be mo…
Browse files Browse the repository at this point in the history 
…re feasible
  • Loading branch information
@XaverStiensmeier
XaverStiensmeier committed Aug 9, 2023
1 parent bb9713c commit 60c7ca1
Showing 1 changed file with 47 additions and 47 deletions.
94 changes: 47 additions & 47 deletions bibigrid/core/actions/create.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
PREFIX = "bibigrid"
SEPARATOR = "-"
PREFIX_WITH_SEP = PREFIX + SEPARATOR
LOG = logging.getLogger("bibigrid")


def get_identifier(identifier, cluster_id, worker_group="", additional=""):
Expand Down Expand Up @@ -62,7 +63,7 @@ class Create: # pylint: disable=too-many-instance-attributes,too-many-arguments
The class Create holds necessary methods to execute the Create-Action
"""

def __init__(self, providers, configurations, config_path, debug=False, cluster_id=None, log_name="bibigrid"):
def __init__(self, providers, configurations, config_path, debug=False, cluster_id=None):
"""
Additionally sets (unique) cluster_id, public_key_commands (to copy public keys to master) and key_name.
Call create() to actually start server.
Expand All @@ -81,8 +82,7 @@ def __init__(self, providers, configurations, config_path, debug=False, cluster_
configurations[0].get("sshPublicKeyFiles"))
self.config_path = config_path
self.master_ip = None
self.log = logging.getLogger(log_name)
self.log.debug("Cluster-ID: %s", self.cluster_id)
LOG.debug("Cluster-ID: %s", self.cluster_id)
self.name = AC_NAME.format(cluster_id=self.cluster_id)
self.key_name = KEY_NAME.format(cluster_id=self.cluster_id)
self.default_security_group_name = DEFAULT_SECURITY_GROUP_NAME.format(cluster_id=self.cluster_id)
Expand All @@ -92,7 +92,7 @@ def __init__(self, providers, configurations, config_path, debug=False, cluster_
self.vpn_counter = 0
self.thread_lock = threading.Lock()
self.use_master_with_public_ip = configurations[0].get("useMasterWithPublicIp", True)
self.log.debug("Keyname: %s", self.key_name)
LOG.debug("Keyname: %s", self.key_name)

def generate_keypair(self):
"""
Expand All @@ -105,11 +105,11 @@ def generate_keypair(self):
"""
# create KEY_FOLDER if it doesn't exist
if not os.path.isdir(KEY_FOLDER):
self.log.info("%s not found. Creating folder.", KEY_FOLDER)
LOG.info("%s not found. Creating folder.", KEY_FOLDER)
os.mkdir(KEY_FOLDER)
# generate keyfile
res = subprocess.check_output(f'ssh-keygen -t ecdsa -f {KEY_FOLDER}{self.key_name} -P ""', shell=True).decode()
self.log.debug(res)
LOG.debug(res)
# read private keyfile
with open(f"{os.path.join(KEY_FOLDER, self.key_name)}.pub", mode="r", encoding="UTF-8") as key_file:
public_key = key_file.read()
Expand Down Expand Up @@ -170,7 +170,7 @@ def start_vpn_or_master_instance(self, configuration, provider):
name = identifier(cluster_id=self.cluster_id, # pylint: disable=redundant-keyword-arg
additional=self.vpn_counter) # pylint: disable=redundant-keyword-arg
self.vpn_counter += 1
self.log.info(f"Starting instance/server {name} on {provider.cloud_specification['identifier']}")
LOG.info(f"Starting instance/server {name} on {provider.cloud_specification['identifier']}")
flavor = instance_type["type"]
image = instance_type["image"]
network = configuration["network"]
Expand Down Expand Up @@ -214,7 +214,7 @@ def prepare_vpn_or_master_args(self, configuration, provider):
identifier = VPN_WORKER_IDENTIFIER
volumes = [] # only master has volumes
else:
self.log.warning("Configuration %s has no vpngtw or master and is therefore unreachable.", configuration)
LOG.warning("Configuration %s has no vpngtw or master and is therefore unreachable.", configuration)
raise KeyError
return identifier, instance_type, volumes

Expand All @@ -240,23 +240,23 @@ def prepare_volumes(self, provider, mounts):
:param mounts: volumes or snapshots
:return: list of pre-existing and newly created volumes
"""
self.log.info("Preparing volumes")
LOG.info("Preparing volumes")
volumes = []
for mount in mounts:
volume_id = provider.get_volume_by_id_or_name(mount)["id"]
if volume_id:
volumes.append(volume_id)
else:
self.log.debug("Volume %s does not exist. Checking for snapshot.", mount)
LOG.debug("Volume %s does not exist. Checking for snapshot.", mount)
volume_id = provider.create_volume_from_snapshot(mount)
if volume_id:
volumes.append(volume_id)
else:
self.log.warning("Mount %s is neither a snapshot nor a volume.", mount)
LOG.warning("Mount %s is neither a snapshot nor a volume.", mount)
ret_volumes = set(volumes)
if len(ret_volumes) < len(volumes):
self.log.warning("Identical mounts found in masterMounts list. "
"Trying to set() to save the run. Check configurations!")
LOG.warning("Identical mounts found in masterMounts list. "
"Trying to set() to save the run. Check configurations!")
return ret_volumes

def prepare_configurations(self):
Expand All @@ -270,13 +270,13 @@ def prepare_configurations(self):
if not configuration.get("network"):
configuration["network"] = provider.get_network_id_by_subnet(configuration["subnet"])
if not configuration["network"]:
self.log.warning("Unable to set network. "
f"Subnet doesn't exist in cloud {configuration['cloud_identifier']}")
LOG.warning("Unable to set network. "
f"Subnet doesn't exist in cloud {configuration['cloud_identifier']}")
raise ConfigurationException(f"Subnet doesn't exist in cloud {configuration['cloud_identifier']}")
elif not configuration.get("subnet"):
configuration["subnet"] = provider.get_subnet_ids_by_network(configuration["network"])
if not configuration["subnet"]:
self.log.warning("Unable to set subnet. Network doesn't exist.")
LOG.warning("Unable to set subnet. Network doesn't exist.")
raise ConfigurationException("Network doesn't exist.")
configuration["subnet_cidrs"] = provider.get_subnet_by_id_or_name(configuration["subnet"])["cidr"]
configuration["sshUser"] = self.ssh_user # is used in ansibleConfigurator
Expand All @@ -288,7 +288,7 @@ def upload_data(self):
"""
for folder in [aRP.VARS_FOLDER, aRP.GROUP_VARS_FOLDER, aRP.HOST_VARS_FOLDER]:
if not os.path.isdir(folder):
self.log.info("%s not found. Creating folder.", folder)
LOG.info("%s not found. Creating folder.", folder)
os.mkdir(folder)
if not os.path.isfile(aRP.HOSTS_FILE):
with open(aRP.HOSTS_FILE, 'a', encoding='utf-8') as hosts_file:
Expand All @@ -297,13 +297,14 @@ def upload_data(self):
ansible_configurator.configure_ansible_yaml(providers=self.providers, configurations=self.configurations,
cluster_id=self.cluster_id)
ssh_handler.execute_ssh(floating_ip=self.master_ip, private_key=KEY_FOLDER + self.key_name,
username=self.ssh_user, filepaths=[(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE),
(biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)],
username=self.ssh_user,
filepaths=[(aRP.PLAYBOOK_PATH, aRP.PLAYBOOK_PATH_REMOTE),
(biRP.BIN_PATH, biRP.BIN_PATH_REMOTE)],
commands=[
ssh_handler.get_ac_command(
self.providers,
AC_NAME.format(
cluster_id=self.cluster_id))] + ssh_handler.ANSIBLE_START)
ssh_handler.get_ac_command(
self.providers,
AC_NAME.format(
cluster_id=self.cluster_id))] + ssh_handler.ANSIBLE_START)

def start_start_instance_threads(self):
"""
Expand Down Expand Up @@ -334,9 +335,8 @@ def extended_network_configuration(self):
for configuration_b in self.configurations:
# ... and pick all other configuration
if configuration_a != configuration_b:
self.log.info(
f"{configuration_a['private_v4']} --> allowed_address_pair({configuration_a['mac_addr']},"
f"{configuration_b['subnet_cidrs']})")
LOG.info(f"{configuration_a['private_v4']} --> allowed_address_pair({configuration_a['mac_addr']},"
f"{configuration_b['subnet_cidrs']})")
# add provider_b network as allowed network
allowed_addresses.append(
{'ip_address': configuration_b["subnet_cidrs"], 'mac_address': configuration_a["mac_addr"]})
Expand Down Expand Up @@ -364,43 +364,43 @@ def create(self): # pylint: disable=too-many-branches,too-many-statements
self.upload_data()
self.print_cluster_start_info()
if self.debug:
self.log.info("DEBUG MODE: Entering termination...")
terminate.terminate(cluster_id=self.cluster_id, providers=self.providers, debug=self.debug)
LOG.info("DEBUG MODE: Entering termination...")
terminate.terminate(cluster_id=self.cluster_id, providers=self.providers,
debug=self.debug)
except exceptions.ConnectionException:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error("Connection couldn't be established. Check Provider connection.")
LOG.error(traceback.format_exc())
LOG.error("Connection couldn't be established. Check Provider connection.")
except paramiko.ssh_exception.NoValidConnectionsError:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error("SSH connection couldn't be established. Check keypair.")
LOG.error(traceback.format_exc())
LOG.error("SSH connection couldn't be established. Check keypair.")
except KeyError as exc:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(
f"Tried to access dictionary key {str(exc)}, but couldn't. Please check your configurations.")
LOG.error(traceback.format_exc())
LOG.error(f"Tried to access dictionary key {str(exc)}, but couldn't. Please check your configurations.")
except FileNotFoundError as exc:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(f"Tried to access resource files but couldn't. No such file or directory: {str(exc)}")
LOG.error(traceback.format_exc())
LOG.error(f"Tried to access resource files but couldn't. No such file or directory: {str(exc)}")
except TimeoutError as exc:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(f"Timeout while connecting to master. Maybe you are trying to create a master without "
f"public ip "
f"while not being in the same network: {str(exc)}")
LOG.error(traceback.format_exc())
LOG.error(f"Timeout while connecting to master. Maybe you are trying to create a master without "
f"public ip "
f"while not being in the same network: {str(exc)}")
except ExecutionException as exc:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(f"Execution of cmd on remote host fails: {str(exc)}")
LOG.error(traceback.format_exc())
LOG.error(f"Execution of cmd on remote host fails: {str(exc)}")
except ConfigurationException as exc:
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(f"Configuration invalid: {str(exc)}")
LOG.error(traceback.format_exc())
LOG.error(f"Configuration invalid: {str(exc)}")
except Exception as exc: # pylint: disable=broad-except
if self.debug:
self.log.error(traceback.format_exc())
self.log.error(f"Unexpected error: '{str(exc)}' ({type(exc)}) Contact a developer!)")
LOG.error(traceback.format_exc())
LOG.error(f"Unexpected error: '{str(exc)}' ({type(exc)}) Contact a developer!)")
else:
return 0 # will be called if no exception occurred
terminate.terminate(cluster_id=self.cluster_id, providers=self.providers, debug=self.debug)
Expand Down

0 comments on commit 60c7ca1

Please sign in to comment.