Skip to content

Commit

Permalink
changed global logging to local logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@XaverStiensmeier
XaverStiensmeier committed Aug 11, 2023
1 parent 62073f1 commit 5ec389f
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 29 deletions.
57 changes: 29 additions & 28 deletions bibigrid/core/utility/handler/configuration_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
This module contains methods to read the configuration and cloud specification.
"""

import logging
import os

import mergedeep
Expand All @@ -16,12 +15,11 @@
CLOUDS_PUBLIC_NAME_KEY = "profile"
CLOUD_CONFIGURATION_KEY = "cloud"

LOG = logging.getLogger("bibigrid")


def read_configuration(path="bibigrid.yml", configuration_list=True):
def read_configuration(log, path="bibigrid.yml", configuration_list=True):
"""
Reads yaml from file and returns configuration
:param log:
:param path: Path to yaml file
:param configuration_list: True if list is expected
:return: configurations (dict)
Expand All @@ -32,11 +30,11 @@ def read_configuration(path="bibigrid.yml", configuration_list=True):
try:
configuration = yaml.safe_load(stream)
except yaml.YAMLError as exc:
LOG.warning("Couldn't read configuration %s: %s", path, exc)
log.warning("Couldn't read configuration %s: %s", path, exc)
else:
LOG.warning("No such configuration file %s.", path)
log.warning("No such configuration file %s.", path)
if configuration_list and not isinstance(configuration, list):
LOG.warning("Configuration should be list. Attempting to rescue by assuming a single configuration.")
log.warning("Configuration should be list. Attempting to rescue by assuming a single configuration.")
return [configuration]
return configuration

Expand All @@ -57,51 +55,53 @@ def get_list_by_key(configurations, key, get_empty=True):
# for configuration in configurations]


def find_file_in_folders(file_name, folders):
def find_file_in_folders(file_name, folders, log):
"""
Searches all folders for a file with name file_name, loads (expects yaml) the first match and returns the dict
@param file_name: name of the file to look for
@param folders: folders to search for file named file_name
@param log:
@return: dict of match content or None if not found
"""
for folder_path in folders:
file_path = os.path.expanduser(os.path.join(folder_path, file_name))
if os.path.isfile(file_path):
LOG.debug("File %s found in folder %s.", file_name, folder_path)
return read_configuration(file_path, False)
LOG.debug("File %s not found in folder %s.", file_name, folder_path)
log.debug("File %s found in folder %s.", file_name, folder_path)
return read_configuration(log, file_path, False)
log.debug("File %s not found in folder %s.", file_name, folder_path)
return None


def get_clouds_files():
def get_clouds_files(log):
"""
Wrapper to call find_file_in_folders with the right arguments to find the clouds.yaml and clouds-public.yaml
@return: tuple of dicts containing the clouds.yaml and clouds-public.yaml data or None if not found.
"""
clouds_yaml = find_file_in_folders(CLOUDS_YAML, CLOUDS_YAML_PATHS)
clouds_public_yaml = find_file_in_folders(CLOUDS_PUBLIC_YAML, CLOUDS_YAML_PATHS)
clouds_yaml = find_file_in_folders(CLOUDS_YAML, CLOUDS_YAML_PATHS, log)
clouds_public_yaml = find_file_in_folders(CLOUDS_PUBLIC_YAML, CLOUDS_YAML_PATHS, log)
clouds = None
clouds_public = None
if clouds_yaml:
clouds = clouds_yaml.get(CLOUD_ROOT_KEY)
if not clouds:
LOG.warning("%s is not valid. Must contain key '%s:'", CLOUDS_YAML, CLOUD_ROOT_KEY)
log.warning("%s is not valid. Must contain key '%s:'", CLOUDS_YAML, CLOUD_ROOT_KEY)
else:
LOG.warning("No %s at %s! Please copy your %s to one of those listed folders. Aborting...", CLOUDS_YAML,
log.warning("No %s at %s! Please copy your %s to one of those listed folders. Aborting...", CLOUDS_YAML,
CLOUDS_YAML_PATHS, CLOUDS_YAML)
if clouds_public_yaml:
clouds_public = clouds_public_yaml.get(CLOUD_PUBLIC_ROOT_KEY)
if not clouds_public:
LOG.warning("%s is not valid. Must contain key '%s'", CLOUDS_PUBLIC_YAML, CLOUD_PUBLIC_ROOT_KEY)
log.warning("%s is not valid. Must contain key '%s'", CLOUDS_PUBLIC_YAML, CLOUD_PUBLIC_ROOT_KEY)
return clouds, clouds_public


def get_cloud_specification(cloud_name, clouds, clouds_public):
def get_cloud_specification(cloud_name, clouds, clouds_public, log):
"""
As in openstack cloud_public_specification will be overwritten by cloud_private_specification
:param cloud_name: name of the cloud to look for in clouds.yaml
:param clouds: dict containing the data loaded from clouds.yaml
:param clouds_public: dict containing the data loaded from clouds-public.yaml
:param log:
:return:
"""
cloud_full_specification = {}
Expand All @@ -110,42 +110,43 @@ def get_cloud_specification(cloud_name, clouds, clouds_public):
cloud_full_specification = cloud_private_specification
public_cloud_name = cloud_private_specification.get(CLOUDS_PUBLIC_NAME_KEY)
if public_cloud_name and clouds_public:
LOG.debug("Trying to find profile...")
log.debug("Trying to find profile...")
cloud_public_specification = clouds_public.get(public_cloud_name)
if not cloud_public_specification:
LOG.warning("%s is not a valid profile name. "
log.warning("%s is not a valid profile name. "
"Must be contained under key '%s'", public_cloud_name, CLOUD_PUBLIC_ROOT_KEY)
else:
LOG.debug("Profile found. Merging begins...")
log.debug("Profile found. Merging begins...")
try:
mergedeep.merge(cloud_full_specification, cloud_public_specification,
strategy=mergedeep.Strategy.TYPESAFE_REPLACE)
except TypeError as exc:
LOG.warning("Existing %s and %s configuration keys don't match in type: %s", CLOUDS_YAML,
log.warning("Existing %s and %s configuration keys don't match in type: %s", CLOUDS_YAML,
CLOUDS_PUBLIC_YAML, exc)
return {}
else:
LOG.debug("Using only clouds.yaml since no clouds-public profile is set.")
log.debug("Using only clouds.yaml since no clouds-public profile is set.")

if not cloud_full_specification.get("identifier"):
cloud_full_specification["identifier"] = cloud_name
else:
LOG.warning("%s is not a valid cloud name. Must be contained under key '%s'", cloud_name, CLOUD_ROOT_KEY)
log.warning("%s is not a valid cloud name. Must be contained under key '%s'", cloud_name, CLOUD_ROOT_KEY)
return cloud_full_specification


def get_cloud_specifications(configurations):
def get_cloud_specifications(configurations, log):
"""
Calls get_cloud_specification to get the cloud_specification for every configuration
@param configurations:
@param log:
@return: list of dicts: cloud_specifications of every configuration
"""
clouds, clouds_public = get_clouds_files()
LOG.debug("Loaded clouds.yml and clouds_public.yml")
clouds, clouds_public = get_clouds_files(log)
log.debug("Loaded clouds.yml and clouds_public.yml")
cloud_specifications = []
if isinstance(clouds, dict):
for configuration in configurations:
cloud = configuration.get(CLOUD_CONFIGURATION_KEY)
if cloud:
cloud_specifications.append(get_cloud_specification(cloud, clouds, clouds_public)) # might be None
cloud_specifications.append(get_cloud_specification(cloud, clouds, clouds_public, log)) # might be None
return cloud_specifications
3 changes: 2 additions & 1 deletion bibigrid/core/utility/handler/provider_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,10 @@ def get_providers(configurations, log):
Determines list of providers by provider_names and returns it.
If providers don't match a key error is thrown and the program exits with failure state 1.
:param configurations:
:param log:
:return:
"""
cloud_specifications = configuration_handler.get_cloud_specifications(configurations)
cloud_specifications = configuration_handler.get_cloud_specifications(configurations, log)
if cloud_specifications:
try:
provider_names = configuration_handler.get_list_by_key(configurations, "infrastructure")
Expand Down

0 comments on commit 5ec389f

Please sign in to comment.