Skip to content

Commit

Permalink
HDXDSYS-589 kwargs take precedence over environment variables (#64)
Browse files Browse the repository at this point in the history
* Make kwargs take precedence over environment variables which take precedence over configuration dictionary or file

* Fix test
  • Loading branch information
mcarans authored Mar 20, 2024
1 parent 49fef22 commit d2deba6
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 29 deletions.
6 changes: 6 additions & 0 deletions documentation/main.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ The library has detailed API documentation which can be found in the menu at the


## Breaking Changes
From 6.2.6, kwargs take preference over environment variables which take
preference over configuration files

From 6.2.5, environment variables take preference over kwargs which take
preference over configuration files

From 6.1.5, any method or parameter with "reference_period" in it is renamed
to "time_period" and any method or parameter with "file_type" in it is renamed
to "format"
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ urllib3==2.2.1
# via
# libhxl
# requests
validators==0.22.0
validators==0.23.1
# via frictionless
virtualenv==20.25.1
# via pre-commit
Expand Down
34 changes: 19 additions & 15 deletions src/hdx/api/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class Configuration(UserDict):
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
Expand Down Expand Up @@ -198,18 +198,22 @@ def __init__(self, **kwargs: Any) -> None:
prefix=Configuration.prefix, **self.data
)

hdx_url = os.getenv(
"HDX_URL", kwargs.get("hdx_url", self.data.get("hdx_url"))
)
hdx_url = kwargs.get("hdx_url")
if not hdx_url:
hdx_url = os.getenv("HDX_URL")
if not hdx_url:
hdx_url = self.data.get("hdx_url")
if hdx_url:
hdx_site = "custom"
self.hdx_site = "hdx_custom_site"
hdx_url = hdx_url.rstrip("/")
self.data[self.hdx_site] = {"url": hdx_url}
else:
hdx_site = os.getenv(
"HDX_SITE", kwargs.get("hdx_site", self.data.get("hdx_site"))
)
hdx_site = kwargs.get("hdx_site")
if not hdx_site:
hdx_site = os.getenv("HDX_SITE")
if not hdx_site:
hdx_site = self.data.get("hdx_site")
if not hdx_site:
hdx_site = "stage"
self.hdx_site = f"hdx_{hdx_site}_site"
Expand All @@ -228,16 +232,16 @@ def __init__(self, **kwargs: Any) -> None:
if self.hdx_site == "hdx_custom_site":
hdx_key = None
else:
hdx_key = os.getenv(f"HDX_KEY_{hdx_site.upper()}")
hdx_key_site = f"hdx_key_{hdx_site}"
hdx_key = kwargs.get(hdx_key_site)
if not hdx_key:
hdx_key_site = f"hdx_key_{hdx_site}"
hdx_key = kwargs.get(hdx_key_site)
hdx_key = os.getenv(f"HDX_KEY_{hdx_site.upper()}")
if not hdx_key:
hdx_key = self.data.get(hdx_key_site)
if not hdx_key:
hdx_key = os.getenv("HDX_KEY")
hdx_key = kwargs.get("hdx_key")
if not hdx_key:
hdx_key = kwargs.get("hdx_key")
hdx_key = os.getenv("HDX_KEY")
if not hdx_key:
hdx_key = self.data.get("hdx_key")
if hdx_key:
Expand Down Expand Up @@ -529,7 +533,7 @@ def setup(
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
Expand Down Expand Up @@ -569,7 +573,7 @@ def _create(
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
Expand Down Expand Up @@ -608,7 +612,7 @@ def create(
user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yaml.
user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
hdx_url (str): HDX url to use. Overrides hdx_site.
hdx_site (str): HDX site to use eg. prod, test.
hdx_site (str): HDX site to use eg. prod, test. Defaults to stage.
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
Expand Down
18 changes: 17 additions & 1 deletion tests/hdx/api/test_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,26 @@ def test_init(
Configuration.default_hdx_config_yaml = "NOT EXIST"
with pytest.raises(UserAgentError):
Configuration()
with pytest.raises(ConfigurationError):
Configuration(user_agent="test")
configuration = Configuration(user_agent="test", hdx_key="test")
assert configuration.hdx_read_only is False
assert (
configuration.get_hdx_site_url()
== "https://stage.data-humdata-org.ahconu.org"
)
assert (
configuration.get_user_agent()
== f"HDXPythonLibrary/{__version__}-test"
)
configuration = Configuration(full_agent="test", hdx_key="test")
assert configuration.get_user_agent() == "test"
Configuration.default_hdx_config_yaml = default_config_file

Configuration.default_hdx_config_yaml = hdx_config_yaml
assert Configuration(user_agent="test").get_api_key() == "12345"
configuration = Configuration(user_agent="test")
assert configuration.get_api_key() == "12345"

Configuration.default_hdx_config_yaml = default_config_file

with pytest.raises(IOError):
Expand Down
12 changes: 6 additions & 6 deletions tests/hdx/facades/test_keyword_arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
assert testresult.actual_result == my_test_key
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv("HDX_KEY", my_test_key)
monkeypatch.setenv("HDX_KEY", "aaaa")
facade(
my_testfnkw,
hdx_key="aaaa",
hdx_key=my_test_key,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand All @@ -117,10 +117,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
)
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv("HDX_SITE", my_test_hdxsite)
monkeypatch.setenv("HDX_SITE", "feature")
facade(
my_testfnkw,
hdx_site="feature",
hdx_site=my_test_hdxsite,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand All @@ -143,11 +143,11 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
)
assert testresult.actual_result == my_test_hdxurl
UserAgent.clear_global()
my_test_hdxurl2 = "http://other-data.humdata.org/"
my_test_hdxurl2 = "http://other2-data.humdata.org/"
monkeypatch.setenv("HDX_URL", my_test_hdxurl2)
facade(
my_testfnkw,
hdx_site="feature",
hdx_url=my_test_hdxurl,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand Down
12 changes: 6 additions & 6 deletions tests/hdx/facades/test_simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
assert testresult.actual_result == my_test_key
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv("HDX_KEY", my_test_key)
monkeypatch.setenv("HDX_KEY", "aaaa")
facade(
my_testkeyfn,
hdx_key="aaaa",
hdx_key=my_test_key,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand All @@ -109,10 +109,10 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
)
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv("HDX_SITE", my_test_hdxsite)
monkeypatch.setenv("HDX_SITE", "feature")
facade(
my_testfn,
hdx_site="feature",
hdx_site=my_test_hdxsite,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand All @@ -133,11 +133,11 @@ def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
)
assert testresult.actual_result == my_test_hdxurl
UserAgent.clear_global()
my_test_hdxurl2 = "http://other-data.humdata.org/"
my_test_hdxurl2 = "http://other2-data.humdata.org/"
monkeypatch.setenv("HDX_URL", my_test_hdxurl2)
facade(
my_testfn,
hdx_site="feature",
hdx_url=my_test_hdxurl,
user_agent=my_user_agent,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml,
Expand Down

0 comments on commit d2deba6

Please sign in to comment.