Merge branch '2.0' into vid_club_2.0

.secrets.baseline

@@ -361,14 +361,14 @@
         "filename": "tests/unit/test_outlook.py",
         "hashed_secret": "b235838f76594bf21886c6eec9c06a207e9ec5ce",
         "is_verified": false,
-        "line_number": 18
+        "line_number": 17
       },
       {
         "type": "Secret Keyword",
         "filename": "tests/unit/test_outlook.py",
         "hashed_secret": "1089adfb1f11b95df31344030507912b5abdf57a",
         "is_verified": false,
-        "line_number": 49
+        "line_number": 48
       }
     ],
     "tests/unit/test_redshift_spectrum.py": [
@@ -397,5 +397,5 @@
       }
     ]
   },
-  "generated_at": "2024-09-23T12:45:10Z"
+  "generated_at": "2024-10-03T09:38:32Z"
 }

src/viadot/sources/customer_gauge.py

@@ -7,7 +7,7 @@
 from pydantic import BaseModel
 
 from viadot.config import get_source_credentials
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources.base import Source
 from viadot.utils import (
     add_viadot_metadata_columns,
@@ -64,13 +64,9 @@ def __init__(
             CredentialError: If credentials are not provided in local_config or
                 directly as a parameter.
         """
-        credentials = credentials or get_source_credentials(config_key) or None
-        if credentials is None:
-            message = "Missing credentials."
-            raise CredentialError(message)
-        self.credentials = credentials
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(CustomerGaugeCredentials(**raw_creds))
 
-        validated_creds = dict(CustomerGaugeCredentials(**credentials))
         super().__init__(*args, credentials=validated_creds, **kwargs)
 
         self.clean_json = None

src/viadot/sources/epicor.py

@@ -388,13 +388,9 @@ def __init__(
             end_date_field (str, optional) The name of filters field containing
                 end date. Defaults to "EndInvoiceDate".
         """
-        raw_creds = credentials or get_source_credentials(config_key) or {}
-        validated_creds = EpicorCredentials(**raw_creds).dict(
-            by_alias=True
-        )  # validate the credentials
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(EpicorCredentials(**raw_creds))
 
-        self.credentials = validated_creds
-        self.config_key = config_key
         self.base_url = base_url
         self.validate_date_filter = validate_date_filter
         self.start_date_field = start_date_field

src/viadot/sources/genesys.py

@@ -14,7 +14,7 @@
 from pydantic import BaseModel
 
 from viadot.config import get_source_credentials
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources.base import Source
 from viadot.utils import add_viadot_metadata_columns, handle_api_response, validate
 
@@ -109,18 +109,14 @@ def __init__(
                 as a parameter.
             APIError: When the environment variable is not among the available.
         """
-        credentials = credentials or get_source_credentials(config_key) or None
-        if credentials is None:
-            msg = "Missing credentials."
-            raise CredentialError(msg)
-        self.credentials = credentials
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(GenesysCredentials(**raw_creds))
 
-        validated_creds = dict(GenesysCredentials(**credentials))
         super().__init__(*args, credentials=validated_creds, **kwargs)
 
         self.verbose = verbose
         self.data_returned = {}
-        self.new_report = "{}"
+        self.new_report = "{}"  # ???
 
         if environment in self.ENVIRONMENTS:
             self.environment = environment
@@ -302,10 +298,6 @@ def _get_reporting_exports_url(self, entities: list[str]) -> tuple[list[str]]:
         for entity in entities:
             ids.append(entity.get("id"))
             urls.append(entity.get("downloadUrl"))
-            # entity.get("filter").get("queueIds", [-1])[0],
-            # entity.get("filter").get("mediaTypes", [-1])[0],
-            # entity.get("viewType"),
-            # entity.get("interval"),
             status.append(entity.get("status"))
 
         if "FAILED" in status:
@@ -566,7 +558,7 @@ def _merge_conversations(self, data_to_merge: list) -> pd.DataFrame:  # noqa: C9
 
             conversations_df.update({i: dff3})
 
-        # NERGING ALL LEVELS
+        # MERGING ALL LEVELS
         # LEVELS 3
         for i_3, key in enumerate(list(conversations_df.keys())):
             if i_3 == 0:

src/viadot/sources/hubspot.py

@@ -9,7 +9,7 @@
 from pydantic import BaseModel
 
 from viadot.config import get_source_credentials
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources.base import Source
 from viadot.utils import add_viadot_metadata_columns, handle_api_response
 
@@ -74,13 +74,8 @@ def __init__(
             CredentialError: If credentials are not provided in local_config or
                 directly as a parameter.
         """
-        credentials = credentials or get_source_credentials(config_key) or None
-        if credentials is None:
-            msg = "Missing credentials."
-            raise CredentialError(msg)
-        self.credentials = credentials
-
-        validated_creds = dict(HubspotCredentials(**credentials))
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(HubspotCredentials(**raw_creds))
         super().__init__(*args, credentials=validated_creds, **kwargs)
 
         self.full_dataset = None

src/viadot/sources/outlook.py

@@ -9,7 +9,6 @@
 from pydantic import BaseModel
 
 from viadot.config import get_source_credentials
-from viadot.exceptions import CredentialError
 from viadot.sources.base import Source
 from viadot.utils import add_viadot_metadata_columns
 
@@ -78,14 +77,10 @@ def __init__(
             CredentialError: If credentials are not provided in local_config or
                 directly as a parameter.
         """
-        credentials = credentials or get_source_credentials(config_key) or None
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(OutlookCredentials(**raw_creds))
 
-        if credentials is None or not isinstance(credentials, dict):
-            msg = "Missing credentials."
-            raise CredentialError(msg)
-        self.credentials = dict(OutlookCredentials(**credentials))
-
-        super().__init__(*args, credentials=self.credentials, **kwargs)
+        super().__init__(*args, credentials=validated_creds, **kwargs)
 
     @staticmethod
     def _get_subfolders(

src/viadot/sources/redshift_spectrum.py

@@ -1,6 +1,5 @@
 """Amazon Redshift Spectrum connector."""
 
-import os
 from typing import Literal
 
 import pandas as pd
@@ -28,6 +27,7 @@ class RedshiftSpectrumCredentials(BaseModel):
     aws_access_key_id: str  # The AWS access key ID.
     aws_secret_access_key: str  # The AWS secret access key.
     profile_name: str | None = None  # The name of the IAM profile to use.
+    endpoint_url: str | None = None  # The endpoint of the S3 service.
 
     # Below credentials are required only by some methods.
     #
@@ -88,14 +88,8 @@ def __init__(
             redshift.get_schemas()
         ```
         """
-        raw_creds = (
-            credentials
-            or get_source_credentials(config_key)
-            or self._get_env_credentials()
-        )
-        validated_creds = dict(
-            RedshiftSpectrumCredentials(**raw_creds)
-        )  # validate the credentials
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(RedshiftSpectrumCredentials(**raw_creds))
 
         super().__init__(*args, credentials=validated_creds, **kwargs)
 
@@ -127,6 +121,7 @@ def session(self) -> boto3.session.Session:
                 profile_name=self.credentials.get("profile_name"),
                 aws_access_key_id=self.credentials.get("aws_access_key_id"),
                 aws_secret_access_key=self.credentials.get("aws_secret_access_key"),
+                endpoint_url=self.credentials.get("endpoint_url"),
             )
         return self._session
 
@@ -145,13 +140,6 @@ def con(self) -> redshift_connector.Connection:
                 raise ValueError(msg)
         return self._con
 
-    def _get_env_credentials(self):
-        return {
-            "region_name": os.environ.get("AWS_DEFAULT_REGION"),
-            "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID"),
-            "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY"),
-        }
-
     def from_df(
         self,
         df: pd.DataFrame,

src/viadot/sources/sap_bw.py

@@ -8,7 +8,7 @@
 import pyrfc
 
 from viadot.config import get_source_credentials
-from viadot.exceptions import CredentialError, ValidationError
+from viadot.exceptions import ValidationError
 from viadot.sources.base import Source
 from viadot.utils import add_viadot_metadata_columns
 
@@ -75,13 +75,9 @@ def __init__(
             CredentialError: If credentials are not provided in local_config or
                 directly as a parameter.
         """
-        credentials = credentials or get_source_credentials(config_key) or None
-        if credentials is None:
-            message = "Missing credentials."
-            raise CredentialError(message)
-        self.credentials = credentials
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(SAPBWCredentials(**raw_creds))
 
-        validated_creds = dict(SAPBWCredentials(**credentials))
         super().__init__(*args, credentials=validated_creds, **kwargs)
 
         self.query_output = None

src/viadot/sources/supermetrics.py

@@ -8,10 +8,8 @@
 from pydantic import BaseModel
 
 from viadot.config import get_source_credentials
-
-from ..exceptions import CredentialError
-from ..utils import add_viadot_metadata_columns, handle_api_response
-from .base import Source
+from viadot.sources.base import Source
+from viadot.utils import add_viadot_metadata_columns, handle_api_response
 
 
 class SupermetricsCredentials(BaseModel):
@@ -59,7 +57,7 @@ class Supermetrics(Source):
 
     """
 
-    API_ENDPOINT = "https://api.supermetrics.com/enterprise/v2/query/data/json"
+    BASE_URL = "https://api.supermetrics.com/enterprise/v2/query/data/json"
 
     def __init__(
         self,
@@ -87,18 +85,12 @@ def __init__(
                 parameters define the specifics of the data request. Defaults to None.
 
         """
-        credentials = credentials or get_source_credentials(config_key) or None
-
-        if credentials is None or not isinstance(credentials, dict):
-            msg = "Missing credentials."
-            raise CredentialError(msg)
-        self.credentials = dict(SupermetricsCredentials(**credentials))
+        raw_creds = credentials or get_source_credentials(config_key)
+        validated_creds = dict(SupermetricsCredentials(**raw_creds))
 
-        super().__init__(*args, credentials=self.credentials, **kwargs)
+        super().__init__(*args, credentials=validated_creds, **kwargs)
 
         self.query_params = query_params
-        self.api_key = self.credentials["api_key"]
-        self.user = self.credentials["user"]
 
     def to_json(self, timeout: tuple = (3.05, 60 * 30)) -> dict[str, Any]:
         """Download query results to a dictionary.
@@ -131,10 +123,10 @@ def to_json(self, timeout: tuple = (3.05, 60 * 30)) -> dict[str, Any]:
             raise ValueError(msg)
 
         params = {"json": json.dumps(self.query_params)}
-        headers = {"Authorization": f"Bearer {self.api_key}"}
+        headers = {"Authorization": f"Bearer {self.credentials.get('api_key')}"}
 
         response = handle_api_response(
-            url=self.API_ENDPOINT,
+            url=Supermetrics.BASE_URL,
             params=params,
             headers=headers,
             timeout=timeout,
@@ -264,7 +256,7 @@ def to_df(
             msg = "Query parameters are required to fetch data."
             raise ValueError(msg)
 
-        self.query_params["api_key"] = self.api_key
+        self.query_params["api_key"] = self.credentials.get("api_key")
 
         try:
             columns = self._get_col_names()

tests/unit/test_customer_gauge.py

@@ -4,7 +4,7 @@
 
 import pytest  # type: ignore
 
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources.customer_gauge import CustomerGauge, CustomerGaugeCredentials
 
 
@@ -28,7 +28,7 @@ def test_customer_gauge_credentials():
 @pytest.mark.basic
 def test_missing_credentials():
     """Test raise error without Customer Gauge credentials."""
-    with pytest.raises(CredentialError):
+    with pytest.raises(TypeError):
         CustomerGauge(config_key="invalid_key")
 
 

tests/unit/test_genesys.py

@@ -6,7 +6,7 @@
 import pandas as pd
 import pytest
 
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources import Genesys
 
 
@@ -101,7 +101,7 @@ def genesys():
 @patch("viadot.sources.genesys.get_source_credentials", return_value=None)
 def test_init_no_credentials(mock_get_source_credentials):
     """Test raise error without credentials."""
-    with pytest.raises(CredentialError):
+    with pytest.raises(TypeError):
         Genesys()
 
     mock_get_source_credentials.assert_called_once()

tests/unit/test_hubspot.py

@@ -8,7 +8,7 @@
 import pytest
 from requests.models import Response
 
-from viadot.exceptions import APIError, CredentialError
+from viadot.exceptions import APIError
 from viadot.sources import Hubspot
 from viadot.sources.hubspot import HubspotCredentials
 
@@ -48,7 +48,7 @@ def setUpClass(cls):  # noqa: ANN206
     @patch("viadot.sources.hubspot.get_source_credentials", return_value=None)
     def test_init_no_credentials(self, mock_get_source_credentials):
         """Test raise error without credentials."""
-        with pytest.raises(CredentialError):
+        with pytest.raises(TypeError):
             Hubspot()
 
         mock_get_source_credentials.assert_called_once()

tests/unit/test_outlook.py

@@ -7,7 +7,6 @@
 import pandas as pd
 import pytest
 
-from viadot.exceptions import CredentialError
 from viadot.sources import Outlook
 from viadot.sources.outlook import OutlookCredentials
 
@@ -62,7 +61,7 @@ def setUpClass(cls):  # noqa: ANN206
     @patch("viadot.sources.outlook.get_source_credentials", return_value=None)
     def test_missing_credentials(self, mock_get_source_credentials):
         """Test raise error without credentials."""
-        with pytest.raises(CredentialError):
+        with pytest.raises(TypeError):
             Outlook(credentials=None)
 
         mock_get_source_credentials.assert_called_once()