🎉 Source BambooHR: Add custom fields validation (#16826)

* Add custom fields validation * Updated PR number * Deleted employees_directory_stream * Fix tests * auto-bump connector version [ci skip] * Updated docs * Updated changelog Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Sep 27, 2022 · f10550a · f10550a
1 parent 8945f16
commit f10550a
Show file tree

Hide file tree

Showing 11 changed files with 160 additions and 27 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -104,7 +104,7 @@
 - name: BambooHR
   sourceDefinitionId: 90916976-a132-4ce9-8bce-82a03dd58788
   dockerRepository: airbyte/source-bamboo-hr
-  dockerImageTag: 0.2.0
+  dockerImageTag: 0.2.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/bamboo-hr
   icon: bamboohr.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1359,7 +1359,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-bamboo-hr:0.2.0"
+- dockerImage: "airbyte/source-bamboo-hr:0.2.1"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/bamboo-hr"
     connectionSpecification:
@@ -1369,7 +1369,7 @@
       required:
       - "subdomain"
       - "api_key"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         subdomain:
           type: "string"

diff --git a/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile b/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.0
+LABEL io.airbyte.version=0.2.1
 LABEL io.airbyte.name=airbyte/source-bamboo-hr
diff --git a/airbyte-integrations/connectors/source-bamboo-hr/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bamboo-hr/acceptance-test-config.yml
@@ -11,6 +11,8 @@ tests:
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.2.0"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json
@@ -1,15 +1,5 @@
 {
   "streams": [
-    {
-      "stream": {
-        "name": "employees_directory_stream",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append_dedup"
-    },
     {
       "stream": {
         "name": "custom_reports_stream",

diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/exception.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/exception.py
@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+class BambooHrError(Exception):
+    message = ""
+
+    def __init__(self):
+        super().__init__(self.message)
+
+
+class NullFieldsError(BambooHrError):
+    message = "Field `custom_reports_fields` cannot be empty if `custom_reports_include_default_fields` is false."
+
+
+class AvailableFieldsAccessDeniedError(BambooHrError):
+    message = "You hasn't access to any report fields. Please check your access level."
+
+
+class CustomFieldsAccessDeniedError(Exception):
+    def __init__(self, denied_fields):
+        self.message = f"Access to fields: {', '.join(denied_fields)} - denied. Please check your access level."
+        super().__init__(self.message)
diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py
@@ -15,6 +15,9 @@
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 
+from .exception import AvailableFieldsAccessDeniedError, CustomFieldsAccessDeniedError, NullFieldsError
+from .utils import convert_custom_reports_fields_to_list, validate_custom_fields
+
 
 class BambooHrStream(HttpStream, ABC):
     def __init__(self, config: Mapping[str, Any]) -> None:
@@ -72,7 +75,10 @@ def schema(self):
 
     def _get_json_schema_from_config(self):
         if self.config.get("custom_reports_fields"):
-            properties = {field.strip(): {"type": ["null", "string"]} for field in self.config.get("custom_reports_fields").split(",")}
+            properties = {
+                field.strip(): {"type": ["null", "string"]}
+                for field in convert_custom_reports_fields_to_list(self.config.get("custom_reports_fields", ""))
+            }
         else:
             properties = {}
         return {
@@ -141,17 +147,25 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
         Verifies the config and attempts to fetch the fields from the meta/fields endpoint.
         """
         config = SourceBambooHr.add_authenticator_to_config(config)
+
         if not config.get("custom_reports_fields") and not config.get("custom_reports_include_default_fields"):
-            return False, AttributeError("`custom_reports_fields` cannot be empty if `custom_reports_include_default_fields` is false")
+            return False, NullFieldsError()
+
+        available_fields = MetaFieldsStream(config).read_records(sync_mode=SyncMode.full_refresh)
+        custom_fields = convert_custom_reports_fields_to_list(config.get("custom_reports_fields", ""))
+        denied_fields = validate_custom_fields(custom_fields, available_fields)
+
+        if denied_fields:
+            return False, CustomFieldsAccessDeniedError(denied_fields)
+
         try:
-            next(MetaFieldsStream(config).read_records(sync_mode=SyncMode.full_refresh))
+            next(available_fields)
             return True, None
-        except Exception as e:
-            return False, e
+        except StopIteration:
+            return False, AvailableFieldsAccessDeniedError()
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         config = SourceBambooHr.add_authenticator_to_config(config)
         return [
-            EmployeesDirectoryStream(config),
             CustomReportsStream(config),
         ]
diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json
@@ -5,7 +5,7 @@
     "title": "Bamboo HR Spec",
     "type": "object",
     "required": ["subdomain", "api_key"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "subdomain": {
         "type": "string",

diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/utils.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/utils.py
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+def convert_custom_reports_fields_to_list(custom_reports_fields: str) -> list:
+    return custom_reports_fields.split(",") if custom_reports_fields else []
+
+
+def validate_custom_fields(custom_fields, available_fields):
+    denied_fields = []
+    for custom_field in custom_fields:
+        has_access_to_custom_field = any(available_field.get("Name") == custom_field for available_field in available_fields)
+        if not has_access_to_custom_field:
+            denied_fields.append(custom_field)
+
+    return denied_fields
diff --git a/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py
@@ -19,6 +19,38 @@ def test_source_bamboo_hr_client_wrong_credentials():
     assert result.status == Status.FAILED
 
 
+@pytest.mark.parametrize(
+    "custom_reports_fields,custom_reports_include_default_fields,available_fields,expected_message",
+    [
+        (
+            "",
+            False,
+            {},
+            "NullFieldsError('Field `custom_reports_fields` cannot be empty if `custom_reports_include_default_fields` is false.')",
+        ),
+        ("", True, {}, 'AvailableFieldsAccessDeniedError("You hasn\'t access to any report fields. Please check your access level.")'),
+        (
+            "Test",
+            True,
+            [{"name": "NewTest"}],
+            "CustomFieldsAccessDeniedError('Access to fields: Test - denied. Please check your access level.')",
+        ),
+    ],
+)
+def test_check_failed(
+    config, requests_mock, custom_reports_fields, custom_reports_include_default_fields, available_fields, expected_message
+):
+    config["custom_reports_fields"] = custom_reports_fields
+    config["custom_reports_include_default_fields"] = custom_reports_include_default_fields
+    requests_mock.get("https://api.bamboohr.com/api/gateway.php/bar/v1/meta/fields", json=available_fields)
+
+    source = SourceBambooHr()
+    result = source.check(logger=AirbyteLogger, config=config)
+
+    assert result.status == Status.FAILED
+    assert result.message == expected_message
+
+
 def test_employees_directory_stream_url_base(config):
     stream = EmployeesDirectoryStream(config)
     assert stream.url_base == "https://api.bamboohr.com/api/gateway.php/bar/v1/"

diff --git a/docs/integrations/sources/bamboo-hr.md b/docs/integrations/sources/bamboo-hr.md
@@ -6,9 +6,8 @@ The BambooHr source supports Full Refresh sync. You can choose if this connector
 
 ### Output schema
 
-This connector outputs the following streams:
+This connector outputs the following stream:
 
-* [Employees](https://documentation.bamboohr.com/reference#get-employees-directory-1)
 * [Custom Reports](https://documentation.bamboohr.com/reference/request-custom-report-1)
 
 ### Features
@@ -31,9 +30,64 @@ BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-detai
 * BambooHr Account
 * BambooHr [Api key](https://documentation.bamboohr.com/docs)
 
+# Bamboo HR
+
+This page contains the setup guide and reference information for the Bamboo HR source connector.
+
+## Prerequisites
+
+* BambooHr Account
+* BambooHr [Api key](https://documentation.bamboohr.com/docs)
+
+## Setup guide
+## Step 1: Set up the Bamboo HR connector in Airbyte
+
+### For Airbyte Cloud:
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
+3. On the Set up the source page, enter the name for the Bamboo HR connector and select **Bamboo HR** from the Source type dropdown.
+3. Enter your `subdomain`
+4. Enter your `api_key`
+5. Enter your `custom_reports_fields` if need
+6. Choose `custom_reports_include_default_fields` flag value
+7. Click **Set up source**
+
+### For Airbyte OSS:
+
+1. Navigate to the Airbyte Open Source dashboard
+2. Set the name for your source 
+3. Enter your `subdomain`
+4. Enter your `api_key`
+5. Enter your `custom_reports_fields` if need
+6. Choose `custom_reports_include_default_fields` flag value
+7. Click **Set up source**
+
+## Supported sync modes
+
+The Bamboo HR source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+| Feature | Supported? |
+| :--- | :--- |
+| Full Refresh Sync | Yes |
+| Incremental - Append Sync | No |
+| SSL connection | Yes |
+| Namespaces | No |
+
+
+## Supported Streams
+
+* [Custom Reports](https://documentation.bamboohr.com/reference/request-custom-report-1)
+
+## Performance considerations
+
+BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-details), but the connector should not run into API limitations under normal usage. 
+Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully.
+
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-|:--------| :--- | :--- | :--- |
-| 0.2.0   | 2022-03-24 | [11326](https://github.com/airbytehq/airbyte/pull/11326) | Added support for Custom Reports endpoint |
-| 0.1.0   | 2021-08-27 | [5054](https://github.com/airbytehq/airbyte/pull/5054) | Initial release with Employees API |
+| Version | Date       | Pull Request                                             | Subject                                   |
+|:--------| :--------- | :------------------------------------------------------  | :---------------------------------------- |
+| 0.2.1   | 2022-09-16 | [16826](https://github.com/airbytehq/airbyte/pull/16826) | Add custom fields validation during check |
+| 0.2.0   | 2022-03-24 | [11326](https://github.com/airbytehq/airbyte/pull/11326) | Add support for Custom Reports endpoint   |
+| 0.1.0   | 2021-08-27 | [5054](https://github.com/airbytehq/airbyte/pull/5054)   | Initial release with Employees API        |