diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index 4d606c8cd..4f21927ad 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -6,6 +6,7 @@ saas_config: connector_params: - name: domain + default_value: api.hubapi.com - name: hapikey client_config: @@ -27,8 +28,8 @@ saas_config: read: path: /crm/v3/objects/contacts/search method: POST - body: - '{ + body: | + { "filterGroups": [{ "filters": [{ "value": "", @@ -36,7 +37,7 @@ saas_config: "operator": "EQ" }] }] - }' + } query_params: - name: limit value: 100 @@ -55,7 +56,10 @@ saas_config: update: path: /crm/v3/objects/contacts/ method: PATCH - body: '{}' + body: | + { + + } param_values: - name: contactId references: @@ -95,13 +99,13 @@ saas_config: update: path: /communication-preferences/v3/unsubscribe method: POST - body: - '{ + body: | + { "emailAddress": "", "subscriptionId": "", "legalBasis": "LEGITIMATE_INTEREST_CLIENT", "legalBasisExplanation": "At users request, we opted them out" - }' + } data_path: subscriptionStatuses param_values: - name: email @@ -115,4 +119,4 @@ saas_config: - strategy: filter configuration: field: status - value: SUBSCRIBED \ No newline at end of file + value: SUBSCRIBED diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index 4b864f807..3f83c8c1c 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -21,70 +21,73 @@ saas_config: test_request: method: GET path: /3.0/lists - + endpoints: - - name: messages - requests: - read: - method: GET - path: /3.0/conversations//messages - param_values: - - name: conversation_id - references: - - dataset: mailchimp_connector_example - field: conversations.id - direction: from - data_path: conversation_messages - postprocessors: - - strategy: filter + - name: messages + requests: + read: + method: GET + path: /3.0/conversations//messages + param_values: + - name: conversation_id + references: + - dataset: mailchimp_connector_example + field: conversations.id + direction: from + data_path: conversation_messages + postprocessors: + - strategy: filter + configuration: + field: from_email + value: + identity: email + - name: conversations + requests: + read: + method: GET + path: /3.0/conversations + query_params: + - name: count + value: 1000 + - name: offset + value: 0 + param_values: + - name: placeholder + identity: email + data_path: conversations + pagination: + strategy: offset configuration: - field: from_email - value: - identity: email - - name: conversations - requests: - read: - method: GET - path: /3.0/conversations - query_params: - - name: count - value: 1000 - - name: offset - value: 0 - param_values: - - name: placeholder - identity: email - data_path: conversations - pagination: - strategy: offset - configuration: - incremental_param: offset - increment_by: 1000 - limit: 10000 - - name: member - requests: - read: - method: GET - path: /3.0/search-members - query_params: - - name: query - value: - param_values: - - name: email - identity: email - data_path: exact_matches.members - update: - method: PUT - path: /3.0/lists//members/ - param_values: - - name: list_id - references: - - dataset: mailchimp_connector_example - field: member.list_id - direction: from - - name: subscriber_hash - references: - - dataset: mailchimp_connector_example - field: member.id - direction: from - body: '{}' + incremental_param: offset + increment_by: 1000 + limit: 10000 + - name: member + requests: + read: + method: GET + path: /3.0/search-members + query_params: + - name: query + value: + param_values: + - name: email + identity: email + data_path: exact_matches.members + update: + method: PUT + path: /3.0/lists//members/ + param_values: + - name: list_id + references: + - dataset: mailchimp_connector_example + field: member.list_id + direction: from + - name: subscriber_hash + references: + - dataset: mailchimp_connector_example + field: member.id + direction: from + body: | + { + + } diff --git a/data/saas/config/outreach_config.yml b/data/saas/config/outreach_config.yml index 56aecbc42..e31b77e1c 100644 --- a/data/saas/config/outreach_config.yml +++ b/data/saas/config/outreach_config.yml @@ -6,6 +6,7 @@ saas_config: connector_params: - name: domain + default_value: platform.segmentapis.com - name: requester_email description: The email of the Outreach user to associate with each automated compliance request (data_protection_request) - name: client_id diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index 61cabd164..da29c199b 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -6,10 +6,11 @@ saas_config: connector_params: - name: domain + default_value: localhost - name: username - name: api_key - name: api_version - - name: page_limit + - name: page_size client_config: protocol: https @@ -23,152 +24,161 @@ saas_config: test_request: method: GET path: /3.0/lists - + endpoints: - - name: messages - requests: - read: - method: GET - path: /3.0/conversations//messages - param_values: - - name: conversation_id - references: - - dataset: saas_connector_example - field: conversations.id - direction: from - postprocessors: - - strategy: unwrap - configuration: - data_path: conversation_messages - - strategy: filter - configuration: - field: from_email - value: - identity: email - - name: conversations - requests: - read: - method: GET - ignore_errors: true - path: /3.0/conversations - query_params: - - name: count - value: 1000 - - name: offset - value: 0 - param_values: - - name: placeholder - identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: conversations - - name: member - requests: - read: - method: GET - path: /3.0/search-members - query_params: - - name: query - value: - param_values: - - name: email - identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: exact_matches.members - update: - method: PUT - path: /3.0/lists//members/ - param_values: - - name: list_id - references: - - dataset: saas_connector_example - field: member.list_id - direction: from - - name: subscriber_hash - references: - - dataset: saas_connector_example - field: member.id - direction: from - body: '{}' - - name: payment_methods - requests: - read: - method: GET - path: //payment_methods - headers: - - name: Content-Type - value: application/json - - name: On-Behalf-Of - value: - - name: Token - value: Custom - query_params: - - name: limit - value: - - name: query - value: - param_values: - - name: limit - connector_param: page_limit - - name: version - connector_param: api_version - - name: email - identity: email - - name: api_key - connector_param: api_key - update: - method: PUT - path: //payment_methods - param_values: - - name: version - connector_param: api_version - body: '{}' - - name: projects - requests: - read: - method: GET - path: /api/0/projects/ - param_values: - - name: placeholder - identity: email - - name: users - after: [saas_connector_example.projects] - requests: - read: - method: GET - path: /api/0/projects///user-feedback/ - query_params: - - name: query - value: - grouped_inputs: [organization_slug, project_slug, query] - param_values: - - name: organization_slug - references: - - dataset: saas_connector_example - field: projects.organization.slug - direction: from - - name: project_slug - references: - - dataset: saas_connector_example - field: projects.slug - direction: from - - name: query - identity: email - - name: customer - requests: - read: - method: GET - path: /v1/customers/1 - param_values: - - name: placeholder - identity: email - update: - method: POST - path: /v1/customers/1 - headers: - - name: Content-Type - value: application/x-www-form-urlencoded - body: '{}' \ No newline at end of file + - name: messages + requests: + read: + method: GET + path: /3.0/conversations//messages + param_values: + - name: conversation_id + references: + - dataset: saas_connector_example + field: conversations.id + direction: from + postprocessors: + - strategy: unwrap + configuration: + data_path: conversation_messages + - strategy: filter + configuration: + field: from_email + value: + identity: email + - name: conversations + requests: + read: + method: GET + ignore_errors: true + path: /3.0/conversations + query_params: + - name: count + value: 1000 + - name: offset + value: 0 + param_values: + - name: placeholder + identity: email + postprocessors: + - strategy: unwrap + configuration: + data_path: conversations + - name: member + requests: + read: + method: GET + path: /3.0/search-members + query_params: + - name: query + value: + param_values: + - name: email + identity: email + postprocessors: + - strategy: unwrap + configuration: + data_path: exact_matches.members + update: + method: PUT + path: /3.0/lists//members/ + param_values: + - name: list_id + references: + - dataset: saas_connector_example + field: member.list_id + direction: from + - name: subscriber_hash + references: + - dataset: saas_connector_example + field: member.id + direction: from + body: | + { + + } + - name: payment_methods + requests: + read: + method: GET + path: //payment_methods + headers: + - name: Content-Type + value: application/json + - name: On-Behalf-Of + value: + - name: Token + value: Custom + query_params: + - name: limit + value: + - name: query + value: + param_values: + - name: limit + connector_param: page_size + - name: version + connector_param: api_version + - name: email + identity: email + - name: api_key + connector_param: api_key + update: + method: PUT + path: //payment_methods + param_values: + - name: version + connector_param: api_version + body: | + { + + } + - name: projects + requests: + read: + method: GET + path: /api/0/projects/ + param_values: + - name: placeholder + identity: email + - name: users + after: [saas_connector_example.projects] + requests: + read: + method: GET + path: /api/0/projects///user-feedback/ + query_params: + - name: query + value: + grouped_inputs: [organization_slug, project_slug, query] + param_values: + - name: organization_slug + references: + - dataset: saas_connector_example + field: projects.organization.slug + direction: from + - name: project_slug + references: + - dataset: saas_connector_example + field: projects.slug + direction: from + - name: query + identity: email + - name: customer + requests: + read: + method: GET + path: /v1/customers/1 + param_values: + - name: placeholder + identity: email + update: + method: POST + path: /v1/customers/1 + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + + } diff --git a/data/saas/config/segment_config.yml b/data/saas/config/segment_config.yml index 906522695..ba413a251 100644 --- a/data/saas/config/segment_config.yml +++ b/data/saas/config/segment_config.yml @@ -6,13 +6,14 @@ saas_config: connector_params: - name: domain + default_value: platform.segmentapis.com - name: personas_domain + default_value: profiles.segment.com - name: workspace - name: access_token - name: namespace_id - name: access_secret - client_config: protocol: https host: @@ -26,104 +27,104 @@ saas_config: path: /v1beta/workspaces/ endpoints: - - name: segment_user - requests: - read: - method: GET - path: /v1/spaces//collections/users/profiles/user_id:/metadata - param_values: - - name: namespace_id - connector_param: namespace_id - - name: user_id - identity: email - client_config: - protocol: https - host: - authentication: - strategy: basic - configuration: - username: - - name: track_events - requests: - read: - method: GET - path: /v1/spaces//collections/users/profiles//events - param_values: - - name: namespace_id - connector_param: namespace_id - - name: segment_id - references: - - dataset: segment_connector_example - field: segment_user.segment_id - direction: from - data_path: data - pagination: - strategy: link - configuration: - source: body - path: cursor.url - client_config: - protocol: https - host: - authentication: - strategy: basic + - name: segment_user + requests: + read: + method: GET + path: /v1/spaces//collections/users/profiles/user_id:/metadata + param_values: + - name: namespace_id + connector_param: namespace_id + - name: user_id + identity: email + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + - name: track_events + requests: + read: + method: GET + path: /v1/spaces//collections/users/profiles//events + param_values: + - name: namespace_id + connector_param: namespace_id + - name: segment_id + references: + - dataset: segment_connector_example + field: segment_user.segment_id + direction: from + data_path: data + pagination: + strategy: link configuration: - username: - - name: traits - requests: - read: - method: GET - path: /v1/spaces//collections/users/profiles//traits - query_params: - - name: limit - value: 17 - param_values: - - name: namespace_id - connector_param: namespace_id - - name: segment_id - references: - - dataset: segment_connector_example - field: segment_user.segment_id - direction: from - data_path: traits - pagination: - strategy: link - configuration: - source: body - path: cursor.url - client_config: - protocol: https - host: - authentication: - strategy: basic + source: body + path: cursor.url + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + - name: traits + requests: + read: + method: GET + path: /v1/spaces//collections/users/profiles//traits + query_params: + - name: limit + value: 17 + param_values: + - name: namespace_id + connector_param: namespace_id + - name: segment_id + references: + - dataset: segment_connector_example + field: segment_user.segment_id + direction: from + data_path: traits + pagination: + strategy: link configuration: - username: - - name: external_ids - requests: - read: - method: GET - path: /v1/spaces//collections/users/profiles//external_ids - param_values: - - name: namespace_id - connector_param: namespace_id - - name: segment_id - references: - - dataset: segment_connector_example - field: segment_user.segment_id - direction: from - data_path: data - pagination: - strategy: link - configuration: - source: body - path: cursor.url - client_config: - protocol: https - host: - authentication: - strategy: basic + source: body + path: cursor.url + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + - name: external_ids + requests: + read: + method: GET + path: /v1/spaces//collections/users/profiles//external_ids + param_values: + - name: namespace_id + connector_param: namespace_id + - name: segment_id + references: + - dataset: segment_connector_example + field: segment_user.segment_id + direction: from + data_path: data + pagination: + strategy: link configuration: - username: + source: body + path: cursor.url + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: data_protection_request: method: POST @@ -136,11 +137,18 @@ saas_config: connector_param: workspace - name: user_id identity: email - body: '{"regulation_type": "Suppress_With_Delete", "attributes": {"name": "userId", "values": [""]}}' + body: | + { + "regulation_type": "Suppress_With_Delete", + "attributes": { + "name": "userId", + "values": [""] + } + } client_config: protocol: https host: authentication: strategy: bearer configuration: - token: \ No newline at end of file + token: diff --git a/data/saas/config/sentry_config.yml b/data/saas/config/sentry_config.yml index 776e60071..413a56df4 100644 --- a/data/saas/config/sentry_config.yml +++ b/data/saas/config/sentry_config.yml @@ -5,12 +5,13 @@ saas_config: version: 0.0.1 connector_params: - - name: host + - name: domain + default_value: sentry.io - name: access_token client_config: protocol: https - host: + host: authentication: strategy: bearer configuration: @@ -78,7 +79,10 @@ saas_config: - dataset: sentry_connector field: issues.id direction: from - body: '{"assignedTo": ""}' + body: | + { + "assignedTo": "" + } read: method: GET path: /api/0/projects///issues/ diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index 548511644..be86e8272 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -5,14 +5,15 @@ saas_config: version: 0.0.1 connector_params: - - name: host + - name: domain + default_value: api.stripe.com - name: api_key - name: payment_types - - name: items_per_page + - name: page_size client_config: protocol: https - host: + host: authentication: strategy: bearer configuration: @@ -21,415 +22,427 @@ saas_config: test_request: method: GET path: /v1/customers - + endpoints: - - name: customer - requests: - read: - method: GET - path: /v1/customers - query_params: - - name: email - value: - param_values: - - name: email - identity: email - data_path: data - update: - method: POST - path: /v1/customers/ - headers: - - name: Content-Type - value: application/x-www-form-urlencoded - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - body: '{}' - - name: charge - requests: - read: - method: GET - path: /v1/charges - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: dispute - requests: - read: - method: GET - path: /v1/disputes - query_params: - - name: charge - value: - - name: payment_intent - value: - - name: limit - value: - param_values: - - name: charge_id - references: - - dataset: stripe_connector_example - field: charge.id - direction: from - - name: payment_intent_id - references: - - dataset: stripe_connector_example - field: payment_intent.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: payment_intent - requests: - read: - method: GET - path: /v1/payment_intents - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: payment_method - requests: - read: - method: GET - path: /v1/customers//payment_methods - query_params: - - name: type - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: type - connector_param: payment_types - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - update: - method: POST - path: /v1/payment_methods/ - headers: - - name: Content-Type - value: application/x-www-form-urlencoded - param_values: - - name: payment_method_id - references: - - dataset: stripe_connector_example - field: payment_method.id - direction: from - body: '{}' - - name: bank_account - requests: - read: - method: GET - path: /v1/customers//sources - query_params: - - name: object - value: bank_account - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - update: - method: POST - path: /v1/customers//sources/ - headers: - - name: Content-Type - value: application/x-www-form-urlencoded - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: bank_account.customer - direction: from - - name: bank_account_id - references: - - dataset: stripe_connector_example - field: bank_account.id - direction: from - body: '{}' - - name: card - requests: - read: - method: GET - path: /v1/customers//sources - query_params: - - name: object - value: card - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - update: - method: POST - path: /v1/customers//sources/ - headers: - - name: Content-Type - value: application/x-www-form-urlencoded - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: card.customer - direction: from - - name: card_id - references: - - dataset: stripe_connector_example - field: card.id - direction: from - body: '{}' - - name: credit_note - requests: - read: - method: GET - path: /v1/credit_notes - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: customer_balance_transaction - requests: - read: - method: GET - path: /v1/customers//balance_transactions - query_params: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - - name: tax_id - requests: - read: - method: GET - path: /v1/customers//tax_ids - query_params: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - delete: - method: DELETE - path: /v1/customers//tax_ids/ - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: tax_id.customer - direction: from - - name: tax_id - references: - - dataset: stripe_connector_example - field: tax_id.id - direction: from - - name: invoice - requests: - read: - method: GET - path: /v1/invoices - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - delete: - method: DELETE - ignore_errors: true # You can only delete draft invoices. You can't delete invoices created by subscriptions. - path: /v1/invoices/ - param_values: - - name: invoice_id - references: - - dataset: stripe_connector_example - field: invoice.id - direction: from - - name: invoice_item - requests: - read: - method: GET - path: /v1/invoiceitems - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - delete: - method: DELETE - ignore_errors: true # Can't delete an invoice item that is attached to an invoice that is no longer editable - path: /v1/invoiceitems/ - param_values: - - name: invoice_item_id - references: - - dataset: stripe_connector_example - field: invoice_item.id - direction: from - - name: subscription - requests: - read: - method: GET - path: /v1/subscriptions - query_params: - - name: customer - value: - - name: limit - value: - param_values: - - name: customer_id - references: - - dataset: stripe_connector_example - field: customer.id - direction: from - - name: limit - connector_param: items_per_page - data_path: data - pagination: - strategy: cursor - configuration: - cursor_param: starting_after - field: id - delete: - method: DELETE - path: /v1/subscriptions/ - param_values: - - name: subscription_id - references: - - dataset: stripe_connector_example - field: subscription.id - direction: from \ No newline at end of file + - name: customer + requests: + read: + method: GET + path: /v1/customers + query_params: + - name: email + value: + param_values: + - name: email + identity: email + data_path: data + update: + method: POST + path: /v1/customers/ + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + body: | + { + + } + - name: charge + requests: + read: + method: GET + path: /v1/charges + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: dispute + requests: + read: + method: GET + path: /v1/disputes + query_params: + - name: charge + value: + - name: payment_intent + value: + - name: limit + value: + param_values: + - name: charge_id + references: + - dataset: stripe_connector_example + field: charge.id + direction: from + - name: payment_intent_id + references: + - dataset: stripe_connector_example + field: payment_intent.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: payment_intent + requests: + read: + method: GET + path: /v1/payment_intents + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: payment_method + requests: + read: + method: GET + path: /v1/customers//payment_methods + query_params: + - name: type + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: type + connector_param: payment_types + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + update: + method: POST + path: /v1/payment_methods/ + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + param_values: + - name: payment_method_id + references: + - dataset: stripe_connector_example + field: payment_method.id + direction: from + body: | + { + + } + - name: bank_account + requests: + read: + method: GET + path: /v1/customers//sources + query_params: + - name: object + value: bank_account + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + update: + method: POST + path: /v1/customers//sources/ + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: bank_account.customer + direction: from + - name: bank_account_id + references: + - dataset: stripe_connector_example + field: bank_account.id + direction: from + body: | + { + + } + - name: card + requests: + read: + method: GET + path: /v1/customers//sources + query_params: + - name: object + value: card + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + update: + method: POST + path: /v1/customers//sources/ + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: card.customer + direction: from + - name: card_id + references: + - dataset: stripe_connector_example + field: card.id + direction: from + body: | + { + + } + - name: credit_note + requests: + read: + method: GET + path: /v1/credit_notes + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: customer_balance_transaction + requests: + read: + method: GET + path: /v1/customers//balance_transactions + query_params: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + - name: tax_id + requests: + read: + method: GET + path: /v1/customers//tax_ids + query_params: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + delete: + method: DELETE + path: /v1/customers//tax_ids/ + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: tax_id.customer + direction: from + - name: tax_id + references: + - dataset: stripe_connector_example + field: tax_id.id + direction: from + - name: invoice + requests: + read: + method: GET + path: /v1/invoices + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + delete: + method: DELETE + ignore_errors: true # You can only delete draft invoices. You can't delete invoices created by subscriptions. + path: /v1/invoices/ + param_values: + - name: invoice_id + references: + - dataset: stripe_connector_example + field: invoice.id + direction: from + - name: invoice_item + requests: + read: + method: GET + path: /v1/invoiceitems + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + delete: + method: DELETE + ignore_errors: true # Can't delete an invoice item that is attached to an invoice that is no longer editable + path: /v1/invoiceitems/ + param_values: + - name: invoice_item_id + references: + - dataset: stripe_connector_example + field: invoice_item.id + direction: from + - name: subscription + requests: + read: + method: GET + path: /v1/subscriptions + query_params: + - name: customer + value: + - name: limit + value: + param_values: + - name: customer_id + references: + - dataset: stripe_connector_example + field: customer.id + direction: from + - name: limit + connector_param: page_size + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: starting_after + field: id + delete: + method: DELETE + path: /v1/subscriptions/ + param_values: + - name: subscription_id + references: + - dataset: stripe_connector_example + field: subscription.id + direction: from diff --git a/docs/fidesops/docs/saas_connectors/saas_config.md b/docs/fidesops/docs/saas_connectors/saas_config.md index 3d0a104e4..cf91f6c7f 100644 --- a/docs/fidesops/docs/saas_connectors/saas_config.md +++ b/docs/fidesops/docs/saas_connectors/saas_config.md @@ -125,13 +125,16 @@ And the following complex fields which we will cover in detail below: - `data_protection_request` #### Connector params -The `connector_params` field is used to describe a list of settings which a user must configure as part of the setup. This section should just include the name of the parameter but not the actual value. These are added as part of the ConnectionConfig [secrets](/docs/fidesops/docs/guides/database_connectors.md#set-the-connectionconfigs-secrets). +The `connector_params` field is used to describe a list of settings which a user must configure as part of the setup. A `default_value` can also be used to include values such as a standard base domain for an API or a recommended page size for pagination. Make sure to not include confidential values such as passwords or API keys, these values are added as part of the ConnectionConfig [secrets](/docs/fidesops/docs/guides/database_connectors.md#set-the-connectionconfigs-secrets). When configuring a connector's secrets for the first time, the default values will be used if a value is not provided. ```yaml connector_params: - - name: host + - name: domain + default_value: api.stripe.com - name: username - name: password + - name: page_size + default_value: 100 ``` #### Client config diff --git a/saas_config.toml b/saas_config.toml index 4185c3f19..0771f0271 100644 --- a/saas_config.toml +++ b/saas_config.toml @@ -3,7 +3,7 @@ domain = "domain" username = "username" api_key = "api_key" api_version = "2.0" -page_limit = "10" +page_size = "10" [mailchimp] domain = "" @@ -20,7 +20,7 @@ identity_email = "" host = "" api_key = "" payment_types = "" -page_limit = "" +page_size = "" identity_email = "" [sentry] diff --git a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py index d234e5f02..837267f1a 100644 --- a/src/fidesops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fidesops/api/v1/endpoints/saas_config_endpoints.py @@ -144,7 +144,7 @@ def patch_saas_config( logger.info( f"Updating SaaS config '{saas_config.fides_key}' on connection config '{connection_config.key}'" ) - connection_config.update(db, data={"saas_config": saas_config.dict()}) + connection_config.update_saas_config(db, saas_config=saas_config) return connection_config.saas_config diff --git a/src/fidesops/models/connectionconfig.py b/src/fidesops/models/connectionconfig.py index b97155a19..8cef10d81 100644 --- a/src/fidesops/models/connectionconfig.py +++ b/src/fidesops/models/connectionconfig.py @@ -91,6 +91,25 @@ def get_saas_config(self) -> Optional[SaaSConfig]: """Returns a SaaSConfig object from a yaml config""" return SaaSConfig(**self.saas_config) if self.saas_config else None + def update_saas_config( + self, + db: Session, + saas_config: SaaSConfig, + ) -> None: + """ + Updates the SaaS config and initializes any empty secrets with + connector param default values if available (will not override any existing secrets) + """ + default_secrets = { + connector_param.name: connector_param.default_value + for connector_param in saas_config.connector_params + if connector_param.default_value + } + updated_secrets = {**default_secrets, **(self.secrets or {})} + self.secrets = updated_secrets + self.saas_config = saas_config.dict() + self.save(db) + def update_test_status( self, test_status: ConnectionTestStatus, db: Session ) -> None: diff --git a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py index cc157e977..a32c02ab0 100644 --- a/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/schemas/connection_configuration/connection_secrets_saas.py @@ -19,7 +19,9 @@ def required_components_supplied( # type: ignore ) -> Dict[str, Any]: """Validate that the minimum required components have been supplied.""" - required_components = cls.__fields__.keys() + required_components = [ + name for name, attributes in cls.__fields__.items() if attributes.required + ] min_fields_present = all( [values.get(component) for component in required_components] ) @@ -49,10 +51,13 @@ def __init__(self, saas_config: SaaSConfig): # Pydantic uses the shorthand of (str, ...) to denote a required field of type str def get_saas_schema(self) -> Type[SaaSSchema]: """Returns the schema for the current configuration""" - field_definitions: Dict[str, Any] = { - connector_param.name: (str, ...) - for connector_param in self.saas_config.connector_params - } + field_definitions: Dict[str, Any] = {} + for connector_param in self.saas_config.connector_params: + field_definitions[connector_param.name] = ( + connector_param.default_value + if connector_param.default_value + else (str, ...) + ) return create_model( f"{self.saas_config.fides_key}_schema", **field_definitions, diff --git a/src/fidesops/schemas/saas/saas_config.py b/src/fidesops/schemas/saas/saas_config.py index 06cbd8585..9911b7b52 100644 --- a/src/fidesops/schemas/saas/saas_config.py +++ b/src/fidesops/schemas/saas/saas_config.py @@ -169,6 +169,7 @@ class ConnectorParam(BaseModel): """Used to define the required parameters for the connector (user-provided and constants)""" name: str + default_value: Optional[str] description: Optional[str] diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index ac16c3bf9..66aec058f 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -22,7 +22,8 @@ @pytest.fixture(scope="function") def sentry_secrets(): return { - "host": pydash.get(saas_config, "sentry.host") or os.environ.get("SENTRY_HOST"), + "domain": pydash.get(saas_config, "sentry.domain") + or os.environ.get("SENTRY_DOMAIN"), "access_token": pydash.get(saas_config, "sentry.access_token") or os.environ.get("SENTRY_ACCESS_TOKEN"), "erasure_access_token": pydash.get(saas_config, "sentry.erasure_access_token") diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 900a22ca5..6640f1a3d 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -24,13 +24,14 @@ @pytest.fixture(scope="function") def stripe_secrets(): return { - "host": pydash.get(saas_config, "stripe.host") or os.environ.get("STRIPE_HOST"), + "domain": pydash.get(saas_config, "stripe.domain") + or os.environ.get("STRIPE_DOMAIN"), "api_key": pydash.get(saas_config, "stripe.api_key") or os.environ.get("STRIPE_API_KEY"), "payment_types": pydash.get(saas_config, "stripe.payment_types") or os.environ.get("STRIPE_PAYMENT_TYPES"), - "items_per_page": pydash.get(saas_config, "stripe.items_per_page") - or os.environ.get("STRIPE_ITEMS_PER_PAGE"), + "page_size": pydash.get(saas_config, "stripe.page_size") + or os.environ.get("STRIPE_PAGE_SIZE"), } diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 4cfd42387..aba6e4244 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -34,7 +34,7 @@ def saas_example_secrets(): "username": pydash.get(saas_config, "saas_example.username"), "api_key": pydash.get(saas_config, "saas_example.api_key"), "api_version": pydash.get(saas_config, "saas_example.api_version"), - "page_limit": pydash.get(saas_config, "saas_example.page_limit"), + "page_size": pydash.get(saas_config, "saas_example.page_size"), } @@ -202,7 +202,7 @@ def oauth2_connection_config(db: Session, oauth2_configuration) -> Generator: "name": "OAuth2 Connector", "description": "Generic OAuth2 connector for testing", "version": "0.0.1", - "connector_params": [{"name": item} for item in secrets.values()], + "connector_params": [{"name": item} for item in secrets.keys()], "client_config": { "protocol": "https", "host": secrets["domain"], diff --git a/tests/models/test_connectionconfig.py b/tests/models/test_connectionconfig.py index de7658962..163a5d10b 100644 --- a/tests/models/test_connectionconfig.py +++ b/tests/models/test_connectionconfig.py @@ -7,6 +7,8 @@ ConnectionConfig, ConnectionType, ) +from fidesops.schemas.saas import saas_config +from fidesops.schemas.saas.saas_config import SaaSConfig from fidesops.util.text import to_snake_case @@ -109,3 +111,32 @@ def test_setting_disabled_at(self, db, connection_config): connection_config.save(db) assert connection_config.disabled_at is not None assert connection_config.disabled_at > original_disabled_time + + def test_default_value_saas_config( + self, db, saas_example_config, saas_example_secrets + ): + connection_config: ConnectionConfig = ConnectionConfig.create( + db=db, + data={ + "key": "not_configured", + "name": "not_configured", + "connection_type": ConnectionType.saas, + "access": AccessLevel.read, + }, + ) + saas_config = SaaSConfig(**saas_example_config) + assert connection_config.secrets is None + + # verify that setting the SaaS config for the first time populates + # the secrets with default values + connection_config.update_saas_config(db, saas_config=saas_config) + assert connection_config.secrets == {"domain": "localhost"} + + # verify that a user-defined secret overrides the default value + connection_config.update(db, data={"secrets": saas_example_secrets}) + assert connection_config.secrets["domain"] == saas_example_secrets["domain"] + + # verify that updating the SaaS config after configuring the secrets + # does not override any user-defined values + connection_config.update_saas_config(db, saas_config=saas_config) + assert connection_config.secrets["domain"] == saas_example_secrets["domain"] diff --git a/tests/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/schemas/connection_configuration/test_connection_secrets_saas.py index 08e2d787b..adf188bf5 100644 --- a/tests/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/schemas/connection_configuration/test_connection_secrets_saas.py @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from pydantic import ValidationError @@ -11,7 +13,7 @@ @pytest.mark.unit_saas class TestSaaSConnectionSecrets: @pytest.fixture(scope="function") - def saas_config(self, saas_example_config) -> SaaSConfig: + def saas_config(self, saas_example_config: Dict[str, Any]) -> SaaSConfig: return SaaSConfig(**saas_example_config) def test_get_saas_schema(self, saas_config): @@ -23,23 +25,31 @@ def test_get_saas_schema(self, saas_config): assert schema.__name__ == f"{saas_config.fides_key}_schema" assert issubclass(schema.__base__, SaaSSchema) - def test_validation(self, saas_config, saas_example_secrets): + def test_validation( + self, saas_config: SaaSConfig, saas_example_secrets: Dict[str, Any] + ): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = saas_example_secrets schema.parse_obj(config) - def test_missing_fields(self, saas_config): + def test_missing_fields(self, saas_config: SaaSConfig): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = {"domain": "domain", "username": "username"} with pytest.raises(ValidationError) as exc: schema.parse_obj(config) + required_fields = [ + connector_param.name + for connector_param in saas_config.connector_params + if not connector_param.default_value + ] assert ( f"{saas_config.fides_key}_schema must be supplied all of: " - f"[{', '.join([connector_param.name for connector_param in saas_config.connector_params])}]." - in str(exc.value) + f"[{', '.join(required_fields)}]." in str(exc.value) ) - def test_extra_fields(self, saas_config, saas_example_secrets): + def test_extra_fields( + self, saas_config: SaaSConfig, saas_example_secrets: Dict[str, Any] + ): schema = SaaSSchemaFactory(saas_config).get_saas_schema() config = { **saas_example_secrets, @@ -48,3 +58,17 @@ def test_extra_fields(self, saas_config, saas_example_secrets): with pytest.raises(ValidationError) as exc: schema.parse_obj(config) assert exc.value.errors()[0]["msg"] == "extra fields not permitted" + + def test_default_value_fields( + self, saas_config: SaaSConfig, saas_example_secrets: Dict[str, Any] + ): + schema = SaaSSchemaFactory(saas_config).get_saas_schema() + domain_param = next( + connector_param + for connector_param in saas_config.connector_params + if connector_param.name == "domain" + ) + assert domain_param.default_value + del saas_example_secrets["domain"] + config = saas_example_secrets + schema.parse_obj(config) diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index d41d91278..f28f3a0d6 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -668,7 +668,7 @@ def test_generate_query( config = SaaSQueryConfig( payment_methods, endpoints, - {"api_version": "2.0", "page_limit": 10, "api_key": "letmein"}, + {"api_version": "2.0", "page_size": 10, "api_key": "letmein"}, ) prepared_request = config.generate_query( {"email": ["customer-1@example.com"]}, policy @@ -688,7 +688,7 @@ def test_generate_query( # query and path params with connector param references config = SaaSQueryConfig( - payment_methods, endpoints, {"api_version": "2.0", "page_limit": 10} + payment_methods, endpoints, {"api_version": "2.0", "page_size": 10} ) prepared_request = config.generate_query( {"email": ["customer-1@example.com"]}, policy @@ -730,10 +730,9 @@ def test_generate_update_stmt( assert prepared_request.path == "/3.0/lists/abc/members/123" assert prepared_request.headers == {"Content-Type": "application/json"} assert prepared_request.query_params == {} - assert prepared_request.body == json.dumps( - { - "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, - } + assert ( + prepared_request.body + == '{\n "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n' ) def test_generate_update_stmt_custom_http_method( @@ -769,10 +768,9 @@ def test_generate_update_stmt_custom_http_method( assert prepared_request.path == "/3.0/lists/abc/members/123" assert prepared_request.headers == {"Content-Type": "application/json"} assert prepared_request.query_params == {} - assert prepared_request.body == json.dumps( - { - "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, - } + assert ( + prepared_request.body + == '{\n "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n' ) def test_generate_update_stmt_with_request_body( @@ -849,7 +847,7 @@ def test_generate_update_stmt_with_request_body( assert prepared_request.path == "/2.0/payment_methods" assert prepared_request.headers == {"Content-Type": "application/json"} assert prepared_request.query_params == {} - assert prepared_request.body == json.dumps({"customer_name": "MASKED"}) + assert prepared_request.body == '{\n "customer_name": "MASKED"\n}\n' def test_generate_update_stmt_with_url_encoded_body( self, diff --git a/tests/util/test_saas_util.py b/tests/util/test_saas_util.py index 8dcad230f..3c046a7e2 100644 --- a/tests/util/test_saas_util.py +++ b/tests/util/test_saas_util.py @@ -263,7 +263,7 @@ def test_single_placeholder_with_string_value(self): assert assign_placeholders("", {"access_key": "123"}) == "123" def test_single_placeholder_with_int_value(self): - assert assign_placeholders("", {"page_limit": 10}) == "10" + assert assign_placeholders("", {"page_size": 10}) == "10" def test_multiple_string_placeholders(self): assert (