Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move all fields test to new framework #239

Merged
merged 12 commits into from
Dec 18, 2023
Merged

Move all fields test to new framework #239

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
60aa211
Move all fields test to new framework
bhuvana-talend Nov 1, 2023
bd451ee
As per PR review, removed the overrridden methods and moved some comm…
bhuvana-talend Nov 20, 2023
16279de
Moved some fields to tap-tester base case
bhuvana-talend Nov 27, 2023
8e118c9
Implemented PR Review comments
bhuvana-talend Nov 28, 2023
b279239
Improvements in new framework all fields test to incorporate the logi…
bhuvana-talend Dec 4, 2023
cb62f07
Removed not-used method and constants
bhuvana-talend Dec 4, 2023
1026c65
Removed the extra fields for deals as it is covered in bad prefix
bhuvana-talend Dec 5, 2023
b9ef05e
Fixed removed_bad_keys to add to bad keys only if it is not in the ot…
bhuvana-talend Dec 5, 2023
fcd97dc
Merge branch 'master' into tdl-24438-2
bhuvana-talend Dec 5, 2023
de4bdb1
Merge branch 'master' into tdl-24438-2
bhuvana-talend Dec 11, 2023
25ba332
Merge branch 'master' into tdl-24438-2
bhuvana-talend Dec 13, 2023
e57af17
Removed unneeded line
bhuvana-talend Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
283 changes: 283 additions & 0 deletions tests/base_hubspot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,283 @@
import os
import unittest
from datetime import datetime as dt
from datetime import timedelta

import tap_tester.menagerie as menagerie
import tap_tester.connections as connections
import tap_tester.runner as runner
from tap_tester.base_suite_tests.base_case import BaseCase
from tap_tester import LOGGER


class HubspotBaseCase(BaseCase):

# set the default start date which can be overridden in the tests.
start_date = BaseCase.timedelta_formatted(dt.utcnow(), delta=timedelta(days=-1))


FIELDS_ADDED_BY_TAP = {
# In 'contacts' streams 'versionTimeStamp' is not available in response of the second call.
# In the 1st call, Tap retrieves records of all contacts and from those records, it collects vids(id of contact).
# These records contain the versionTimestamp field.
# In the 2nd call, vids collected from the 1st call will be used to retrieve the whole contact record.
# Here, the records collected for detailed contact information do not contain the versionTimestamp field.
# So, we add the versionTimestamp field(fetched from 1st call records) explicitly in the record of 2nd call.
"contacts": { "versionTimestamp" }
}

KNOWN_EXTRA_FIELDS = {
'deals': {
# BUG_TDL-14993 | https://jira.talendforge.org/browse/TDL-14993
# Has an value of object with key 'value' and value 'Null'
'property_hs_date_entered_1258834',
'property_hs_time_in_example_stage1660743867503491_315775040'
},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is only used for the all_fields test. This is duplicative of the bad keys below and isn't necessary as there are just specific examples. We should move the bug info down to the method for removing the bad keys.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah.. This just got carried over from the old test.
So, we added these extra fields and removed the bad prefixes later.
Not needed here, will remove it.

}

KNOWN_MISSING_FIELDS = {
'contacts':{ # BUG https://jira.talendforge.org/browse/TDL-16016
'property_hs_latest_source',
'property_hs_latest_source_data_1',
'property_hs_latest_source_data_2',
'property_hs_latest_source_timestamp',
'property_hs_timezone',
'property_hs_v2_cumulative_time_in_lead',
'property_hs_v2_cumulative_time_in_opportunity',
'property_hs_v2_cumulative_time_in_subscriber',
'property_hs_v2_date_entered_customer',
'property_hs_v2_date_entered_lead',
'property_hs_v2_date_entered_opportunity',
'property_hs_v2_date_entered_subscriber',
'property_hs_v2_date_exited_lead',
'property_hs_v2_date_exited_opportunity',
'property_hs_v2_date_exited_subscriber',
'property_hs_v2_latest_time_in_lead',
'property_hs_v2_latest_time_in_opportunity',
'property_hs_v2_latest_time_in_subscriber',
},
'contact_lists': { # BUG https://jira.talendforge.org/browse/TDL-14996
'authorId',
'teamIds',
'internal',
'ilsFilterBranch',
'limitExempt',
},
'email_events': { # BUG https://jira.talendforge.org/browse/TDL-14997
'portalSubscriptionStatus',
'attempt',
'source',
'subscriptions',
'sourceId',
'replyTo',
'suppressedMessage',
'bcc',
'suppressedReason',
'cc',
},
'engagements': { # BUG https://jira.talendforge.org/browse/TDL-14997
'scheduledTasks',
},
'workflows': { # BUG https://jira.talendforge.org/browse/TDL-14998
'migrationStatus',
'updateSource',
'description',
'originalAuthorUserId',
'lastUpdatedByUserId',
'creationSource',
'portalId',
'contactCounts',
},
'owners': { # BUG https://jira.talendforge.org/browse/TDL-15000
'activeSalesforceId'
},
'forms': { # BUG https://jira.talendforge.org/browse/TDL-15001
'alwaysCreateNewCompany',
'themeColor',
'publishAt',
'editVersion',
'embedVersion',
'themeName',
'style',
'thankYouMessageJson',
'createMarketableContact',
'kickbackEmailWorkflowId',
'businessUnitId',
'portableKey',
'parentId',
'kickbackEmailsJson',
'unpublishAt',
'internalUpdatedAt',
'multivariateTest',
'publishedAt',
'customUid',
'isPublished',
'paymentSessionTemplateIds',
'selectedExternalOptions',
},
'companies': { # BUG https://jira.talendforge.org/browse/TDL-15003
'mergeAudits',
'stateChanges',
'isDeleted',
'additionalDomains',
'property_hs_analytics_latest_source',
'property_hs_analytics_latest_source_data_2',
'property_hs_analytics_latest_source_data_1',
'property_hs_analytics_latest_source_timestamp',
},
'campaigns': { # BUG https://jira.talendforge.org/browse/TDL-15003
'lastProcessingStateChangeAt',
'lastProcessingFinishedAt',
'processingState',
'lastProcessingStartedAt',
},
'deals': { # BUG https://jira.talendforge.org/browse/TDL-14999
'imports',
'property_hs_num_associated_deal_splits',
'property_hs_is_deal_split',
'stateChanges',
'property_hs_num_associated_active_deal_registrations',
'property_hs_num_associated_deal_registrations',
'property_hs_analytics_latest_source',
'property_hs_analytics_latest_source_timestamp_contact',
'property_hs_analytics_latest_source_data_1_contact',
'property_hs_analytics_latest_source_timestamp',
'property_hs_analytics_latest_source_data_1',
'property_hs_analytics_latest_source_contact',
'property_hs_analytics_latest_source_company',
'property_hs_analytics_latest_source_data_1_company',
'property_hs_analytics_latest_source_data_2_company',
'property_hs_analytics_latest_source_data_2',
'property_hs_analytics_latest_source_data_2_contact',
},
'subscription_changes':{
'normalizedEmailId'
}
}

def setUp(self):
missing_envs = [x for x in [
'TAP_HUBSPOT_REDIRECT_URI',
'TAP_HUBSPOT_CLIENT_ID',
'TAP_HUBSPOT_CLIENT_SECRET',
'TAP_HUBSPOT_REFRESH_TOKEN'
] if os.getenv(x) is None]
if missing_envs:
raise Exception("Missing environment variables: {}".format(missing_envs))

@staticmethod
def get_type():
return "platform.hubspot"

@staticmethod
def tap_name():
return "tap-hubspot"

def get_properties(self):
return {'start_date': self.start_date }

def get_credentials(self):
return {'refresh_token': os.getenv('TAP_HUBSPOT_REFRESH_TOKEN'),
'client_secret': os.getenv('TAP_HUBSPOT_CLIENT_SECRET'),
'redirect_uri': os.getenv('TAP_HUBSPOT_REDIRECT_URI'),
'client_id': os.getenv('TAP_HUBSPOT_CLIENT_ID')}

def expected_check_streams(self):
return set(self.expected_metadata().keys())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the same as this method in tap-tester with a different name. It would be best to remove this and rename the calls to it to the method in tap-tester. At the minimum if we want to keep this name for some reason, just can make it a wrapper and call the underlying tap-tester method in it as follows:

Suggested change
def expected_check_streams(self):
return set(self.expected_metadata().keys())
@classmethod
def expected_check_streams(cls):
return cls.expected_stream_names()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method is used only start_date test. So, I removed this. I think we can change start date test when we move that to the new framework.


@classmethod
def expected_metadata(cls): # DOCS_BUG https://stitchdata.atlassian.net/browse/DOC-1523)
"""The expected streams and metadata about the streams"""

return {
"campaigns": {
BaseCase.PRIMARY_KEYS: {"id"},
BaseCase.REPLICATION_METHOD: BaseCase.FULL_TABLE,
BaseCase.OBEYS_START_DATE: False
},
"companies": {
BaseCase.PRIMARY_KEYS: {"companyId"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"property_hs_lastmodifieddate"},
BaseCase.API_LIMIT: 250,
BaseCase.OBEYS_START_DATE: True
},
"contact_lists": {
BaseCase.PRIMARY_KEYS: {"listId"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"updatedAt"},
BaseCase.API_LIMIT: 250,
BaseCase.OBEYS_START_DATE: True
},
"contacts": {
BaseCase.PRIMARY_KEYS: {"vid"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"versionTimestamp"},
BaseCase.API_LIMIT: 100,
BaseCase.OBEYS_START_DATE: True
},
"contacts_by_company": {
BaseCase.PRIMARY_KEYS: {"company-id", "contact-id"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.API_LIMIT: 100,
BaseCase.OBEYS_START_DATE: True,
HubspotBaseCase.PARENT_STREAM: 'companies'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure how common parent stream is, If this is unique to hubspot then doing it like this is appropriate. If we think there are more cases where this would exist we should also move this concept to the tap-tester framework.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see PARENT_STREAM defined. Can you show me where this is? Did it move to BaseCase? If so, we should specify that instead of HubspotBaseCase.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

missed it.. It is moved to tap-tester base case. I changed to use BaseCase.PARENT_STREAM

},
"deal_pipelines": {
BaseCase.PRIMARY_KEYS: {"pipelineId"},
BaseCase.REPLICATION_METHOD: BaseCase.FULL_TABLE,
BaseCase.OBEYS_START_DATE: False,
},
"deals": {
BaseCase.PRIMARY_KEYS: {"dealId"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"property_hs_lastmodifieddate"},
BaseCase.OBEYS_START_DATE: True
},
"email_events": {
BaseCase.PRIMARY_KEYS: {"id"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"startTimestamp"},
BaseCase.API_LIMIT: 1000,
BaseCase.OBEYS_START_DATE: True
},
"engagements": {
BaseCase.PRIMARY_KEYS: {"engagement_id"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"lastUpdated"},
BaseCase.API_LIMIT: 250,
BaseCase.OBEYS_START_DATE: True
},
"forms": {
BaseCase.PRIMARY_KEYS: {"guid"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"updatedAt"},
BaseCase.OBEYS_START_DATE: True
},
"owners": {
BaseCase.PRIMARY_KEYS: {"ownerId"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"updatedAt"},
BaseCase.OBEYS_START_DATE: True # TODO is this a BUG?
},
"subscription_changes": {
BaseCase.PRIMARY_KEYS: {"timestamp", "portalId", "recipient"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"startTimestamp"},
BaseCase.API_LIMIT: 1000,
BaseCase.OBEYS_START_DATE: True
},
"workflows": {
BaseCase.PRIMARY_KEYS: {"id"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"updatedAt"},
BaseCase.OBEYS_START_DATE: True
},
"tickets": {
BaseCase.PRIMARY_KEYS: {"id"},
BaseCase.REPLICATION_METHOD: BaseCase.INCREMENTAL,
BaseCase.REPLICATION_KEYS: {"updatedAt"},
BaseCase.API_LIMIT: 100,
BaseCase.OBEYS_START_DATE: True
}
}
Loading