Skip to content

Commit

Permalink
Added support OAuth2.0
Browse files Browse the repository at this point in the history
Signed-off-by: Sergey Chvalyuk <[email protected]>
  • Loading branch information
grubberr committed Mar 25, 2022
1 parent 86d9090 commit 1359d44
Show file tree
Hide file tree
Showing 7 changed files with 120 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
- name: Amazon Ads
sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
dockerRepository: airbyte/source-amazon-ads
dockerImageTag: 0.1.3
dockerImageTag: 0.1.4
documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
icon: amazonads.svg
sourceType: api
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.3
LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.name=airbyte/source-amazon-ads
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator

from .schemas import Profile
from .spec import AmazonAdsConfig
from .spec import AmazonAdsConfig, advanced_auth
from .streams import (
Profiles,
SponsoredBrandsAdGroups,
Expand Down Expand Up @@ -95,6 +95,7 @@ def spec(self, *args) -> ConnectorSpecification:
return ConnectorSpecification(
documentationUrl="https://docs.airbyte.io/integrations/sources/amazon-ads",
connectionSpecification=AmazonAdsConfig.schema(),
advanced_auth=advanced_auth,
)

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from typing import List

from airbyte_cdk.models import AdvancedAuth, AuthFlowType, OAuthConfigSpecification
from pydantic import BaseModel, Field
from source_amazon_ads.constants import AmazonAdsRegion

Expand All @@ -12,6 +13,8 @@ class AmazonAdsConfig(BaseModel):
class Config:
title = "Amazon Ads Spec"

auth_type: str = Field(default="oauth2.0", const=True, order=0, enum=["oauth2.0"])

client_id: str = Field(
name="Client ID",
description=(
Expand Down Expand Up @@ -79,3 +82,30 @@ def schema(cls, **kvargs):
schema["properties"]["region"].pop("allOf", None)
schema["properties"]["region"].pop("$ref", None)
return schema


advanced_auth = AdvancedAuth(
auth_flow_type=AuthFlowType.oauth2_0,
predicate_key=["auth_type"],
predicate_value="oauth2.0",
oauth_config_specification=OAuthConfigSpecification(
complete_oauth_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["refresh_token"]}},
},
complete_oauth_server_input_specification={
"type": "object",
"additionalProperties": False,
"properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}},
},
complete_oauth_server_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {
"client_id": {"type": "string", "path_in_connector_config": ["client_id"]},
"client_secret": {"type": "string", "path_in_connector_config": ["client_secret"]},
},
},
),
)
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class OAuthImplementationFactory {

public OAuthImplementationFactory(final ConfigRepository configRepository, final HttpClient httpClient) {
OAUTH_FLOW_MAPPING = ImmutableMap.<String, OAuthFlowImplementation>builder()
.put("airbyte/source-amazon-ads", new AmazonAdsOAuthFlow(configRepository, httpClient))
.put("airbyte/source-asana", new AsanaOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-pages", new FacebookPagesOAuthFlow(configRepository, httpClient))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.oauth.flows;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.oauth.BaseOAuth2Flow;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.apache.http.client.utils.URIBuilder;

public class AmazonAdsOAuthFlow extends BaseOAuth2Flow {

private static final String AUTHORIZE_URL = "https://www.amazon.com/ap/oa";
private static final String ACCESS_TOKEN_URL = "https://api.amazon.com/auth/o2/token";

public AmazonAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
super(configRepository, httpClient);
}

public AmazonAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
super(configRepository, httpClient, stateSupplier);
}

/**
* Depending on the OAuth flow implementation, the URL to grant user's consent may differ,
* especially in the query parameters to be provided. This function should generate such consent URL
* accordingly.
*
* @param definitionId The configured definition ID of this client
* @param clientId The configured client ID
* @param redirectUrl the redirect URL
*/
@Override
protected String formatConsentUrl(final UUID definitionId,
final String clientId,
final String redirectUrl,
final JsonNode inputOAuthConfiguration)
throws IOException {
try {
return new URIBuilder(AUTHORIZE_URL)
.addParameter("client_id", clientId)
.addParameter("scope", "profile")
.addParameter("response_type", "code")
.addParameter("redirect_uri", redirectUrl)
.addParameter("state", getState())
.build().toString();
} catch (final URISyntaxException e) {
throw new IOException("Failed to format Consent URL for OAuth flow", e);
}
}

@Override
protected Map<String, String> getAccessTokenQueryParameters(final String clientId,
final String clientSecret,
final String authCode,
final String redirectUrl) {
return ImmutableMap.<String, String>builder()
// required
.put("client_id", clientId)
.put("redirect_uri", redirectUrl)
.put("client_secret", clientSecret)
.put("code", authCode)
.put("grant_type", "authorization_code")
.build();
}

/**
* Returns the URL where to retrieve the access token from.
*
*/
@Override
protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
return ACCESS_TOKEN_URL;
}

}
1 change: 1 addition & 0 deletions docs/integrations/sources/amazon-ads.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ Start date used for generating reports starting from the specified start date. S

| Version | Date | Pull Request | Subject |
| :--- | :--- | :--- | :--- |
| `0.1.4` | 2022-03-25 | [\#](https://github.com/airbytehq/airbyte/pull/) | `Added support OAuth2.0` |
| `0.1.3` | 2021-12-28 | [\#8388](https://github.com/airbytehq/airbyte/pull/8388) | `Add retry if recoverable error occured for reporting stream processing` |
| `0.1.2` | 2021-10-01 | [\#6367](https://github.com/airbytehq/airbyte/pull/6461) | `Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback` |
| `0.1.1` | 2021-09-22 | [\#6367](https://github.com/airbytehq/airbyte/pull/6367) | `Add seller and vendor filters to profiles stream` |
Expand Down

1 comment on commit 1359d44

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SonarQube Report

SonarQube report for Airbyte Connectors Source Amazon Ads(#11430)

Measures

Name Value Name Value Name Value
Lines of Code 1499 Reliability Rating A Bugs 0
Coverage 98.9 Duplicated Lines (%) 0.0 Lines to Cover 643
Vulnerabilities 0 Security Rating A Quality Gate Status OK
Duplicated Blocks 0 Code Smells 62 Blocker Issues 0
Critical Issues 1 Major Issues 12 Minor Issues 49

Detected Issues

Rule File Description Message
python:S1192 (CRITICAL) source_amazon_ads/spec.py:16 String literals should not be duplicated Define a constant instead of duplicating this literal "oauth2.0" 3 times.
python:mypy_no_any_return (MINOR) report_streams/report_streams.py:376 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "str" . Code line: return profile_time.strftime(ReportStream.REPORT_DATE_FORMAT)
python:mypy_arg_type (MINOR) source_amazon_ads/source.py:72 Check argument types in calls Argument 1 to "_choose_profiles" of "SourceAmazonAds" has incompatible type "Mapping[str, Any]"; expected "AmazonAdsConfig" . Code line: ... stream_args["profiles"] = self._choose_profiles(config, profiles_list...
python:S1066 (MAJOR) schemas/common.py:32 Collapsible "if" statements should be merged Merge this if statement with the enclosing one.
python:mypy_call_overload (MINOR) schemas/common.py:45 Check that an overload variant matches arguments No overload variant of "create_model" matches argument types "str", "Dict[str, Tuple[Type[str], None]]", "Type[CatalogModel]" . Code line: metrics_obj_model = create_model("MetricObjModel", **{f: (str,...
python:mypy_return_value (MINOR) schemas/common.py:46 Check that return value is compatible with signature Incompatible return value type (got "Type[MetricsReport]", expected "CatalogModel") . Code line: return create_model("MetricsModel", metric=(metrics_obj_model,...
python:S5890 (MAJOR) schemas/profile.py:14 Values assigned to variables should match their type annotations Assign to "name" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/profile.py:14 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: name: str = None
python:S5890 (MAJOR) schemas/profile.py:15 Values assigned to variables should match their type annotations Assign to "subType" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/profile.py:15 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: subType: str = None
python:S5890 (MAJOR) schemas/profile.py:16 Values assigned to variables should match their type annotations Assign to "validPaymentMethod" a value of type "bool" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/profile.py:16 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "bool") . Code line: validPaymentMethod: bool = None
python:S5890 (MAJOR) schemas/profile.py:21 Values assigned to variables should match their type annotations Assign to "countryCode" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/profile.py:21 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: countryCode: str = None
python:mypy_assignment (MINOR) schemas/profile.py:22 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: currencyCode: str = None
python:S5890 (MAJOR) schemas/profile.py:22 Values assigned to variables should match their type annotations Assign to "currencyCode" a value of type "str" instead of "NoneType" or update its type hint.
python:S5890 (MAJOR) schemas/profile.py:23 Values assigned to variables should match their type annotations Assign to "dailyBudget" a value of type "Decimal" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/profile.py:23 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "Decimal") . Code line: dailyBudget: Decimal = None
python:S5890 (MAJOR) schemas/sponsored_brands.py:23 Values assigned to variables should match their type annotations Assign to "bidOptimization" a value of type "bool" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/sponsored_brands.py:23 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "bool") . Code line: bidOptimization: bool = None
python:S5890 (MAJOR) schemas/sponsored_brands.py:24 Values assigned to variables should match their type annotations Assign to "bidMultiplier" a value of type "Decimal" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/sponsored_brands.py:24 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "Decimal") . Code line: bidMultiplier: Decimal = None
python:S5890 (MAJOR) schemas/sponsored_display.py:16 Values assigned to variables should match their type annotations Assign to "endDate" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/sponsored_display.py:16 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: endDate: str = None
python:S5890 (MAJOR) schemas/sponsored_display.py:19 Values assigned to variables should match their type annotations Assign to "portfolioId" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/sponsored_display.py:19 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: portfolioId: str = None
python:S5890 (MAJOR) schemas/sponsored_products.py:31 Values assigned to variables should match their type annotations Assign to "endDate" a value of type "str" instead of "NoneType" or update its type hint.
python:mypy_assignment (MINOR) schemas/sponsored_products.py:31 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "None", variable has type "str") . Code line: endDate: str = None
python:mypy_valid_type (MINOR) source_amazon_ads/source.py:42 Check that type (annotation) is valid Function "builtins.any" is not valid as a type . Code line: ...logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
python:mypy_assignment (MINOR) source_amazon_ads/source.py:48 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "AmazonAdsConfig", variable has type "Mapping[str, Any]") . Code line: config = AmazonAdsConfig(**config)
python:mypy_arg_type (MINOR) source_amazon_ads/source.py:54 Check argument types in calls Argument 1 to "Profiles" has incompatible type "Mapping[str, Any]"; expected "AmazonAdsConfig" . Code line: Profiles(config, authenticator=self._make_authenticator(config...
python:mypy_arg_type (MINOR) source_amazon_ads/source.py:54 Check argument types in calls Argument 1 to "_make_authenticator" of "SourceAmazonAds" has incompatible type "Mapping[str, Any]"; expected "AmazonAdsConfig" . Code line: ...s(config, authenticator=self._make_authenticator(config)).get_all_prof...
python:mypy_assignment (MINOR) source_amazon_ads/source.py:62 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "AmazonAdsConfig", variable has type "Mapping[str, Any]") . Code line: config = AmazonAdsConfig(**config)
python:mypy_arg_type (MINOR) source_amazon_ads/source.py:63 Check argument types in calls Argument 1 to "_make_authenticator" of "SourceAmazonAds" has incompatible type "Mapping[str, Any]"; expected "AmazonAdsConfig" . Code line: auth = self._make_authenticator(config)
python:mypy_import (MINOR) streams/common.py:9 Require that imported module can be found or has stubs Library stubs not installed for "requests" (or incompatible with Python 3.7) . Code line: import requests
python:mypy_override (MINOR) streams/common.py:171 Check that method override is compatible with base class Return type "Optional[int]" of "next_page_token" incompatible with return type "Optional[Mapping[str, Any]]" in supertype "AmazonAdsStream" . Code line: def next_page_token(self, response: requests.Response) -> Optional...
python:mypy_no_any_return (MINOR) streams/common.py:182 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "Optional[int]" . Code line: return next_offset
python:mypy_import (MINOR) streams/profiles.py:7 Require that imported module can be found or has stubs Library stubs not installed for "requests" (or incompatible with Python 3.7) . Code line: import requests
python:mypy_assignment (MINOR) streams/profiles.py:20 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[Profile]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = Profile
python:mypy_has_type (MINOR) streams/profiles.py:27 Check that type of reference can be determined Cannot determine type of "model" . Code line: profile_id_obj = self.model.parse_obj(record)
python:mypy_has_type (MINOR) streams/profiles.py:48 Check that type of reference can be determined Cannot determine type of "model" . Code line: return [self.model.parse_obj(profile) for profile in self.read...
python:mypy_import (MINOR) report_streams/report_streams.py:17 Require that imported module can be found or has stubs Library stubs not installed for "pytz" (or incompatible with Python 3.7) . Code line: import pytz
python:mypy_import (MINOR) report_streams/report_streams.py:18 Require that imported module can be found or has stubs Library stubs not installed for "requests" (or incompatible with Python 3.7) . Code line: import requests
python:mypy_no_any_return (MINOR) report_streams/report_streams.py:116 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "CatalogModel" . Code line: return self._model
python:mypy_assignment (MINOR) report_streams/report_streams.py:280 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "str", variable has type "datetime") . Code line: next_date = next_date.strftime(ReportStream.REPORT_DATE_FO...
python:mypy_misc (MINOR) report_streams/report_streams.py:281 Miscellaneous other checks Incompatible types in "yield" (actual type "datetime", expected type "str") . Code line: yield next_date
python:mypy_assignment (MINOR) report_streams/report_streams.py:373 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "datetime", variable has type "str") . Code line: report_date = datetime.strptime(report_date, ReportStream.REPO...
python:mypy_attr_defined (MINOR) report_streams/report_streams.py:375 Check that attribute exists "str" has no attribute "astimezone" . Code line: profile_time = report_date.astimezone(profile_tz)
python:mypy_no_any_return (MINOR) report_streams/report_streams.py:390 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "List[Dict[Any, Any]]" . Code line: return json.loads(raw_string)
python:mypy_assignment (MINOR) streams/sponsored_brands.py:16 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[BrandsCampaign]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = BrandsCampaign
python:mypy_assignment (MINOR) streams/sponsored_brands.py:29 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[BrandsAdGroup]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = BrandsAdGroup
python:mypy_assignment (MINOR) streams/sponsored_brands.py:42 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[BrandsAdGroup]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = BrandsAdGroup
python:mypy_assignment (MINOR) streams/sponsored_display.py:16 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[DisplayCampaign]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = DisplayCampaign
python:mypy_assignment (MINOR) streams/sponsored_display.py:29 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[DisplayAdGroup]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = DisplayAdGroup
python:mypy_assignment (MINOR) streams/sponsored_display.py:42 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[DisplayProductAds]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = DisplayProductAds
python:mypy_assignment (MINOR) streams/sponsored_display.py:55 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[DisplayTargeting]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = DisplayTargeting
python:mypy_assignment (MINOR) streams/sponsored_products.py:16 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[ProductCampaign]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = ProductCampaign
python:mypy_assignment (MINOR) streams/sponsored_products.py:29 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[ProductAdGroups]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = ProductAdGroups
python:mypy_assignment (MINOR) streams/sponsored_products.py:42 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[Keywords]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = Keywords
python:mypy_assignment (MINOR) streams/sponsored_products.py:55 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[NegativeKeywords]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = NegativeKeywords
python:mypy_assignment (MINOR) streams/sponsored_products.py:68 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[ProductAd]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = ProductAd
python:mypy_assignment (MINOR) streams/sponsored_products.py:81 Check that assigned value is compatible with target Incompatible types in assignment (expression has type "Type[ProductTargeting]", base class "BasicAmazonAdsStream" defined the type as "CatalogModel") . Code line: model = ProductTargeting

Coverage (98.9%)

File Coverage File Coverage
source_amazon_ads/init.py 100.0 source_amazon_ads/constants.py 100.0
source_amazon_ads/schemas/init.py 100.0 source_amazon_ads/schemas/common.py 100.0
source_amazon_ads/schemas/profile.py 100.0 source_amazon_ads/schemas/sponsored_brands.py 100.0
source_amazon_ads/schemas/sponsored_display.py 100.0 source_amazon_ads/schemas/sponsored_products.py 100.0
source_amazon_ads/source.py 97.1 source_amazon_ads/spec.py 100.0
source_amazon_ads/streams/init.py 100.0 source_amazon_ads/streams/common.py 100.0
source_amazon_ads/streams/profiles.py 100.0 source_amazon_ads/streams/report_streams/init.py 100.0
source_amazon_ads/streams/report_streams/brands_report.py 100.0 source_amazon_ads/streams/report_streams/brands_video_report.py 100.0
source_amazon_ads/streams/report_streams/display_report.py 100.0 source_amazon_ads/streams/report_streams/products_report.py 100.0
source_amazon_ads/streams/report_streams/report_streams.py 96.8 source_amazon_ads/streams/sponsored_brands.py 100.0
source_amazon_ads/streams/sponsored_display.py 100.0 source_amazon_ads/streams/sponsored_products.py 100.0

Please sign in to comment.