diff --git a/airbyte-integrations/connectors/source-nasa-apod/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-nasa-apod/integration_tests/configured_catalog.json index 4286e4c985bf..8e0d39246cde 100644 --- a/airbyte-integrations/connectors/source-nasa-apod/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-nasa-apod/integration_tests/configured_catalog.json @@ -3,116 +3,7 @@ { "stream": { "name": "nasa_apod_stream", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "resource": { - "type": [ - "null", - "object" - ], - "properties": { - "image_set": { - "type": [ - "null", - "string" - ] - }, - "planet": { - "type": [ - "null", - "string" - ] - } - } - }, - "concept_tags": { - "type": [ - "null", - "boolean" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "date": { - "type": [ - "null", - "string" - ], - "format": "%Y-%m-%d" - }, - "url": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "hdurl": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "media_type": { - "type": [ - "null", - "string" - ], - "enum": [ - "image", - "video" - ] - }, - "explanation": { - "type": [ - "null", - "string" - ] - }, - "concepts": { - "type": [ - "null", - "object", - "string" - ], - "patternProperties": { - "^[0-9]+$": { - "type": [ - "null", - "string" - ] - } - } - }, - "thumbnail_url": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "copyright": { - "type": [ - "null", - "string" - ] - }, - "service_version": { - "type": [ - "null", - "string" - ], - "pattern": "^v[0-9]$" - } - } - }, + "json_schema": {}, "supported_sync_modes": [ "full_refresh" ], diff --git a/airbyte-integrations/connectors/source-nasa-apod/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-nasa-apod/sample_files/configured_catalog.json deleted file mode 100644 index f9644ac5d3ea..000000000000 --- a/airbyte-integrations/connectors/source-nasa-apod/sample_files/configured_catalog.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "nasa_apod_stream", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "resource": { - "type": [ - "null", - "object" - ], - "properties": { - "image_set": { - "type": [ - "null", - "string" - ] - }, - "planet": { - "type": [ - "null", - "string" - ] - } - } - }, - "concept_tags": { - "type": [ - "null", - "boolean" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "date": { - "type": [ - "null", - "string" - ], - "format": "%Y-%m-%d" - }, - "url": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "hdurl": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "media_type": { - "type": [ - "null", - "string" - ], - "enum": [ - "image", - "video" - ] - }, - "explanation": { - "type": [ - "null", - "string" - ] - }, - "concepts": { - "type": [ - "null", - "object", - "string" - ], - "patternProperties": { - "^[0-9]+$": { - "type": [ - "null", - "string" - ] - } - } - }, - "thumbnail_url": { - "type": [ - "null", - "string" - ], - "format": "uri" - }, - "copyright": { - "type": [ - "null", - "string" - ] - }, - "service_version": { - "type": [ - "null", - "string" - ], - "pattern": "^v[0-9]$" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "date" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/source.py b/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/source.py index 99549dbe503f..8ba17751382d 100644 --- a/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/source.py +++ b/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/source.py @@ -39,15 +39,13 @@ def request_params( return self.config def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - r = response.json() - yield [r] if type(r) is dict else r + yield response.json() # Source class SourceNasaApod(AbstractSource): count_key = "count" - date_key = "date" start_date_key = "start_date" end_date_key = "end_date" min_count_value, max_count_value = 1, 101 @@ -83,16 +81,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - if self.date_key in config: - date = self._parse_date(config[self.date_key]) - if type(date) is not datetime: - return False, date - - if self.count_key in config or self.start_date_key in config or self.end_date_key in config: - return False, self.invalid_conbination_message_template.format( - self.date_key, f"any of {', '.join([self.count_key, self.start_date_key, self.end_date_key])}" - ) - if self.start_date_key in config: start_date = self._parse_date(config[self.start_date_key]) if type(start_date) is not datetime: @@ -112,8 +100,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: if self.start_date_key not in config: return False, f"Cannot use {self.end_date_key} without specifying {self.start_date_key}." - if start_date >= end_date: - return False, f"Invalid values. start_date ({start_date}) needs to be lower than end_date ({end_date})." + if start_date > end_date: + return False, f"Invalid values. start_date ({start_date}) needs to be lower than or equal to end_date ({end_date})." if self.count_key in config: count_value = config[self.count_key] diff --git a/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/spec.yaml b/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/spec.yaml index 06608a03651e..3f59626ee5e3 100644 --- a/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/spec.yaml +++ b/airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/spec.yaml @@ -10,30 +10,41 @@ connectionSpecification: type: string description: API access key used to retrieve data from the NASA APOD API. airbyte_secret: true - date: - type: string - description: Date of the APOD image. Defaults to today's date. Must be after 1995-06-16, the first day an APOD picture was posted. There are no images for tomorrow available. - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - examples: - - "%Y-%m-%d" concept_tags: type: boolean - description: Indicates whether concept tags should be returned with the rest of the response. The concept tags are not necessarily included in the explanation, but rather derived from common search tags that are associated with the description text. (Better than just pure text search.) Defaults to False. + default: false + description: >- + Indicates whether concept tags should be returned with the rest of the response. + The concept tags are not necessarily included in the explanation, but rather derived + from common search tags that are associated with the description text. (Better than + just pure text search.) Defaults to False. count: type: integer - description: A positive integer, no greater than 100. If this is specified then `count` randomly chosen images will be returned in a JSON array. Cannot be used in conjunction with `date` or `start_date` and `end_date`. + minimum: 1 + maximum: 100 + description: >- + A positive integer, no greater than 100. If this is specified then `count` randomly + chosen images will be returned in a JSON array. Cannot be used in conjunction with + `date` or `start_date` and `end_date`. start_date: type: string - description: Indicates the start of a date range. All images in the range from `start_date` to `end_date` will be returned in a JSON array. Cannot be used with `date`. + description: >- + Indicates the start of a date range. All images in the range from `start_date` to + `end_date` will be returned in a JSON array. Cannot be used with `date`. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ examples: - - "%Y-%m-%d" + - "2022-10-20" end_date: type: string - description: Indicates that end of a date range. If `start_date` is specified without an `end_date` then `end_date` defaults to the current date. + description: >- + Indicates that end of a date range. If `start_date` is specified without an `end_date` + then `end_date` defaults to the current date. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ examples: - - "%Y-%m-%d" + - "2022-10-20" thumbs: type: boolean - description: Indicates whether the API should return a thumbnail image URL for video files. If set to True, the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored. + default: false + description: >- + Indicates whether the API should return a thumbnail image URL for video files. If set to True, + the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored. diff --git a/airbyte-integrations/connectors/source-nasa-apod/unit_tests/test_source.py b/airbyte-integrations/connectors/source-nasa-apod/unit_tests/test_source.py index e47878228790..5d2da74ee1e2 100644 --- a/airbyte-integrations/connectors/source-nasa-apod/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-nasa-apod/unit_tests/test_source.py @@ -19,28 +19,6 @@ ("config", "expected_return"), [ ({"api_key": "foobar"}, (True, None)), - ({"api_key": "foobar", "date": valid_date_str}, (True, None)), - ({"api_key": "foobar", "date": "x"}, (False, "Invalid date value: x. It should be formatted as '%Y-%m-%d'.")), - ( - {"api_key": "foobar", "date": "1990-01-01"}, - (False, f"Invalid date value: 1990-01-01. The value should be in the range [{min_date},{tomorrow})."), - ), - ( - {"api_key": "foobar", "date": after_tomorrow_str}, - (False, f"Invalid date value: {after_tomorrow_str}. The value should be in the range [{min_date},{tomorrow})."), - ), - ( - {"api_key": "foobar", "date": valid_date_str, "count": 5}, - (False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."), - ), - ( - {"api_key": "foobar", "date": valid_date_str, "start_date": valid_date_str}, - (False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."), - ), - ( - {"api_key": "foobar", "date": valid_date_str, "end_date": valid_date_str}, - (False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."), - ), ({"api_key": "foobar", "start_date": valid_date_str}, (True, None)), ( {"api_key": "foobar", "start_date": valid_date_str, "count": 5}, @@ -55,7 +33,7 @@ {"api_key": "foobar", "start_date": valid_date_str, "end_date": min_date.strftime(date_format)}, ( False, - f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than end_date ({min_date}).", + f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than or equal to end_date ({min_date}).", ), ), ({"api_key": "foobar", "start_date": min_date.strftime(date_format), "end_date": valid_date_str}, (True, None)),