Skip to content

Commit

Permalink
date input removal in favor of start_date + Minor fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Xabilahu
Xabilahu committed Oct 25, 2022
1 parent fae2583 commit 41b5605
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 290 deletions.
111 changes: 1 addition & 110 deletions airbyte-integrations/connectors/source-nasa-apod/integration_tests/configured_catalog.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,116 +3,7 @@
{
"stream": {
"name": "nasa_apod_stream",
"json_schema": {
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"resource": {
"type": [
"null",
"object"
],
"properties": {
"image_set": {
"type": [
"null",
"string"
]
},
"planet": {
"type": [
"null",
"string"
]
}
}
},
"concept_tags": {
"type": [
"null",
"boolean"
]
},
"title": {
"type": [
"null",
"string"
]
},
"date": {
"type": [
"null",
"string"
],
"format": "%Y-%m-%d"
},
"url": {
"type": [
"null",
"string"
],
"format": "uri"
},
"hdurl": {
"type": [
"null",
"string"
],
"format": "uri"
},
"media_type": {
"type": [
"null",
"string"
],
"enum": [
"image",
"video"
]
},
"explanation": {
"type": [
"null",
"string"
]
},
"concepts": {
"type": [
"null",
"object",
"string"
],
"patternProperties": {
"^[0-9]+$": {
"type": [
"null",
"string"
]
}
}
},
"thumbnail_url": {
"type": [
"null",
"string"
],
"format": "uri"
},
"copyright": {
"type": [
"null",
"string"
]
},
"service_version": {
"type": [
"null",
"string"
],
"pattern": "^v[0-9]$"
}
}
},
"json_schema": {},
"supported_sync_modes": [
"full_refresh"
],
Expand Down
129 changes: 0 additions & 129 deletions airbyte-integrations/connectors/source-nasa-apod/sample_files/configured_catalog.json
View file

This file was deleted.

18 changes: 3 additions & 15 deletions airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,13 @@ def request_params(
return self.config

def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
r = response.json()
yield [r] if type(r) is dict else r
yield response.json()


# Source
class SourceNasaApod(AbstractSource):

count_key = "count"
date_key = "date"
start_date_key = "start_date"
end_date_key = "end_date"
min_count_value, max_count_value = 1, 101
Expand Down Expand Up @@ -83,16 +81,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
:param logger: logger object
:return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
"""
if self.date_key in config:
date = self._parse_date(config[self.date_key])
if type(date) is not datetime:
return False, date

if self.count_key in config or self.start_date_key in config or self.end_date_key in config:
return False, self.invalid_conbination_message_template.format(
self.date_key, f"any of {', '.join([self.count_key, self.start_date_key, self.end_date_key])}"
)

if self.start_date_key in config:
start_date = self._parse_date(config[self.start_date_key])
if type(start_date) is not datetime:
Expand All @@ -112,8 +100,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
if self.start_date_key not in config:
return False, f"Cannot use {self.end_date_key} without specifying {self.start_date_key}."

if start_date >= end_date:
return False, f"Invalid values. start_date ({start_date}) needs to be lower than end_date ({end_date})."
if start_date > end_date:
return False, f"Invalid values. start_date ({start_date}) needs to be lower than or equal to end_date ({end_date})."

if self.count_key in config:
count_value = config[self.count_key]
Expand Down
37 changes: 24 additions & 13 deletions airbyte-integrations/connectors/source-nasa-apod/source_nasa_apod/spec.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,30 +10,41 @@ connectionSpecification:
type: string
description: API access key used to retrieve data from the NASA APOD API.
airbyte_secret: true
date:
type: string
description: Date of the APOD image. Defaults to today's date. Must be after 1995-06-16, the first day an APOD picture was posted. There are no images for tomorrow available.
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$
examples:
- "%Y-%m-%d"
concept_tags:
type: boolean
description: Indicates whether concept tags should be returned with the rest of the response. The concept tags are not necessarily included in the explanation, but rather derived from common search tags that are associated with the description text. (Better than just pure text search.) Defaults to False.
default: false
description: >-
Indicates whether concept tags should be returned with the rest of the response.
The concept tags are not necessarily included in the explanation, but rather derived
from common search tags that are associated with the description text. (Better than
just pure text search.) Defaults to False.
count:
type: integer
description: A positive integer, no greater than 100. If this is specified then `count` randomly chosen images will be returned in a JSON array. Cannot be used in conjunction with `date` or `start_date` and `end_date`.
minimum: 1
maximum: 100
description: >-
A positive integer, no greater than 100. If this is specified then `count` randomly
chosen images will be returned in a JSON array. Cannot be used in conjunction with
`date` or `start_date` and `end_date`.
start_date:
type: string
description: Indicates the start of a date range. All images in the range from `start_date` to `end_date` will be returned in a JSON array. Cannot be used with `date`.
description: >-
Indicates the start of a date range. All images in the range from `start_date` to
`end_date` will be returned in a JSON array. Cannot be used with `date`.
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$
examples:
- "%Y-%m-%d"
- "2022-10-20"
end_date:
type: string
description: Indicates that end of a date range. If `start_date` is specified without an `end_date` then `end_date` defaults to the current date.
description: >-
Indicates that end of a date range. If `start_date` is specified without an `end_date`
then `end_date` defaults to the current date.
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$
examples:
- "%Y-%m-%d"
- "2022-10-20"
thumbs:
type: boolean
description: Indicates whether the API should return a thumbnail image URL for video files. If set to True, the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored.
default: false
description: >-
Indicates whether the API should return a thumbnail image URL for video files. If set to True,
the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored.
24 changes: 1 addition & 23 deletions airbyte-integrations/connectors/source-nasa-apod/unit_tests/test_source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,6 @@
("config", "expected_return"),
[
({"api_key": "foobar"}, (True, None)),
({"api_key": "foobar", "date": valid_date_str}, (True, None)),
({"api_key": "foobar", "date": "x"}, (False, "Invalid date value: x. It should be formatted as '%Y-%m-%d'.")),
(
{"api_key": "foobar", "date": "1990-01-01"},
(False, f"Invalid date value: 1990-01-01. The value should be in the range [{min_date},{tomorrow})."),
),
(
{"api_key": "foobar", "date": after_tomorrow_str},
(False, f"Invalid date value: {after_tomorrow_str}. The value should be in the range [{min_date},{tomorrow})."),
),
(
{"api_key": "foobar", "date": valid_date_str, "count": 5},
(False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."),
),
(
{"api_key": "foobar", "date": valid_date_str, "start_date": valid_date_str},
(False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."),
),
(
{"api_key": "foobar", "date": valid_date_str, "end_date": valid_date_str},
(False, "Invalid parameter combination. Cannot use date and any of count, start_date, end_date together."),
),
({"api_key": "foobar", "start_date": valid_date_str}, (True, None)),
(
{"api_key": "foobar", "start_date": valid_date_str, "count": 5},
Expand All @@ -55,7 +33,7 @@
{"api_key": "foobar", "start_date": valid_date_str, "end_date": min_date.strftime(date_format)},
(
False,
f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than end_date ({min_date}).",
f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than or equal to end_date ({min_date}).",
),
),
({"api_key": "foobar", "start_date": min_date.strftime(date_format), "end_date": valid_date_str}, (True, None)),
Expand Down

0 comments on commit 41b5605

Please sign in to comment.