From 9f56e96566a27d0c9f3644968fc544f525efe804 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 11:43:34 -0700 Subject: [PATCH 01/20] chore: initial refactor for readability --- singer_sdk/plugin_base.py | 130 +++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 52 deletions(-) diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 6e1908b72..e19173523 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -21,6 +21,7 @@ ) import click +import yaml from jsonschema import Draft4Validator, SchemaError, ValidationError from singer_sdk import metrics @@ -341,64 +342,89 @@ def print_about(cls: Type["PluginBase"], format: Optional[str] = None) -> None: if format == "json": print(json.dumps(info, indent=2, default=str)) + return - elif format == "markdown": - max_setting_len = cast( - int, max(len(k) for k in info["settings"]["properties"].keys()) - ) + if format == "markdown": + cls._print_about_markdown(info) + return - # Set table base for markdown - table_base = ( - f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" - f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" - ) + if format == "yaml": + cls._print_about_yaml(info) + return - # Empty list for string parts - md_list = [] - # Get required settings for table - required_settings = info["settings"].get("required", []) + formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) + print(formatted) - # Iterate over Dict to set md - md_list.append( - f"# `{info['name']}`\n\n" - f"{info['description']}\n\n" - f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" - ) - for key, value in info.items(): - - if key == "capabilities": - capabilities = f"## {key.title()}\n\n" - capabilities += "\n".join([f"* `{v}`" for v in value]) - capabilities += "\n\n" - md_list.append(capabilities) - - if key == "settings": - setting = f"## {key.title()}\n\n" - for k, v in info["settings"].get("properties", {}).items(): - md_description = v.get("description", "").replace("\n", "
") - table_base += ( - f"| {k}{' ' * (max_setting_len - len(k))}" - f"| {'True' if k in required_settings else 'False':8} | " - f"{v.get('default', 'None'):7} | " - f"{md_description:11} |\n" - ) - setting += table_base - setting += ( - "\n" - + "\n".join( - [ - "A full list of supported settings and capabilities " - f"is available by running: `{info['name']} --about`" - ] - ) - + "\n" + @classmethod + def _print_about_markdown(cls: Type["PluginBase"], info: dict) -> None: + """Print about info as markdown. + + Args: + info: The collected metadata for the class. + """ + max_setting_len = cast( + int, max(len(k) for k in info["settings"]["properties"].keys()) + ) + + # Set table base for markdown + table_base = ( + f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" + f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" + ) + + # Empty list for string parts + md_list = [] + # Get required settings for table + required_settings = info["settings"].get("required", []) + + # Iterate over Dict to set md + md_list.append( + f"# `{info['name']}`\n\n" + f"{info['description']}\n\n" + f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" + ) + for key, value in info.items(): + + if key == "capabilities": + capabilities = f"## {key.title()}\n\n" + capabilities += "\n".join([f"* `{v}`" for v in value]) + capabilities += "\n\n" + md_list.append(capabilities) + + if key == "settings": + setting = f"## {key.title()}\n\n" + for k, v in info["settings"].get("properties", {}).items(): + md_description = v.get("description", "").replace("\n", "
") + table_base += ( + f"| {k}{' ' * (max_setting_len - len(k))}" + f"| {'True' if k in required_settings else 'False':8} | " + f"{v.get('default', 'None'):7} | " + f"{md_description:11} |\n" + ) + setting += table_base + setting += ( + "\n" + + "\n".join( + [ + "A full list of supported settings and capabilities " + f"is available by running: `{info['name']} --about`" + ] ) - md_list.append(setting) + + "\n" + ) + md_list.append(setting) - print("".join(md_list)) - else: - formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) - print(formatted) + print("".join(md_list)) + + @classmethod + def _print_about_yaml(cls: Type["PluginBase"], info: dict) -> None: + """Print about info as YAML. + + Args: + info: The collected metadata for the class. + """ + yaml_structure: Dict[str, Any] = {} + print(yaml.dump(yaml_structure)) @classproperty def cli(cls) -> Callable: From b85760a6f9888965b56a73d9cf8e85c9f81d3869 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 12:53:18 -0700 Subject: [PATCH 02/20] feat: add Meltano rendering logic in private helper module --- singer_sdk/helpers/_meltano.py | 110 +++++++++++++++++++++++++++++++++ singer_sdk/helpers/_typing.py | 61 ++++++++++++++++++ singer_sdk/plugin_base.py | 15 +---- 3 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 singer_sdk/helpers/_meltano.py diff --git a/singer_sdk/helpers/_meltano.py b/singer_sdk/helpers/_meltano.py new file mode 100644 index 000000000..0e9a7e987 --- /dev/null +++ b/singer_sdk/helpers/_meltano.py @@ -0,0 +1,110 @@ +"""Helper functions for Meltano and MeltanoHub interop.""" + +from __future__ import annotations + +from ._typing import ( + is_array_type, + is_boolean_type, + is_datetime_type, + is_integer_type, + is_object_type, + is_secret_type, + is_string_type, +) + + +def _to_meltano_kind(jsonschema_type: dict) -> str | None: + """Returns a Meltano `kind` indicator for the provided JSON Schema type. + + For reference: + https://docs.meltano.com/reference/plugin-definition-syntax#settingskind + + Args: + jsonschema_type: JSON Schema type to check. + + Returns: + A string representing the meltano 'kind'. + """ + if is_secret_type(jsonschema_type): + return "password" + + if is_string_type(jsonschema_type): + return "string" + + if is_object_type(jsonschema_type): + return "object" + + if is_array_type(jsonschema_type): + return "array" + + if is_boolean_type(jsonschema_type): + return "boolean" + + if is_datetime_type(jsonschema_type): + return "date_iso8601" + + if is_integer_type(jsonschema_type): + return "integer" + + return None + + +def meltano_yaml_str( + plugin_name: str, + capabilities: list[str], + config_jsonschema: dict, +) -> str: + """Returns a Meltano plugin definition as a yaml string. + + Args: + plugin_name: Name of the plugin. + capabilities: List of capabilities. + config_jsonschema: JSON Schema of the expected config. + + Returns: + A string representing the Meltano plugin Yaml definition. + """ + capabilities_str: str = "\n".join( + [" - {capability}" for capability in capabilities] + ) + settings_str: str = "\n".join( + [ + f""" +- name: {setting_name} + label: {setting_name.replace("_", " ").proper()} + kind: {_to_meltano_kind(type_dict["type"])}, + description: {type_dict.get("description", 'null')} +""" + for setting_name, type_dict in config_jsonschema["properties"].items() + ] + ) + required_settings = [ + setting_name + for setting_name, type_dict in config_jsonschema.items() + if setting_name in config_jsonschema.get("required", []) + or type_dict.get("required", False) + ] + settings_group_validation_str = " - - " + "\n - ".join(required_settings) + + return f""" +name: {plugin_name} +namespace: {plugin_name.replace('-', '_')} + +## The following could not be auto-detected: +# maintenance_status: # +# repo: # +# variant: # +# label: # +# description: # +# pip_url: # +# domain_url: # +# logo_url: # +# keywords: [] # + +capabilities: +{capabilities_str} +settings_group_validation: +{settings_group_validation_str} +settings: +{settings_str} + """ diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index dc8389681..6e2b84e10 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -54,6 +54,40 @@ def append_type(type_dict: dict, new_type: str) -> dict: ) +def is_secret_type(type_dict: dict) -> bool: + """Return True if JSON Schema type definition appears to be a secret. + + Will return true if either `writeOnly` or `sensitive` are true on this type + or any of the type's subproperties. + + Args: + type_dict: The JSON Schema type to check. + + Raises: + ValueError: If type_dict is None or empty. + + Returns: + True if we detect any sensitive property nodes. + """ + if not type_dict: + raise ValueError( + "Could not detect type from empty type_dict. " + "Did you forget to define a property in the stream schema?" + ) + + if type_dict.get("writeOnly") or type_dict.get("sensitive"): + return True + + if "properties" in type_dict: + # Recursively check subproperties and return True if any child is secret. + return any( + is_secret_type(child_type_dict) + for child_type_dict in type_dict["properties"].values() + ) + + return False + + def is_object_type(property_schema: dict) -> Optional[bool]: """Return true if the JSON Schema type is an object or None if detection fails.""" if "anyOf" not in property_schema and "type" not in property_schema: @@ -152,6 +186,23 @@ def is_string_array_type(type_dict: dict) -> bool: return "array" in type_dict["type"] and bool(is_string_type(type_dict["items"])) +def is_array_type(type_dict: dict) -> bool: + """Return True if JSON Schema type definition is a string array.""" + if not type_dict: + raise ValueError( + "Could not detect type from empty type_dict. " + "Did you forget to define a property in the stream schema?" + ) + + if "anyOf" in type_dict: + return any([is_array_type(t) for t in type_dict["anyOf"]]) + + if "type" not in type_dict: + raise ValueError(f"Could not detect type from schema '{type_dict}'") + + return "array" in type_dict["type"] + + def is_boolean_type(property_schema: dict) -> Optional[bool]: """Return true if the JSON Schema type is a boolean or None if detection fails.""" if "anyOf" not in property_schema and "type" not in property_schema: @@ -162,6 +213,16 @@ def is_boolean_type(property_schema: dict) -> Optional[bool]: return False +def is_integer_type(property_schema: dict) -> Optional[bool]: + """Return true if the JSON Schema type is a boolean or None if detection fails.""" + if "anyOf" not in property_schema and "type" not in property_schema: + return None # Could not detect data type + for property_type in property_schema.get("anyOf", [property_schema.get("type")]): + if "integer" in property_type or property_type == "integer": + return True + return False + + def is_string_type(property_schema: dict) -> Optional[bool]: """Return true if the JSON Schema type is a boolean or None if detection fails.""" if "anyOf" not in property_schema and "type" not in property_schema: diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index e19173523..5ebfc3de4 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -29,6 +29,7 @@ from singer_sdk.exceptions import ConfigValidationError from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import metadata +from singer_sdk.helpers._meltano import meltano_yaml_str from singer_sdk.helpers._secrets import SecretString, is_common_secret_key from singer_sdk.helpers._util import read_json_file from singer_sdk.helpers.capabilities import ( @@ -348,8 +349,8 @@ def print_about(cls: Type["PluginBase"], format: Optional[str] = None) -> None: cls._print_about_markdown(info) return - if format == "yaml": - cls._print_about_yaml(info) + if format == "meltano": + print(meltano_yaml_str(cls.name, cls.capabilities, cls.config_jsonschema)) return formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) @@ -416,16 +417,6 @@ def _print_about_markdown(cls: Type["PluginBase"], info: dict) -> None: print("".join(md_list)) - @classmethod - def _print_about_yaml(cls: Type["PluginBase"], info: dict) -> None: - """Print about info as YAML. - - Args: - info: The collected metadata for the class. - """ - yaml_structure: Dict[str, Any] = {} - print(yaml.dump(yaml_structure)) - @classproperty def cli(cls) -> Callable: """Handle command line execution. From 8d95b22fd42ff907a47bdab1a1ed2a676fdba247 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 13:47:00 -0700 Subject: [PATCH 03/20] feat: add `secret=True` support in JSON Schema type helpers --- singer_sdk/typing.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index e373c2113..c43122fb1 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -352,21 +352,30 @@ def __init__( required: bool = False, default: _JsonValue = None, description: str = None, + secret: bool = False, ) -> None: """Initialize Property object. + Note: Properties containing secrets should be specified with `secret=True`. + Doing so will add the annotation `writeOnly=True`, in accordance with JSON + Schema Draft 7 and later, and `secret=True` as an additional hint to readers. + + More info: https://json-schema.org/draft-07/json-schema-release-notes.html + Args: name: Property name. wrapped: JSON Schema type of the property. required: Whether this is a required property. default: Default value in the JSON Schema. description: Long-text property description. + secret: True if this is a credential or other secret. """ self.name = name self.wrapped = wrapped self.optional = not required self.default = default self.description = description + self.secret = secret @property def type_dict(self) -> dict: # type: ignore # OK: @classproperty vs @property @@ -402,6 +411,13 @@ def to_dict(self) -> dict: type_dict.update({"default": self.default}) if self.description: type_dict.update({"description": self.description}) + if self.secret: + type_dict.update( + { + "secret": True, + "writeOnly": True, + } + ) return {self.name: type_dict} From 2820b91d4cd6c17c1998582c32c45909823c6084 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 13:55:29 -0700 Subject: [PATCH 04/20] change: update examples to use 'secret=True' for protected settings --- .../{{cookiecutter.library_name}}/tap.py | 1 + .../{{cookiecutter.library_name}}/target.py | 7 +++++++ samples/sample_tap_gitlab/gitlab_tap.py | 2 +- samples/sample_tap_google_analytics/ga_tap.py | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index 42c3f8ab5..e41c337d7 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -38,6 +38,7 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type == "auth_token", th.StringType, required=True, + secret=True, # Flag config as protected. description="The token to authenticate against the API service" ), th.Property( diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py b/cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py index b6976b337..35b4a4675 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py @@ -21,6 +21,7 @@ class Target{{ cookiecutter.destination_name }}({{ target_class }}): th.Property( "sqlalchemy_url", th.StringType, + secret=True, # Flag config as protected. description="SQLAlchemy connection string", ), {%- else %} @@ -34,6 +35,12 @@ class Target{{ cookiecutter.destination_name }}({{ target_class }}): th.StringType, description="The scheme with which output files will be named" ), + th.Property( + "auth_token", + th.StringType, + secret=True, # Flag config as protected. + description="The path to the target output file" + ), {%- endif %} ).to_dict() diff --git a/samples/sample_tap_gitlab/gitlab_tap.py b/samples/sample_tap_gitlab/gitlab_tap.py index 6bdb04fb0..0f02697df 100644 --- a/samples/sample_tap_gitlab/gitlab_tap.py +++ b/samples/sample_tap_gitlab/gitlab_tap.py @@ -34,7 +34,7 @@ class SampleTapGitlab(Tap): name: str = "sample-tap-gitlab" config_jsonschema = PropertiesList( - Property("auth_token", StringType, required=True), + Property("auth_token", StringType, required=True, secret=True), Property("project_ids", ArrayType(StringType), required=True), Property("group_ids", ArrayType(StringType), required=True), Property("start_date", DateTimeType, required=True), diff --git a/samples/sample_tap_google_analytics/ga_tap.py b/samples/sample_tap_google_analytics/ga_tap.py index 8044df0b2..76fd952b2 100644 --- a/samples/sample_tap_google_analytics/ga_tap.py +++ b/samples/sample_tap_google_analytics/ga_tap.py @@ -24,7 +24,7 @@ class SampleTapGoogleAnalytics(Tap): config_jsonschema = PropertiesList( Property("view_id", StringType(), required=True), Property("client_email", StringType(), required=True), - Property("private_key", StringType(), required=True), + Property("private_key", StringType(), required=True, secret=True), ).to_dict() def discover_streams(self) -> List[SampleGoogleAnalyticsStream]: From e74be3f35926b39f48824337b6a468f8a0d0fabe Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 13:58:18 -0700 Subject: [PATCH 05/20] chore: flake8 fix --- singer_sdk/plugin_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 5ebfc3de4..152ae14d9 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -21,7 +21,6 @@ ) import click -import yaml from jsonschema import Draft4Validator, SchemaError, ValidationError from singer_sdk import metrics From e0239b5ae40e75d4b9f8b2570a90f429462dafd8 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 14:20:57 -0700 Subject: [PATCH 06/20] add unit tests for type helpers --- tests/core/test_jsonschema_helpers.py | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 175d0b577..f670912d8 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -253,6 +253,52 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): assert json_type.type_dict == expected_json_schema +@pytest.mark.parametrize( + "property_obj,expected_jsonschema", + [ + ( + Property("my_prop1", StringType, required=True), + {"my_prop1": {"type": ["string"]}}, + ), + ( + Property("my_prop2", StringType, required=False), + {"my_prop2": {"type": ["string", "null"]}}, + ), + ( + Property("my_prop3", StringType, secret=True), + { + "my_prop3": { + "type": ["string", "null"], + "secret": True, + "writeOnly": True, + } + }, + ), + ( + Property("my_prop4", StringType, description="This is a property."), + { + "my_prop4": { + "description": "This is a property.", + "type": ["string", "null"], + } + }, + ), + ( + Property("my_prop5", StringType, default="some_val"), + { + "my_prop5": { + "default": "some_val", + "type": ["string", "null"], + } + }, + ), + ], +) +def test_property_creation(property_obj: Property, expected_jsonschema: dict) -> None: + assert property_obj.to_dict() == expected_jsonschema + # assert property_obj.type_dict == expected_jsonschema["type"] + + def test_wrapped_type_dict(): with pytest.raises( ValueError, From 17905b12751aff5ac954baa182011e4ce8298411 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 14:24:16 -0700 Subject: [PATCH 07/20] fix missing secret flag on unit test --- tests/core/test_jsonschema_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index f670912d8..0ce444603 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -43,7 +43,7 @@ class ConfigTestTap(Tap): config_jsonschema = PropertiesList( Property("host", StringType, required=True), Property("username", StringType, required=True), - Property("password", StringType, required=True), + Property("password", StringType, required=True, secret=True), Property("batch_size", IntegerType, default=-1), ).to_dict() From 9d6a36425589474d5b2aed35c56dde35693b5027 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 15:25:52 -0700 Subject: [PATCH 08/20] chore: get tests passing --- singer_sdk/helpers/_meltano.py | 47 +++++------ singer_sdk/helpers/_typing.py | 34 ++++++++ tests/core/test_meltano_helpers.py | 125 +++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 25 deletions(-) create mode 100644 tests/core/test_meltano_helpers.py diff --git a/singer_sdk/helpers/_meltano.py b/singer_sdk/helpers/_meltano.py index 0e9a7e987..346099deb 100644 --- a/singer_sdk/helpers/_meltano.py +++ b/singer_sdk/helpers/_meltano.py @@ -5,7 +5,7 @@ from ._typing import ( is_array_type, is_boolean_type, - is_datetime_type, + is_date_or_datetime_type, is_integer_type, is_object_type, is_secret_type, @@ -13,8 +13,8 @@ ) -def _to_meltano_kind(jsonschema_type: dict) -> str | None: - """Returns a Meltano `kind` indicator for the provided JSON Schema type. +def _to_meltano_kind(jsonschema_def: dict) -> str | None: + """Returns a Meltano `kind` indicator for the provided JSON Schema property node. For reference: https://docs.meltano.com/reference/plugin-definition-syntax#settingskind @@ -25,25 +25,25 @@ def _to_meltano_kind(jsonschema_type: dict) -> str | None: Returns: A string representing the meltano 'kind'. """ - if is_secret_type(jsonschema_type): + if is_secret_type(jsonschema_def): return "password" - if is_string_type(jsonschema_type): + if is_date_or_datetime_type(jsonschema_def): + return "date_iso8601" + + if is_string_type(jsonschema_def): return "string" - if is_object_type(jsonschema_type): + if is_object_type(jsonschema_def): return "object" - if is_array_type(jsonschema_type): + if is_array_type(jsonschema_def): return "array" - if is_boolean_type(jsonschema_type): + if is_boolean_type(jsonschema_def): return "boolean" - if is_datetime_type(jsonschema_type): - return "date_iso8601" - - if is_integer_type(jsonschema_type): + if is_integer_type(jsonschema_def): return "integer" return None @@ -65,29 +65,26 @@ def meltano_yaml_str( A string representing the Meltano plugin Yaml definition. """ capabilities_str: str = "\n".join( - [" - {capability}" for capability in capabilities] + [f" - {capability}" for capability in capabilities] ) settings_str: str = "\n".join( [ - f""" -- name: {setting_name} - label: {setting_name.replace("_", " ").proper()} - kind: {_to_meltano_kind(type_dict["type"])}, - description: {type_dict.get("description", 'null')} -""" - for setting_name, type_dict in config_jsonschema["properties"].items() + f"""- name: {setting_name} + label: {setting_name.replace("_", " ").title()} + kind: {_to_meltano_kind(property_node)}, + description: {property_node.get("description", 'null')}""" + for setting_name, property_node in config_jsonschema["properties"].items() ] ) required_settings = [ setting_name - for setting_name, type_dict in config_jsonschema.items() + for setting_name, type_dict in config_jsonschema["properties"].items() if setting_name in config_jsonschema.get("required", []) or type_dict.get("required", False) ] - settings_group_validation_str = " - - " + "\n - ".join(required_settings) + settings_group_validation_str = " - - " + "\n - ".join(required_settings) - return f""" -name: {plugin_name} + return f"""name: {plugin_name} namespace: {plugin_name.replace('-', '_')} ## The following could not be auto-detected: @@ -107,4 +104,4 @@ def meltano_yaml_str( {settings_group_validation_str} settings: {settings_str} - """ +""" diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6e2b84e10..d7463040b 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -120,6 +120,40 @@ def is_datetime_type(type_dict: dict) -> bool: ) +def is_date_or_datetime_type(type_dict: dict) -> bool: + """Return True if JSON Schema type definition is a 'date'/'date-time' type. + + Also returns True if type is nested within an 'anyOf' type Array. + + Args: + type_dict: The JSON Schema definition. + + Raises: + ValueError: If type is empty or null. + + Returns: + True if date or date-time, else False. + """ + if not type_dict: + raise ValueError( + "Could not detect type from empty type_dict. " + "Did you forget to define a property in the stream schema?" + ) + + if "anyOf" in type_dict: + for type_dict in type_dict["anyOf"]: + if is_date_or_datetime_type(type_dict): + return True + return False + + if "type" in type_dict: + return type_dict.get("format") in {"date", "date-time"} + + raise ValueError( + f"Could not detect type of replication key using schema '{type_dict}'" + ) + + def get_datelike_property_type(property_schema: Dict) -> Optional[str]: """Return one of 'date-time', 'time', or 'date' if property is date-like. diff --git a/tests/core/test_meltano_helpers.py b/tests/core/test_meltano_helpers.py new file mode 100644 index 000000000..53ebcb4c8 --- /dev/null +++ b/tests/core/test_meltano_helpers.py @@ -0,0 +1,125 @@ +"""Test sample sync.""" + +from __future__ import annotations + +import pytest + +from singer_sdk.helpers._meltano import _to_meltano_kind, meltano_yaml_str +from singer_sdk.typing import ( + ArrayType, + BooleanType, + DateTimeType, + DateType, + DurationType, + EmailType, + HostnameType, + IntegerType, + IPv4Type, + IPv6Type, + JSONPointerType, + NumberType, + ObjectType, + PropertiesList, + Property, + RegexType, + RelativeJSONPointerType, + StringType, + TimeType, + URIReferenceType, + URITemplateType, + URIType, + UUIDType, +) + + +@pytest.mark.parametrize( + "plugin_name,capabilities,config_jsonschema,expected_yaml", + [ + ( + "tap-from-source", + ["about", "stream-maps"], + PropertiesList( + Property("username", StringType, required=True), + Property("password", StringType, required=True, secret=True), + Property("start_date", DateType), + ).to_dict(), + """name: tap-from-source +namespace: tap_from_source + +## The following could not be auto-detected: +# maintenance_status: # +# repo: # +# variant: # +# label: # +# description: # +# pip_url: # +# domain_url: # +# logo_url: # +# keywords: [] # + +capabilities: + - about + - stream-maps +settings_group_validation: + - - username + - password +settings: +- name: username + label: Username + kind: string, + description: null +- name: password + label: Password + kind: password, + description: null +- name: start_date + label: Start Date + kind: date_iso8601, + description: null +""", + ) + ], +) +def test_meltano_yml_creation( + plugin_name: str, + capabilities: list[str], + config_jsonschema: dict, + expected_yaml: str, +): + assert expected_yaml == meltano_yaml_str( + plugin_name, capabilities, config_jsonschema + ) + + +@pytest.mark.parametrize( + "type_dict,expected_kindstr", + [ + # Handled Types: + (StringType.type_dict, "string"), + (DateTimeType.type_dict, "date_iso8601"), + (DateType.type_dict, "date_iso8601"), + (BooleanType.type_dict, "boolean"), + (IntegerType.type_dict, "integer"), + ( + DurationType.type_dict, + "string", + ), + # Treat as strings: + (TimeType.type_dict, "string"), + (EmailType.type_dict, "string"), + (HostnameType.type_dict, "string"), + (IPv4Type.type_dict, "string"), + (IPv6Type.type_dict, "string"), + (UUIDType.type_dict, "string"), + (URIType.type_dict, "string"), + (URIReferenceType.type_dict, "string"), + (URITemplateType.type_dict, "string"), + (JSONPointerType.type_dict, "string"), + (RelativeJSONPointerType.type_dict, "string"), + (RegexType.type_dict, "string"), + # No handling and no compatible default: + (NumberType.type_dict, None), + ], +) +def test_meltano_type_to_kind(type_dict: dict, expected_kindstr: str | None) -> None: + assert _to_meltano_kind(type_dict) == expected_kindstr From 8ad77278d2d88dd1b4e289ce2fb15078220e47df Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 15:31:07 -0700 Subject: [PATCH 09/20] chore: add test for description --- singer_sdk/helpers/_meltano.py | 2 +- tests/core/test_meltano_helpers.py | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/singer_sdk/helpers/_meltano.py b/singer_sdk/helpers/_meltano.py index 346099deb..0b397dff0 100644 --- a/singer_sdk/helpers/_meltano.py +++ b/singer_sdk/helpers/_meltano.py @@ -71,7 +71,7 @@ def meltano_yaml_str( [ f"""- name: {setting_name} label: {setting_name.replace("_", " ").title()} - kind: {_to_meltano_kind(property_node)}, + kind: {_to_meltano_kind(property_node)} description: {property_node.get("description", 'null')}""" for setting_name, property_node in config_jsonschema["properties"].items() ] diff --git a/tests/core/test_meltano_helpers.py b/tests/core/test_meltano_helpers.py index 53ebcb4c8..a0df899f8 100644 --- a/tests/core/test_meltano_helpers.py +++ b/tests/core/test_meltano_helpers.py @@ -39,8 +39,19 @@ "tap-from-source", ["about", "stream-maps"], PropertiesList( - Property("username", StringType, required=True), - Property("password", StringType, required=True, secret=True), + Property( + "username", + StringType, + required=True, + description="The username to connect with.", + ), + Property( + "password", + StringType, + required=True, + secret=True, + description="The user's password.", + ), Property("start_date", DateType), ).to_dict(), """name: tap-from-source @@ -66,15 +77,15 @@ settings: - name: username label: Username - kind: string, - description: null + kind: string + description: The username to connect with. - name: password label: Password - kind: password, - description: null + kind: password + description: The user's password. - name: start_date label: Start Date - kind: date_iso8601, + kind: date_iso8601 description: null """, ) From 2792aa1b8979911cf2ec410380bf4b59b62f21ac Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 15:37:41 -0700 Subject: [PATCH 10/20] chore: remove commented code --- tests/core/test_jsonschema_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 0ce444603..fc9328433 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -296,7 +296,6 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): ) def test_property_creation(property_obj: Property, expected_jsonschema: dict) -> None: assert property_obj.to_dict() == expected_jsonschema - # assert property_obj.type_dict == expected_jsonschema["type"] def test_wrapped_type_dict(): From b86cfc412d79dd534c1bd6aea77cefa0bebeb73e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 16:18:51 -0700 Subject: [PATCH 11/20] chore: remove files related to #1094 --- singer_sdk/helpers/_meltano.py | 107 ----------------------- tests/core/test_meltano_helpers.py | 136 ----------------------------- 2 files changed, 243 deletions(-) delete mode 100644 singer_sdk/helpers/_meltano.py delete mode 100644 tests/core/test_meltano_helpers.py diff --git a/singer_sdk/helpers/_meltano.py b/singer_sdk/helpers/_meltano.py deleted file mode 100644 index 0b397dff0..000000000 --- a/singer_sdk/helpers/_meltano.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Helper functions for Meltano and MeltanoHub interop.""" - -from __future__ import annotations - -from ._typing import ( - is_array_type, - is_boolean_type, - is_date_or_datetime_type, - is_integer_type, - is_object_type, - is_secret_type, - is_string_type, -) - - -def _to_meltano_kind(jsonschema_def: dict) -> str | None: - """Returns a Meltano `kind` indicator for the provided JSON Schema property node. - - For reference: - https://docs.meltano.com/reference/plugin-definition-syntax#settingskind - - Args: - jsonschema_type: JSON Schema type to check. - - Returns: - A string representing the meltano 'kind'. - """ - if is_secret_type(jsonschema_def): - return "password" - - if is_date_or_datetime_type(jsonschema_def): - return "date_iso8601" - - if is_string_type(jsonschema_def): - return "string" - - if is_object_type(jsonschema_def): - return "object" - - if is_array_type(jsonschema_def): - return "array" - - if is_boolean_type(jsonschema_def): - return "boolean" - - if is_integer_type(jsonschema_def): - return "integer" - - return None - - -def meltano_yaml_str( - plugin_name: str, - capabilities: list[str], - config_jsonschema: dict, -) -> str: - """Returns a Meltano plugin definition as a yaml string. - - Args: - plugin_name: Name of the plugin. - capabilities: List of capabilities. - config_jsonschema: JSON Schema of the expected config. - - Returns: - A string representing the Meltano plugin Yaml definition. - """ - capabilities_str: str = "\n".join( - [f" - {capability}" for capability in capabilities] - ) - settings_str: str = "\n".join( - [ - f"""- name: {setting_name} - label: {setting_name.replace("_", " ").title()} - kind: {_to_meltano_kind(property_node)} - description: {property_node.get("description", 'null')}""" - for setting_name, property_node in config_jsonschema["properties"].items() - ] - ) - required_settings = [ - setting_name - for setting_name, type_dict in config_jsonschema["properties"].items() - if setting_name in config_jsonschema.get("required", []) - or type_dict.get("required", False) - ] - settings_group_validation_str = " - - " + "\n - ".join(required_settings) - - return f"""name: {plugin_name} -namespace: {plugin_name.replace('-', '_')} - -## The following could not be auto-detected: -# maintenance_status: # -# repo: # -# variant: # -# label: # -# description: # -# pip_url: # -# domain_url: # -# logo_url: # -# keywords: [] # - -capabilities: -{capabilities_str} -settings_group_validation: -{settings_group_validation_str} -settings: -{settings_str} -""" diff --git a/tests/core/test_meltano_helpers.py b/tests/core/test_meltano_helpers.py deleted file mode 100644 index a0df899f8..000000000 --- a/tests/core/test_meltano_helpers.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Test sample sync.""" - -from __future__ import annotations - -import pytest - -from singer_sdk.helpers._meltano import _to_meltano_kind, meltano_yaml_str -from singer_sdk.typing import ( - ArrayType, - BooleanType, - DateTimeType, - DateType, - DurationType, - EmailType, - HostnameType, - IntegerType, - IPv4Type, - IPv6Type, - JSONPointerType, - NumberType, - ObjectType, - PropertiesList, - Property, - RegexType, - RelativeJSONPointerType, - StringType, - TimeType, - URIReferenceType, - URITemplateType, - URIType, - UUIDType, -) - - -@pytest.mark.parametrize( - "plugin_name,capabilities,config_jsonschema,expected_yaml", - [ - ( - "tap-from-source", - ["about", "stream-maps"], - PropertiesList( - Property( - "username", - StringType, - required=True, - description="The username to connect with.", - ), - Property( - "password", - StringType, - required=True, - secret=True, - description="The user's password.", - ), - Property("start_date", DateType), - ).to_dict(), - """name: tap-from-source -namespace: tap_from_source - -## The following could not be auto-detected: -# maintenance_status: # -# repo: # -# variant: # -# label: # -# description: # -# pip_url: # -# domain_url: # -# logo_url: # -# keywords: [] # - -capabilities: - - about - - stream-maps -settings_group_validation: - - - username - - password -settings: -- name: username - label: Username - kind: string - description: The username to connect with. -- name: password - label: Password - kind: password - description: The user's password. -- name: start_date - label: Start Date - kind: date_iso8601 - description: null -""", - ) - ], -) -def test_meltano_yml_creation( - plugin_name: str, - capabilities: list[str], - config_jsonschema: dict, - expected_yaml: str, -): - assert expected_yaml == meltano_yaml_str( - plugin_name, capabilities, config_jsonschema - ) - - -@pytest.mark.parametrize( - "type_dict,expected_kindstr", - [ - # Handled Types: - (StringType.type_dict, "string"), - (DateTimeType.type_dict, "date_iso8601"), - (DateType.type_dict, "date_iso8601"), - (BooleanType.type_dict, "boolean"), - (IntegerType.type_dict, "integer"), - ( - DurationType.type_dict, - "string", - ), - # Treat as strings: - (TimeType.type_dict, "string"), - (EmailType.type_dict, "string"), - (HostnameType.type_dict, "string"), - (IPv4Type.type_dict, "string"), - (IPv6Type.type_dict, "string"), - (UUIDType.type_dict, "string"), - (URIType.type_dict, "string"), - (URIReferenceType.type_dict, "string"), - (URITemplateType.type_dict, "string"), - (JSONPointerType.type_dict, "string"), - (RelativeJSONPointerType.type_dict, "string"), - (RegexType.type_dict, "string"), - # No handling and no compatible default: - (NumberType.type_dict, None), - ], -) -def test_meltano_type_to_kind(type_dict: dict, expected_kindstr: str | None) -> None: - assert _to_meltano_kind(type_dict) == expected_kindstr From 2b83266990d3dd2504f665ebc7ed3eb8d1757060 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 16:40:22 -0700 Subject: [PATCH 12/20] chore: revert --about updates --- singer_sdk/plugin_base.py | 120 +++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 68 deletions(-) diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 152ae14d9..6e1908b72 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -28,7 +28,6 @@ from singer_sdk.exceptions import ConfigValidationError from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._compat import metadata -from singer_sdk.helpers._meltano import meltano_yaml_str from singer_sdk.helpers._secrets import SecretString, is_common_secret_key from singer_sdk.helpers._util import read_json_file from singer_sdk.helpers.capabilities import ( @@ -342,79 +341,64 @@ def print_about(cls: Type["PluginBase"], format: Optional[str] = None) -> None: if format == "json": print(json.dumps(info, indent=2, default=str)) - return - if format == "markdown": - cls._print_about_markdown(info) - return + elif format == "markdown": + max_setting_len = cast( + int, max(len(k) for k in info["settings"]["properties"].keys()) + ) - if format == "meltano": - print(meltano_yaml_str(cls.name, cls.capabilities, cls.config_jsonschema)) - return + # Set table base for markdown + table_base = ( + f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" + f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" + ) - formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) - print(formatted) + # Empty list for string parts + md_list = [] + # Get required settings for table + required_settings = info["settings"].get("required", []) - @classmethod - def _print_about_markdown(cls: Type["PluginBase"], info: dict) -> None: - """Print about info as markdown. - - Args: - info: The collected metadata for the class. - """ - max_setting_len = cast( - int, max(len(k) for k in info["settings"]["properties"].keys()) - ) - - # Set table base for markdown - table_base = ( - f"| {'Setting':{max_setting_len}}| Required | Default | Description |\n" - f"|:{'-' * max_setting_len}|:--------:|:-------:|:------------|\n" - ) - - # Empty list for string parts - md_list = [] - # Get required settings for table - required_settings = info["settings"].get("required", []) - - # Iterate over Dict to set md - md_list.append( - f"# `{info['name']}`\n\n" - f"{info['description']}\n\n" - f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" - ) - for key, value in info.items(): - - if key == "capabilities": - capabilities = f"## {key.title()}\n\n" - capabilities += "\n".join([f"* `{v}`" for v in value]) - capabilities += "\n\n" - md_list.append(capabilities) - - if key == "settings": - setting = f"## {key.title()}\n\n" - for k, v in info["settings"].get("properties", {}).items(): - md_description = v.get("description", "").replace("\n", "
") - table_base += ( - f"| {k}{' ' * (max_setting_len - len(k))}" - f"| {'True' if k in required_settings else 'False':8} | " - f"{v.get('default', 'None'):7} | " - f"{md_description:11} |\n" - ) - setting += table_base - setting += ( - "\n" - + "\n".join( - [ - "A full list of supported settings and capabilities " - f"is available by running: `{info['name']} --about`" - ] + # Iterate over Dict to set md + md_list.append( + f"# `{info['name']}`\n\n" + f"{info['description']}\n\n" + f"Built with the [Meltano Singer SDK](https://sdk.meltano.com).\n\n" + ) + for key, value in info.items(): + + if key == "capabilities": + capabilities = f"## {key.title()}\n\n" + capabilities += "\n".join([f"* `{v}`" for v in value]) + capabilities += "\n\n" + md_list.append(capabilities) + + if key == "settings": + setting = f"## {key.title()}\n\n" + for k, v in info["settings"].get("properties", {}).items(): + md_description = v.get("description", "").replace("\n", "
") + table_base += ( + f"| {k}{' ' * (max_setting_len - len(k))}" + f"| {'True' if k in required_settings else 'False':8} | " + f"{v.get('default', 'None'):7} | " + f"{md_description:11} |\n" + ) + setting += table_base + setting += ( + "\n" + + "\n".join( + [ + "A full list of supported settings and capabilities " + f"is available by running: `{info['name']} --about`" + ] + ) + + "\n" ) - + "\n" - ) - md_list.append(setting) + md_list.append(setting) - print("".join(md_list)) + print("".join(md_list)) + else: + formatted = "\n".join([f"{k.title()}: {v}" for k, v in info.items()]) + print(formatted) @classproperty def cli(cls) -> Callable: From 0160f01a49c1d3717533d08470fb50c7d53ed01e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 17:01:15 -0700 Subject: [PATCH 13/20] use constants for annotation keys --- singer_sdk/helpers/_typing.py | 9 ++++++--- singer_sdk/typing.py | 11 ++++++++--- tests/core/test_jsonschema_helpers.py | 8 ++++++-- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index d7463040b..a2a1fec1a 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -11,6 +11,8 @@ _MAX_TIMESTAMP = "9999-12-31 23:59:59.999999" _MAX_TIME = "23:59:59.999999" +JSONSCHEMA_ANNOTATION_SECRET = "secret" +JSONSCHEMA_ANNOTATION_WRITEONLY = "writeOnly" class DatetimeErrorTreatmentEnum(Enum): @@ -57,7 +59,7 @@ def append_type(type_dict: dict, new_type: str) -> dict: def is_secret_type(type_dict: dict) -> bool: """Return True if JSON Schema type definition appears to be a secret. - Will return true if either `writeOnly` or `sensitive` are true on this type + Will return true if either `writeOnly` or `secret` are true on this type or any of the type's subproperties. Args: @@ -74,8 +76,9 @@ def is_secret_type(type_dict: dict) -> bool: "Could not detect type from empty type_dict. " "Did you forget to define a property in the stream schema?" ) - - if type_dict.get("writeOnly") or type_dict.get("sensitive"): + if type_dict.get(JSONSCHEMA_ANNOTATION_WRITEONLY) or type_dict.get( + JSONSCHEMA_ANNOTATION_SECRET + ): return True if "properties" in type_dict: diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index c43122fb1..d39072ac3 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -48,7 +48,12 @@ from jsonschema import validators from singer_sdk.helpers._classproperty import classproperty -from singer_sdk.helpers._typing import append_type, get_datelike_property_type +from singer_sdk.helpers._typing import ( + JSONSCHEMA_ANNOTATION_SECRET, + JSONSCHEMA_ANNOTATION_WRITEONLY, + append_type, + get_datelike_property_type, +) if sys.version_info >= (3, 10): from typing import TypeAlias @@ -414,8 +419,8 @@ def to_dict(self) -> dict: if self.secret: type_dict.update( { - "secret": True, - "writeOnly": True, + JSONSCHEMA_ANNOTATION_SECRET: True, + JSONSCHEMA_ANNOTATION_WRITEONLY: True, } ) return {self.name: type_dict} diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index fc9328433..94cddb895 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -5,6 +5,10 @@ import pytest +from singer_sdk.helpers._typing import ( + JSONSCHEMA_ANNOTATION_SECRET, + JSONSCHEMA_ANNOTATION_WRITEONLY, +) from singer_sdk.streams.core import Stream from singer_sdk.tap_base import Tap from singer_sdk.typing import ( @@ -269,8 +273,8 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): { "my_prop3": { "type": ["string", "null"], - "secret": True, - "writeOnly": True, + JSONSCHEMA_ANNOTATION_SECRET: True, + JSONSCHEMA_ANNOTATION_WRITEONLY: True, } }, ), From d9647bd412db8db7b2c91aeb49b7955576ed34b5 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 20 Oct 2022 17:06:37 -0700 Subject: [PATCH 14/20] chore: bump validator to Draft7 --- singer_sdk/plugin_base.py | 4 ++-- singer_sdk/sinks/core.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 6e1908b72..0d4ba30db 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -21,7 +21,7 @@ ) import click -from jsonschema import Draft4Validator, SchemaError, ValidationError +from jsonschema import Draft7Validator, SchemaError, ValidationError from singer_sdk import metrics from singer_sdk.configuration._dict_config import parse_environment_config @@ -42,7 +42,7 @@ SDK_PACKAGE_NAME = "singer_sdk" -JSONSchemaValidator = extend_validator_with_defaults(Draft4Validator) +JSONSchemaValidator = extend_validator_with_defaults(Draft7Validator) class PluginBase(metaclass=abc.ABCMeta): diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index 186ca28b2..aafa864c2 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -13,7 +13,7 @@ from typing import IO, Any, Mapping, Sequence from dateutil import parser -from jsonschema import Draft4Validator, FormatChecker +from jsonschema import Draft7Validator, FormatChecker from singer_sdk.helpers._batch import ( BaseBatchFileEncoding, @@ -29,7 +29,7 @@ ) from singer_sdk.plugin_base import PluginBase -JSONSchemaValidator = Draft4Validator +JSONSchemaValidator = Draft7Validator class Sink(metaclass=abc.ABCMeta): @@ -80,7 +80,7 @@ def __init__( self._batch_records_read: int = 0 self._batch_dupe_records_merged: int = 0 - self._validator = Draft4Validator(schema, format_checker=FormatChecker()) + self._validator = Draft7Validator(schema, format_checker=FormatChecker()) def _get_context(self, record: dict) -> dict: """Return an empty dictionary by default. From 40041ed1fb8d303464c04d3de844d81e1dc191f0 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 21 Oct 2022 14:33:05 -0700 Subject: [PATCH 15/20] chore: add testing for is_secret_type --- singer_sdk/helpers/_typing.py | 5 ----- tests/core/test_jsonschema_helpers.py | 13 +++++++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index a2a1fec1a..3aa39180e 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -71,11 +71,6 @@ def is_secret_type(type_dict: dict) -> bool: Returns: True if we detect any sensitive property nodes. """ - if not type_dict: - raise ValueError( - "Could not detect type from empty type_dict. " - "Did you forget to define a property in the stream schema?" - ) if type_dict.get(JSONSCHEMA_ANNOTATION_WRITEONLY) or type_dict.get( JSONSCHEMA_ANNOTATION_SECRET ): diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 94cddb895..aab83eb7e 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -8,6 +8,7 @@ from singer_sdk.helpers._typing import ( JSONSCHEMA_ANNOTATION_SECRET, JSONSCHEMA_ANNOTATION_WRITEONLY, + is_secret_type, ) from singer_sdk.streams.core import Stream from singer_sdk.tap_base import Tap @@ -258,15 +259,17 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): @pytest.mark.parametrize( - "property_obj,expected_jsonschema", + "property_obj,expected_jsonschema,is_secret", [ ( Property("my_prop1", StringType, required=True), {"my_prop1": {"type": ["string"]}}, + False, ), ( Property("my_prop2", StringType, required=False), {"my_prop2": {"type": ["string", "null"]}}, + False, ), ( Property("my_prop3", StringType, secret=True), @@ -277,6 +280,7 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): JSONSCHEMA_ANNOTATION_WRITEONLY: True, } }, + False, ), ( Property("my_prop4", StringType, description="This is a property."), @@ -286,6 +290,7 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): "type": ["string", "null"], } }, + False, ), ( Property("my_prop5", StringType, default="some_val"), @@ -295,11 +300,15 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): "type": ["string", "null"], } }, + False, ), ], ) -def test_property_creation(property_obj: Property, expected_jsonschema: dict) -> None: +def test_property_creation( + property_obj: Property, expected_jsonschema: dict, is_secret: bool +) -> None: assert property_obj.to_dict() == expected_jsonschema + assert is_secret_type(property_obj.to_dict()) is is_secret def test_wrapped_type_dict(): From 839b165a674bfb1c01cc6ff18ac5db31a27bca66 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 21 Oct 2022 15:32:19 -0700 Subject: [PATCH 16/20] chore: add tests --- singer_sdk/helpers/_typing.py | 6 ---- tests/core/test_jsonschema_helpers.py | 49 +++++++++++++++++++++------ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 3aa39180e..853b6229b 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -132,12 +132,6 @@ def is_date_or_datetime_type(type_dict: dict) -> bool: Returns: True if date or date-time, else False. """ - if not type_dict: - raise ValueError( - "Could not detect type from empty type_dict. " - "Did you forget to define a property in the stream schema?" - ) - if "anyOf" in type_dict: for type_dict in type_dict["anyOf"]: if is_date_or_datetime_type(type_dict): diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index aab83eb7e..b7b0a4201 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -1,14 +1,21 @@ """Test sample sync.""" +from __future__ import annotations + import re -from typing import List +from typing import Callable, List import pytest from singer_sdk.helpers._typing import ( JSONSCHEMA_ANNOTATION_SECRET, JSONSCHEMA_ANNOTATION_WRITEONLY, + is_array_type, + is_boolean_type, + is_date_or_datetime_type, + is_datetime_type, is_secret_type, + is_string_type, ) from singer_sdk.streams.core import Stream from singer_sdk.tap_base import Tap @@ -40,6 +47,15 @@ UUIDType, ) +TYPE_FN_CHECKS: set[Callable] = { + is_array_type, + is_boolean_type, + is_date_or_datetime_type, + is_datetime_type, + is_secret_type, + is_string_type, +} + class ConfigTestTap(Tap): """Test tap class.""" @@ -259,17 +275,17 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): @pytest.mark.parametrize( - "property_obj,expected_jsonschema,is_secret", + "property_obj,expected_jsonschema,type_fn_checks_true", [ ( Property("my_prop1", StringType, required=True), {"my_prop1": {"type": ["string"]}}, - False, + {is_string_type}, ), ( Property("my_prop2", StringType, required=False), {"my_prop2": {"type": ["string", "null"]}}, - False, + {is_string_type}, ), ( Property("my_prop3", StringType, secret=True), @@ -280,7 +296,7 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): JSONSCHEMA_ANNOTATION_WRITEONLY: True, } }, - False, + {is_secret_type, is_string_type}, ), ( Property("my_prop4", StringType, description="This is a property."), @@ -290,7 +306,7 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): "type": ["string", "null"], } }, - False, + {is_string_type}, ), ( Property("my_prop5", StringType, default="some_val"), @@ -300,15 +316,28 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): "type": ["string", "null"], } }, - False, + {is_string_type}, ), ], ) def test_property_creation( - property_obj: Property, expected_jsonschema: dict, is_secret: bool + property_obj: Property, + expected_jsonschema: dict, + type_fn_checks_true: set[Callable], ) -> None: - assert property_obj.to_dict() == expected_jsonschema - assert is_secret_type(property_obj.to_dict()) is is_secret + property_dict = property_obj.to_dict() + assert property_dict == expected_jsonschema + for check_fn in TYPE_FN_CHECKS: + property_name = list(property_dict.keys())[0] + property_node = property_dict[property_name] + if check_fn in type_fn_checks_true: + assert ( + check_fn(property_node) is True + ), f"{check_fn.__name__} was not True for {repr(property_dict)}" + else: + assert ( + check_fn(property_node) is False + ), f"{check_fn.__name__} was not False for {repr(property_dict)}" def test_wrapped_type_dict(): From a82851ea74dc5a1ee6e98ef180e755336d3efa71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Oct 2022 22:40:04 +0000 Subject: [PATCH 17/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/core/test_jsonschema_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index b7b0a4201..d48fa6799 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -68,7 +68,7 @@ class ConfigTestTap(Tap): Property("batch_size", IntegerType, default=-1), ).to_dict() - def discover_streams(self) -> List[Stream]: + def discover_streams(self) -> list[Stream]: return [] @@ -468,7 +468,7 @@ def test_array_type(): "requried, duplicates, additional properties", ], ) -def test_object_type(properties: List[Property], addtional_properties: JSONTypeHelper): +def test_object_type(properties: list[Property], addtional_properties: JSONTypeHelper): merged_property_schemas = { name: schema for p in properties for name, schema in p.to_dict().items() } From c2b64d7d3fd5ab9606a2f9fb7f51d4c1bc7cd4af Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 21 Oct 2022 15:41:47 -0700 Subject: [PATCH 18/20] chore: more tests --- tests/core/test_jsonschema_helpers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index b7b0a4201..869acf07b 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -15,6 +15,7 @@ is_date_or_datetime_type, is_datetime_type, is_secret_type, + is_string_array_type, is_string_type, ) from singer_sdk.streams.core import Stream @@ -53,6 +54,7 @@ is_date_or_datetime_type, is_datetime_type, is_secret_type, + is_string_array_type, is_string_type, } @@ -318,6 +320,16 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): }, {is_string_type}, ), + ( + Property("my_prop6", ArrayType(StringType)), + { + "my_prop6": { + "type": ["array", "null"], + "items": {"type": ["string"]}, + } + }, + {is_array_type, is_string_array_type}, + ), ], ) def test_property_creation( From 372fc340163be8c8da703aacf1029b72c917377d Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 21 Oct 2022 15:53:59 -0700 Subject: [PATCH 19/20] docs: add info to FAQ --- docs/faq.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index e0b18c5e0..953cef072 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -11,7 +11,11 @@ However, if you're using an IDE such as VSCode, you should be able to set up the Ensure your interpreter is set to poetry if you've followed the [Dev Guide](./dev_guide.md). Checkout this [gif](https://visualstudiomagazine.com/articles/2021/04/20/~/media/ECG/visualstudiomagazine/Images/2021/04/poetry.ashx) for how to change your interpreter. -## I'm having trouble getting the base class to __init__. +### Handling credentials and other secrets in config + +As of SDK version `0.13.0`, developers can use the `secret=True` indication in the `Property` class constructor to flag secrets such as API tokens and passwords. We recommend all developers use this option where applicable so that orchestrators may consider this designation when determining how to store the user's provided config. + +## I'm having trouble getting the base class to **init**. Ensure you're using the `super()` method to inherit methods from the base class. From 8eb0026fe126edce9a252693a210733ac6d9c8b4 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 21 Oct 2022 16:03:38 -0700 Subject: [PATCH 20/20] chore: add test for integer type --- tests/core/test_jsonschema_helpers.py | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index edcd4d06b..9b731af16 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -14,6 +14,8 @@ is_boolean_type, is_date_or_datetime_type, is_datetime_type, + is_integer_type, + is_object_type, is_secret_type, is_string_array_type, is_string_type, @@ -53,6 +55,7 @@ is_boolean_type, is_date_or_datetime_type, is_datetime_type, + is_integer_type, is_secret_type, is_string_array_type, is_string_type, @@ -330,6 +333,38 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): }, {is_array_type, is_string_array_type}, ), + ( + Property( + "my_prop7", + ObjectType( + Property("not_a_secret", StringType), + Property("is_a_secret", StringType, secret=True), + ), + ), + { + "my_prop7": { + "type": ["object", "null"], + "properties": { + "not_a_secret": {"type": ["string", "null"]}, + "is_a_secret": { + "type": ["string", "null"], + "secret": True, + "writeOnly": True, + }, + }, + } + }, + {is_object_type, is_secret_type}, + ), + ( + Property("my_prop8", IntegerType), + { + "my_prop8": { + "type": ["integer", "null"], + } + }, + {is_integer_type}, + ), ], ) def test_property_creation(