feat: add secrets support for tap and target config, via Property(..., secret=True)

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -38,6 +38,7 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
             "auth_token",
             th.StringType,
             required=True,
+            secret=True,  # Flag config as protected.
             description="The token to authenticate against the API service"
         ),
         th.Property(

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py

@@ -21,6 +21,7 @@ class Target{{ cookiecutter.destination_name }}({{ target_class }}):
         th.Property(
             "sqlalchemy_url",
             th.StringType,
+            secret=True,  # Flag config as protected.
             description="SQLAlchemy connection string",
         ),
         {%- else %}
@@ -34,6 +35,12 @@ class Target{{ cookiecutter.destination_name }}({{ target_class }}):
             th.StringType,
             description="The scheme with which output files will be named"
         ),
+        th.Property(
+            "auth_token",
+            th.StringType,
+            secret=True,  # Flag config as protected.
+            description="The path to the target output file"
+        ),
         {%- endif %}
     ).to_dict()
 

samples/sample_tap_gitlab/gitlab_tap.py

@@ -34,7 +34,7 @@ class SampleTapGitlab(Tap):
 
     name: str = "sample-tap-gitlab"
     config_jsonschema = PropertiesList(
-        Property("auth_token", StringType, required=True),
+        Property("auth_token", StringType, required=True, secret=True),
         Property("project_ids", ArrayType(StringType), required=True),
         Property("group_ids", ArrayType(StringType), required=True),
         Property("start_date", DateTimeType, required=True),

samples/sample_tap_google_analytics/ga_tap.py

@@ -24,7 +24,7 @@ class SampleTapGoogleAnalytics(Tap):
     config_jsonschema = PropertiesList(
         Property("view_id", StringType(), required=True),
         Property("client_email", StringType(), required=True),
-        Property("private_key", StringType(), required=True),
+        Property("private_key", StringType(), required=True, secret=True),
     ).to_dict()
 
     def discover_streams(self) -> List[SampleGoogleAnalyticsStream]:

singer_sdk/helpers/_typing.py

@@ -11,6 +11,8 @@
 
 _MAX_TIMESTAMP = "9999-12-31 23:59:59.999999"
 _MAX_TIME = "23:59:59.999999"
+JSONSCHEMA_ANNOTATION_SECRET = "secret"
+JSONSCHEMA_ANNOTATION_WRITEONLY = "writeOnly"
 
 
 class DatetimeErrorTreatmentEnum(Enum):
@@ -54,6 +56,41 @@ def append_type(type_dict: dict, new_type: str) -> dict:
     )
 
 
+def is_secret_type(type_dict: dict) -> bool:
+    """Return True if JSON Schema type definition appears to be a secret.
+
+    Will return true if either `writeOnly` or `secret` are true on this type
+    or any of the type's subproperties.
+
+    Args:
+        type_dict: The JSON Schema type to check.
+
+    Raises:
+        ValueError: If type_dict is None or empty.
+
+    Returns:
+        True if we detect any sensitive property nodes.
+    """
+    if not type_dict:
+        raise ValueError(
+            "Could not detect type from empty type_dict. "
+            "Did you forget to define a property in the stream schema?"
+        )
+    if type_dict.get(JSONSCHEMA_ANNOTATION_WRITEONLY) or type_dict.get(
+        JSONSCHEMA_ANNOTATION_SECRET
+    ):
+        return True
+
+    if "properties" in type_dict:
+        # Recursively check subproperties and return True if any child is secret.
+        return any(
+            is_secret_type(child_type_dict)
+            for child_type_dict in type_dict["properties"].values()
+        )
+
+    return False
+
+
 def is_object_type(property_schema: dict) -> Optional[bool]:
     """Return true if the JSON Schema type is an object or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -86,6 +123,40 @@ def is_datetime_type(type_dict: dict) -> bool:
     )
 
 
+def is_date_or_datetime_type(type_dict: dict) -> bool:
+    """Return True if JSON Schema type definition is a 'date'/'date-time' type.
+
+    Also returns True if type is nested within an 'anyOf' type Array.
+
+    Args:
+        type_dict: The JSON Schema definition.
+
+    Raises:
+        ValueError: If type is empty or null.
+
+    Returns:
+        True if date or date-time, else False.
+    """
+    if not type_dict:
+        raise ValueError(
+            "Could not detect type from empty type_dict. "
+            "Did you forget to define a property in the stream schema?"
+        )
+
+    if "anyOf" in type_dict:
+        for type_dict in type_dict["anyOf"]:
+            if is_date_or_datetime_type(type_dict):
+                return True
+        return False
+
+    if "type" in type_dict:
+        return type_dict.get("format") in {"date", "date-time"}
+
+    raise ValueError(
+        f"Could not detect type of replication key using schema '{type_dict}'"
+    )
+
+
 def get_datelike_property_type(property_schema: Dict) -> Optional[str]:
     """Return one of 'date-time', 'time', or 'date' if property is date-like.
 
@@ -152,6 +223,23 @@ def is_string_array_type(type_dict: dict) -> bool:
     return "array" in type_dict["type"] and bool(is_string_type(type_dict["items"]))
 
 
+def is_array_type(type_dict: dict) -> bool:
+    """Return True if JSON Schema type definition is a string array."""
+    if not type_dict:
+        raise ValueError(
+            "Could not detect type from empty type_dict. "
+            "Did you forget to define a property in the stream schema?"
+        )
+
+    if "anyOf" in type_dict:
+        return any([is_array_type(t) for t in type_dict["anyOf"]])
+
+    if "type" not in type_dict:
+        raise ValueError(f"Could not detect type from schema '{type_dict}'")
+
+    return "array" in type_dict["type"]
+
+
 def is_boolean_type(property_schema: dict) -> Optional[bool]:
     """Return true if the JSON Schema type is a boolean or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:
@@ -162,6 +250,16 @@ def is_boolean_type(property_schema: dict) -> Optional[bool]:
     return False
 
 
+def is_integer_type(property_schema: dict) -> Optional[bool]:
+    """Return true if the JSON Schema type is a boolean or None if detection fails."""
+    if "anyOf" not in property_schema and "type" not in property_schema:
+        return None  # Could not detect data type
+    for property_type in property_schema.get("anyOf", [property_schema.get("type")]):
+        if "integer" in property_type or property_type == "integer":
+            return True
+    return False
+
+
 def is_string_type(property_schema: dict) -> Optional[bool]:
     """Return true if the JSON Schema type is a boolean or None if detection fails."""
     if "anyOf" not in property_schema and "type" not in property_schema:

singer_sdk/plugin_base.py

@@ -21,7 +21,7 @@
 )
 
 import click
-from jsonschema import Draft4Validator, SchemaError, ValidationError
+from jsonschema import Draft7Validator, SchemaError, ValidationError
 
 from singer_sdk import metrics
 from singer_sdk.configuration._dict_config import parse_environment_config
@@ -42,7 +42,7 @@
 SDK_PACKAGE_NAME = "singer_sdk"
 
 
-JSONSchemaValidator = extend_validator_with_defaults(Draft4Validator)
+JSONSchemaValidator = extend_validator_with_defaults(Draft7Validator)
 
 
 class PluginBase(metaclass=abc.ABCMeta):

singer_sdk/sinks/core.py

@@ -13,7 +13,7 @@
 from typing import IO, Any, Mapping, Sequence
 
 from dateutil import parser
-from jsonschema import Draft4Validator, FormatChecker
+from jsonschema import Draft7Validator, FormatChecker
 
 from singer_sdk.helpers._batch import (
     BaseBatchFileEncoding,
@@ -29,7 +29,7 @@
 )
 from singer_sdk.plugin_base import PluginBase
 
-JSONSchemaValidator = Draft4Validator
+JSONSchemaValidator = Draft7Validator
 
 
 class Sink(metaclass=abc.ABCMeta):
@@ -80,7 +80,7 @@ def __init__(
         self._batch_records_read: int = 0
         self._batch_dupe_records_merged: int = 0
 
-        self._validator = Draft4Validator(schema, format_checker=FormatChecker())
+        self._validator = Draft7Validator(schema, format_checker=FormatChecker())
 
     def _get_context(self, record: dict) -> dict:
         """Return an empty dictionary by default.

singer_sdk/typing.py

@@ -48,7 +48,12 @@
 from jsonschema import validators
 
 from singer_sdk.helpers._classproperty import classproperty
-from singer_sdk.helpers._typing import append_type, get_datelike_property_type
+from singer_sdk.helpers._typing import (
+    JSONSCHEMA_ANNOTATION_SECRET,
+    JSONSCHEMA_ANNOTATION_WRITEONLY,
+    append_type,
+    get_datelike_property_type,
+)
 
 if sys.version_info >= (3, 10):
     from typing import TypeAlias
@@ -352,21 +357,30 @@ def __init__(
         required: bool = False,
         default: _JsonValue = None,
         description: str = None,
+        secret: bool = False,
     ) -> None:
         """Initialize Property object.
 
+        Note: Properties containing secrets should be specified with `secret=True`.
+        Doing so will add the annotation `writeOnly=True`, in accordance with JSON
+        Schema Draft 7 and later, and `secret=True` as an additional hint to readers.
+
+        More info: https://json-schema.org/draft-07/json-schema-release-notes.html
+
         Args:
             name: Property name.
             wrapped: JSON Schema type of the property.
             required: Whether this is a required property.
             default: Default value in the JSON Schema.
             description: Long-text property description.
+            secret: True if this is a credential or other secret.
         """
         self.name = name
         self.wrapped = wrapped
         self.optional = not required
         self.default = default
         self.description = description
+        self.secret = secret
 
     @property
     def type_dict(self) -> dict:  # type: ignore  # OK: @classproperty vs @property
@@ -402,6 +416,13 @@ def to_dict(self) -> dict:
             type_dict.update({"default": self.default})
         if self.description:
             type_dict.update({"description": self.description})
+        if self.secret:
+            type_dict.update(
+                {
+                    JSONSCHEMA_ANNOTATION_SECRET: True,
+                    JSONSCHEMA_ANNOTATION_WRITEONLY: True,
+                }
+            )
         return {self.name: type_dict}
 
 

tests/core/test_jsonschema_helpers.py

@@ -5,6 +5,10 @@
 
 import pytest
 
+from singer_sdk.helpers._typing import (
+    JSONSCHEMA_ANNOTATION_SECRET,
+    JSONSCHEMA_ANNOTATION_WRITEONLY,
+)
 from singer_sdk.streams.core import Stream
 from singer_sdk.tap_base import Tap
 from singer_sdk.typing import (
@@ -43,7 +47,7 @@ class ConfigTestTap(Tap):
     config_jsonschema = PropertiesList(
         Property("host", StringType, required=True),
         Property("username", StringType, required=True),
-        Property("password", StringType, required=True),
+        Property("password", StringType, required=True, secret=True),
         Property("batch_size", IntegerType, default=-1),
     ).to_dict()
 
@@ -253,6 +257,51 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict):
     assert json_type.type_dict == expected_json_schema
 
 
+@pytest.mark.parametrize(
+    "property_obj,expected_jsonschema",
+    [
+        (
+            Property("my_prop1", StringType, required=True),
+            {"my_prop1": {"type": ["string"]}},
+        ),
+        (
+            Property("my_prop2", StringType, required=False),
+            {"my_prop2": {"type": ["string", "null"]}},
+        ),
+        (
+            Property("my_prop3", StringType, secret=True),
+            {
+                "my_prop3": {
+                    "type": ["string", "null"],
+                    JSONSCHEMA_ANNOTATION_SECRET: True,
+                    JSONSCHEMA_ANNOTATION_WRITEONLY: True,
+                }
+            },
+        ),
+        (
+            Property("my_prop4", StringType, description="This is a property."),
+            {
+                "my_prop4": {
+                    "description": "This is a property.",
+                    "type": ["string", "null"],
+                }
+            },
+        ),
+        (
+            Property("my_prop5", StringType, default="some_val"),
+            {
+                "my_prop5": {
+                    "default": "some_val",
+                    "type": ["string", "null"],
+                }
+            },
+        ),
+    ],
+)
+def test_property_creation(property_obj: Property, expected_jsonschema: dict) -> None:
+    assert property_obj.to_dict() == expected_jsonschema
+
+
 def test_wrapped_type_dict():
     with pytest.raises(
         ValueError,