fix(templates): Tap template: fix style and docstrings, and add test …

…cases for SQL and "Other" sources (#1434)

Co-authored-by: Edgar R. M <[email protected]>

.github/workflows/cookiecutter-e2e.yml

@@ -31,6 +31,8 @@ jobs:
           - { cookiecutter: "tap-template", replay: "tap-rest-custom.json", python-version: "3.9",  os: "ubuntu-latest" }
           - { cookiecutter: "tap-template", replay: "tap-rest-jwt.json", python-version: "3.9",  os: "ubuntu-latest" }
           - { cookiecutter: "tap-template", replay: "tap-rest-oauth2.json", python-version: "3.9",  os: "ubuntu-latest" }
+          - { cookiecutter: "tap-template", replay: "tap-other-custom.json", python-version: "3.9",  os: "ubuntu-latest" }
+          - { cookiecutter: "tap-template", replay: "tap-sql-custom.json", python-version: "3.9",  os: "ubuntu-latest" }
           - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9",  os: "ubuntu-latest" }
 
     steps:

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -41,6 +41,9 @@ isort = "^5.11.5"
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 types-requests = "^2.28.11.12"
 {%- endif %}
+{%- if cookiecutter.stream_type == 'SQL' %}
+sqlalchemy-stubs = "^0.4"
+{%- endif %}
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
@@ -53,6 +56,9 @@ src_paths = "{{cookiecutter.library_name}}"
 [tool.mypy]
 python_version = "3.9"
 warn_unused_configs = true
+{%- if cookiecutter.stream_type == 'SQL' %}
+plugins = "sqlmypy"
+{%- endif %}
 
 [build-system]
 requires = ["poetry-core>=1.0.8"]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -7,7 +7,7 @@
 
 {%- if cookiecutter.stream_type == "SQL" %}
 
-from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
+# from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
 {%- else %}
 
 # TODO: Import your custom stream types here:

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%}

@@ -2,9 +2,7 @@
 
 from __future__ import annotations
 
-import requests
-from pathlib import Path
-from typing import Any, Iterable
+from typing import Iterable
 
 from singer_sdk.streams import Stream
 
@@ -18,6 +16,12 @@ class {{ cookiecutter.source_name }}Stream(Stream):
         The optional `context` argument is used to identify a specific slice of the
         stream if partitioning is required for the stream. Most implementations do not
         require partitioning and should ignore the `context` argument.
+
+        Args:
+            context: Stream partition or context dictionary.
+
+        Raises:
+            NotImplementedError: If the implementation is TODO
         """
         # TODO: Write logic to extract data from the upstream source.
         # records = mysource.getall()

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%}

@@ -4,6 +4,7 @@ This includes {{ cookiecutter.source_name }}Stream and {{ cookiecutter.source_na
 """
 
 from __future__ import annotations
+from typing import Any, Iterable
 
 import sqlalchemy
 
@@ -14,7 +15,14 @@ class {{ cookiecutter.source_name }}Connector(SQLConnector):
     """Connects to the {{ cookiecutter.source_name }} SQL source."""
 
     def get_sqlalchemy_url(cls, config: dict) -> str:
-        """Concatenate a SQLAlchemy URL for use in connecting to the source."""
+        """Concatenate a SQLAlchemy URL for use in connecting to the source.
+
+        Args:
+            config: A dict with connection parameters
+
+        Returns:
+            SQLAlchemy connection string
+        """
         # TODO: Replace this with a valid connection string for your source:
         return (
             f"awsathena+rest://{config['aws_access_key_id']}:"
@@ -25,22 +33,39 @@ class {{ cookiecutter.source_name }}Connector(SQLConnector):
         )
 
     @staticmethod
-    def to_jsonschema_type(sql_type: sqlalchemy.types.TypeEngine) -> dict:
+    def to_jsonschema_type(
+        from_type: str
+        | sqlalchemy.types.TypeEngine
+        | type[sqlalchemy.types.TypeEngine],
+    ) -> dict:
         """Returns a JSON Schema equivalent for the given SQL type.
 
         Developers may optionally add custom logic before calling the default
         implementation inherited from the base class.
+
+        Args:
+            from_type: The SQL type as a string or as a TypeEngine. If a TypeEngine is
+                provided, it may be provided as a class or a specific object instance.
+
+        Returns:
+            A compatible JSON Schema type definition.
         """
         # Optionally, add custom logic before calling the parent SQLConnector method.
         # You may delete this method if overrides are not needed.
-        return SQLConnector.to_jsonschema_type(sql_type)
+        return SQLConnector.to_jsonschema_type(from_type)
 
     @staticmethod
     def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:
         """Returns a JSON Schema equivalent for the given SQL type.
 
         Developers may optionally add custom logic before calling the default
         implementation inherited from the base class.
+
+        Args:
+            jsonschema_type: A dict
+
+        Returns:
+            SQLAlchemy type
         """
         # Optionally, add custom logic before calling the parent SQLConnector method.
         # You may delete this method if overrides are not needed.
@@ -52,7 +77,7 @@ class {{ cookiecutter.source_name }}Stream(SQLStream):
 
     connector_class = {{ cookiecutter.source_name }}Connector
 
-    def get_records(self, partition: Optional[dict]) -> Iterable[Dict[str, Any]]:
+    def get_records(self, partition: dict | None) -> Iterable[dict[str, Any]]:
         """Return a generator of record-type dictionary objects.
 
         Developers may optionally add custom logic before calling the default

e2e-tests/cookiecutters/tap-other-custom.json

@@ -0,0 +1,14 @@
+{
+    "cookiecutter": {
+      "source_name": "AutomaticTestTap",
+      "admin_name": "Automatic Tester",
+      "tap_id": "tap-other-custom",
+      "library_name": "tap_other_custom",
+      "variant": "None (Skip)",
+      "stream_type": "Other",
+      "auth_method": "Custom or N/A",
+      "include_cicd_sample_template": "None (Skip)",
+      "_template": "../tap-template/",
+      "_output_dir": "."
+    }
+}

e2e-tests/cookiecutters/tap-sql-custom.json

@@ -0,0 +1,14 @@
+{
+    "cookiecutter": {
+      "source_name": "AutomaticTestTap",
+      "admin_name": "Automatic Tester",
+      "tap_id": "tap-sql-custom",
+      "library_name": "tap_sql_custom",
+      "variant": "None (Skip)",
+      "stream_type": "SQL",
+      "auth_method": "Custom or N/A",
+      "include_cicd_sample_template": "None (Skip)",
+      "_template": "../tap-template/",
+      "_output_dir": "."
+    }
+}