Tdl 20778 code refactoring (#45)

* added pre-commit and updated gitignore

* updated file structure

* fixed sync.py

* fixed discovery

* fixed streams

* removed pendulum dependancy

* added pagination support to bans stream

*  Fixed unittests as per new code refactoring changes.

* added config param for bans stream page size

* added warning for 400 exception

* added replication method to catalog

* fixed automatic fields issue

* fixed discovery changes

* fixed pylint issue

* minor enhancements

* fixed review comments

* fixed discovery changes

* removed duplicate function

* Tdl 18828 add integration tests (#47)

* add automation fields

* added test for all_fields and automatic fields

* added pagination test

* fix bookmark test

* fixed naming conventions

* fixed assert for automatic fields

Co-authored-by: shantanu73 <[email protected]>

.github/pull_request_template.md

@@ -2,10 +2,10 @@
 (write a short description or paste a link to JIRA)
 
 # Manual QA steps
- - 
- 
+ -
+
 # Risks
- - 
- 
+ -
+
 # Rollback steps
  - revert this branch

.gitignore

@@ -45,20 +45,7 @@ coverage.xml
 *,cover
 .hypothesis/
 
-# Translations
-*.mo
-*.pot
 
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
 
 # Sphinx documentation
 docs/_build/
@@ -72,8 +59,6 @@ target/
 # pyenv
 .python-version
 
-# celery beat schedule file
-celerybeat-schedule
 
 # dotenv
 .env
@@ -92,11 +77,15 @@ ENV/
 ._*
 .DS_Store
 
-# Custom stuff
-env.sh
+state.json
+catalog.json
 config.json
-.autoenv.zsh
 
-rsa-key
-tags
-properties.json
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace

.pre-commit-config.yaml

@@ -0,0 +1,57 @@
+default_stages: [commit]
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-merge-conflict
+      - id: check-docstring-first
+      - id: debug-statements
+      - id: trailing-whitespace
+      - id: check-toml
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: sort-simple-yaml
+      - id: check-json
+      - id: pretty-format-json
+        args: ['--autofix','--no-sort-keys']
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/psf/black
+    rev: 22.8.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        additional_dependencies: [
+          'flake8-print',
+          'flake8-debugger',
+          ]
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.4'
+    hooks:
+    - id: bandit
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.37.3
+    hooks:
+      - id: pyupgrade
+        args: [--py37-plus]
+
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.5.0
+    hooks:
+      - id: docformatter
+        args: [--in-place]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.1
+    hooks:
+      - id: codespell

LICENSE

@@ -617,4 +617,3 @@ Program, unless a warranty or assumption of liability accompanies a
 copy of the Program in return for a fee.
 
                      END OF TERMS AND CONDITIONS
-

README.md

@@ -37,8 +37,8 @@ authorization request), log into your Zendesk Chat / Zopim account, go to
     Settings -> Account -> API -> Add API Client
 
 Once you create the API Client you will receive a client ID and client secret.
-Use these in conjunction with your chose method of performing the OAuth 2
-reqeust to obtain an access token to your (or a third-party) Zendesk Chat /
+Use these in conjunction with your choice method of performing the OAuth 2
+request to obtain an access token to your (or a third-party) Zendesk Chat /
 Zopim account.
 
 3. Create the Config File

config.json.sample

@@ -0,0 +1,4 @@
+{
+    "access_token":"<token_here>",
+    "start_date":"12/01/2010"
+}

pyproject.toml

@@ -0,0 +1,21 @@
+[tool.black]
+line-length = 120
+target-version = ['py37',]
+include = '\.pyi?$'
+
+[flake8]
+profile = "black"
+max-line-length = 120
+exclude = "build,.git,.tox,./tests/.env,tests"
+ignore = "W504,W601,D203"
+
+[tool.pylint]
+max-line-length = 120
+disable = ["R0801",]
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+
+[tool.bandit]
+exclude_dirs = ["tests",".env"]

setup.cfg

@@ -1,2 +1,10 @@
 [metadata]
 description-file = README.md
+
+
+[flake8]
+ignore = W504,W601,D203
+profile = black
+max-line-length = 120
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.git,.tox,./tests/.env,tests
+max-complexity = 10

setup.py

@@ -1,33 +1,24 @@
 #!/usr/bin/env python
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-setup(name="tap-zendesk-chat",
-      version="0.3.2",
-      description="Singer.io tap for extracting data from the Zendesk Chat API",
-      author="Stitch",
-      url="http://singer.io",
-      classifiers=["Programming Language :: Python :: 3 :: Only"],
-      py_modules=["tap_zendesk_chat"],
-      install_requires=[
-          "python-dateutil==2.6.0",  # because of singer-python issue
-          "pendulum==1.2.0",  # because of singer-python issue
-          "singer-python==5.12.1",
-          "requests==2.20.0",
-      ],
-      extras_require={
-          'dev': [
-              'pylint==2.7.4',
-              'ipdb',
-              'nose'
-          ]
-      },
-      entry_points="""
-          [console_scripts]
-          tap-zendesk-chat=tap_zendesk_chat:main
-      """,
-      packages=["tap_zendesk_chat"],
-      package_data = {
-          "schemas": ["tap_zendesk_chat/schemas/*.json"]
-      },
-      include_package_data=True,
+setup(
+    name="tap-zendesk-chat",
+    version="0.3.2",
+    description="Singer.io tap for extracting data from the Zendesk Chat API",
+    author="Stitch",
+    url="https://singer.io",
+    classifiers=["Programming Language :: Python :: 3 :: Only"],
+    py_modules=["tap_zendesk_chat"],
+    install_requires=[
+        "singer-python==5.12.1",
+        "requests==2.20.0",
+    ],
+    extras_require={"dev": ["pylint", "ipdb", "nose"]},
+    entry_points="""
+    [console_scripts]
+    tap-zendesk-chat=tap_zendesk_chat:main
+    """,
+    packages=find_packages(exclude=["tests"]),
+    package_data={"schemas": ["tap_zendesk_chat/schemas/*.json"]},
+    include_package_data=True,
 )

tap_zendesk_chat/__init__.py

@@ -1,134 +1,25 @@
 #!/usr/bin/env python3
-import os
 import singer
-from singer import metrics, utils, metadata
-from singer.catalog import Catalog, CatalogEntry, Schema
-from requests.exceptions import HTTPError
-from . import streams as streams_
+from singer.utils import handle_top_exception, parse_args
+
 from .context import Context
-from .http import Client
+from .discover import discover
+from .sync import sync
 
 REQUIRED_CONFIG_KEYS = ["start_date", "access_token"]
 LOGGER = singer.get_logger()
 
 
-def get_abs_path(path):
-    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
-
-
-def load_schema(tap_stream_id):
-    path = "schemas/{}.json".format(tap_stream_id)
-    schema = utils.load_json(get_abs_path(path))
-    dependencies = schema.pop("tap_schema_dependencies", [])
-    refs = {}
-    for sub_stream_id in dependencies:
-        refs[sub_stream_id] = load_schema(sub_stream_id)
-    if refs:
-        singer.resolve_schema_references(schema, refs)
-    return schema
-
-
-def ensure_credentials_are_authorized(client):
-    # The request will throw an exception if the credentials are not authorized
-    client.request(streams_.DEPARTMENTS.tap_stream_id)
-
-
-def is_account_endpoint_authorized(client):
-    # The account endpoint is restricted to zopim accounts, meaning integrated
-    # Zendesk accounts will get a 403 for this endpoint.
-    try:
-        client.request(streams_.ACCOUNT.tap_stream_id)
-    except HTTPError as e:
-        if e.response.status_code == 403:
-            LOGGER.info(
-                "Ignoring 403 from account endpoint - this must be an "
-                "integrated Zendesk account. This endpoint will be excluded "
-                "from discovery."
-            )
-            return False
-        else:
-            raise
-    return True
-
-
-def discover(config):
-    client = Client(config)
-    ensure_credentials_are_authorized(client)
-    include_account_stream = is_account_endpoint_authorized(client)
-    catalog = Catalog([])
-    for stream in streams_.all_streams:
-        if (not include_account_stream
-            and stream.tap_stream_id == streams_.ACCOUNT.tap_stream_id):
-            continue
-        raw_schema = load_schema(stream.tap_stream_id)
-        mdata = build_metadata(raw_schema, stream)
-        schema = Schema.from_dict(raw_schema)
-        catalog.streams.append(CatalogEntry(
-            stream=stream.tap_stream_id,
-            tap_stream_id=stream.tap_stream_id,
-            key_properties=stream.pk_fields,
-            schema=schema,
-            metadata=metadata.to_list(mdata)
-        ))
-    return catalog
-
-def build_metadata(raw_schema, stream):
-
-    mdata = metadata.new()
-    metadata.write(mdata, (), 'valid-replication-keys', list(stream.replication_key))
-    metadata.write(mdata, (), 'table-key-properties', list(stream.pk_fields))
-    for prop in raw_schema['properties'].keys():
-        if prop in stream.replication_key or prop in stream.pk_fields:
-            metadata.write(mdata, ('properties', prop), 'inclusion', 'automatic')
-        else:
-            metadata.write(mdata, ('properties', prop), 'inclusion', 'available')
-
-    return mdata
-
-
-def output_schema(stream):
-    schema = load_schema(stream.tap_stream_id)
-    singer.write_schema(stream.tap_stream_id, schema, stream.pk_fields)
-
-
-def is_selected(stream):
-    mdata = metadata.to_map(stream.metadata)
-    return metadata.get(mdata, (), 'selected')
-
-def sync(ctx):
-    currently_syncing = ctx.state.get("currently_syncing")
-    start_idx = streams_.all_stream_ids.index(currently_syncing) \
-        if currently_syncing else 0
-    stream_ids_to_sync = [cs.tap_stream_id for cs in ctx.catalog.streams
-                          if is_selected(cs)]
-    streams = [s for s in streams_.all_streams[start_idx:]
-               if s.tap_stream_id in stream_ids_to_sync]
-    for stream in streams:
-        ctx.state["currently_syncing"] = stream.tap_stream_id
-        output_schema(stream)
-        ctx.write_state()
-        stream.sync(ctx)
-    ctx.state["currently_syncing"] = None
-    ctx.write_state()
-
-
-def main_impl():
-    args = utils.parse_args(REQUIRED_CONFIG_KEYS)
+@handle_top_exception(LOGGER)
+def main():
+    """performs sync and discovery."""
+    args = parse_args(REQUIRED_CONFIG_KEYS)
     if args.discover:
         discover(args.config).dump()
-        print()
     else:
-        catalog = Catalog.from_dict(args.properties) \
-            if args.properties else discover(args.config)
-        ctx = Context(args.config, args.state, catalog)
+        ctx = Context(args.config, args.state, args.catalog or discover(args.config))
         sync(ctx)
 
-def main():
-    try:
-        main_impl()
-    except Exception as exc:
-        LOGGER.critical(exc)
-        raise exc
 
 if __name__ == "__main__":
     main()