HDXDSYS-634 Iterable not iterator (#65)

* Function generate_resource_from_iterator is misnamed. Rename to generate_resource_from_iterable and create deprecated version with old name * Use uv instead of pip-tools Use material theme for readthedocs * Update versions * Fix test * Fix styling
OCHA-DAP · Apr 10, 2024 · e4cd53a · e4cd53a
1 parent d2deba6
commit e4cd53a
Show file tree

Hide file tree

Showing 8 changed files with 94 additions and 71 deletions.
diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml
@@ -17,10 +17,11 @@ repos:
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.1
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: v0.1.24
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]
diff --git a/documentation/.readthedocs.yaml b/documentation/.readthedocs.yaml
@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml
diff --git a/documentation/main.md b/documentation/main.md
@@ -52,6 +52,9 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 6.2.7, generate_resource_from_iterator renamed to
+generate_resource_from_iterable with requirement of iterable rather iterator
+
 From 6.2.6, kwargs take preference over environment variables which take
 preference over configuration files
 

diff --git a/documentation/pydoc-markdown.yaml b/documentation/pydoc-markdown.yaml
@@ -11,7 +11,7 @@ renderer:
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python API
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-api"
   markdown:
     source_linker:

diff --git a/pyproject.toml b/pyproject.toml
@@ -9,7 +9,7 @@ build-backend = "hatchling.build"
 [project]
 name = "hdx-python-api"
 description = "HDX Python API for interacting with the Humanitarian Data Exchange"
-authors = [{name = "Michael Rans", email = "[email protected]"}]
+authors = [{name = "Michael Rans"}]
 license = {text = "MIT"}
 keywords = ["HDX", "API", "CKAN", "humanitarian data exchange"]
 classifiers = [
@@ -34,10 +34,10 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "ckanapi>=4.7",
+    "ckanapi>=4.8",
     "defopt>=6.4.0",
     "email_validator",
-    "hdx-python-country>=3.6.9",
+    "hdx-python-country>=3.7.0",
     "hdx-python-utilities>=3.6.7",
     "libhxl>=5.2.1",
     "makefun",

diff --git a/requirements.txt b/requirements.txt
@@ -1,9 +1,5 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
-#
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.6.0
     # via pydantic
 attrs==23.2.0
@@ -23,18 +19,14 @@ chardet==5.2.0
     # via frictionless
 charset-normalizer==3.3.2
     # via requests
-ckanapi==4.7
-    # via hdx-python-api (pyproject.toml)
+ckanapi==4.8
 click==8.1.7
     # via typer
-colorama==0.4.6
-    # via typer
-coverage[toml]==7.4.4
+coverage==7.4.4
     # via pytest-cov
 cryptography==42.0.5
     # via pyopenssl
 defopt==6.4.0
-    # via hdx-python-api (pyproject.toml)
 distlib==0.3.8
     # via virtualenv
 dnspython==2.6.1
@@ -43,30 +35,25 @@ docopt==0.6.2
     # via
     #   ckanapi
     #   num2words
-docutils==0.20.1
+docutils==0.21.post1
     # via defopt
 email-validator==2.1.1
-    # via hdx-python-api (pyproject.toml)
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.13.1
+filelock==3.13.4
     # via virtualenv
 frictionless==5.16.1
     # via hdx-python-utilities
-google-auth==2.28.2
+google-auth==2.29.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-gspread==6.0.2
-    # via hdx-python-api (pyproject.toml)
-hdx-python-country==3.6.9
-    # via hdx-python-api (pyproject.toml)
+gspread==6.1.0
+hdx-python-country==3.7.0
 hdx-python-utilities==3.6.7
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   hdx-python-country
+    # via hdx-python-country
 humanize==4.9.0
     # via frictionless
 identify==2.5.35
@@ -77,7 +64,7 @@ idna==3.6
     #   requests
 ijson==3.2.3
     # via hdx-python-utilities
-inflect==7.0.0
+inflect==7.2.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -94,13 +81,10 @@ jsonschema==4.17.3
     #   frictionless
     #   tableschema-to-template
 libhxl==5.2.1
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   hdx-python-country
+    # via hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
 makefun==1.15.2
-    # via hdx-python-api (pyproject.toml)
 markdown-it-py==3.0.0
     # via rich
 marko==2.0.3
@@ -109,8 +93,9 @@ markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
+more-itertools==10.2.0
+    # via inflect
 ndg-httpsclient==0.5.1
-    # via hdx-python-api (pyproject.toml)
 nodeenv==1.8.0
     # via pre-commit
 num2words==0.5.13
@@ -133,40 +118,31 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.6.2
-    # via hdx-python-api (pyproject.toml)
-pyasn1==0.5.1
+pre-commit==3.7.0
+pyasn1==0.6.0
     # via
-    #   hdx-python-api (pyproject.toml)
     #   ndg-httpsclient
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0
     # via google-auth
-pycparser==2.21
+pycparser==2.22
     # via cffi
 pydantic==2.6.4
-    # via
-    #   frictionless
-    #   inflect
+    # via frictionless
 pydantic-core==2.16.3
     # via pydantic
 pygments==2.17.2
     # via rich
 pyopenssl==24.1.0
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   ndg-httpsclient
+    # via ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
 pytest==8.1.1
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   pytest-cov
-pytest-cov==4.1.0
-    # via hdx-python-api (pyproject.toml)
+    # via pytest-cov
+pytest-cov==5.0.0
 python-dateutil==2.8.2
     # via
     #   frictionless
@@ -184,20 +160,18 @@ pyyaml==6.0.1
     #   pre-commit
     #   tableschema-to-template
 quantulum3==0.9.0
-    # via hdx-python-api (pyproject.toml)
 ratelimit==2.2.1
     # via hdx-python-utilities
 requests==2.31.0
     # via
     #   ckanapi
     #   frictionless
-    #   hdx-python-api (pyproject.toml)
     #   libhxl
     #   requests-file
     #   requests-oauthlib
 requests-file==2.0.0
     # via hdx-python-utilities
-requests-oauthlib==1.4.0
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
@@ -209,10 +183,16 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
+setuptools==69.2.0
+    # via
+    #   ckanapi
+    #   nodeenv
 shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
     # via frictionless
+simplejson==3.19.2
+    # via ckanapi
 six==1.16.0
     # via
     #   ckanapi
@@ -234,14 +214,17 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer[all]==0.9.0
+typeguard==4.2.1
+    # via inflect
+typer==0.12.3
     # via frictionless
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via
     #   frictionless
     #   inflect
     #   pydantic
     #   pydantic-core
+    #   typeguard
     #   typer
 unidecode==1.3.8
     # via
@@ -251,7 +234,7 @@ urllib3==2.2.1
     # via
     #   libhxl
     #   requests
-validators==0.23.1
+validators==0.28.0
     # via frictionless
 virtualenv==20.25.1
     # via pre-commit
@@ -265,6 +248,3 @@ xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py
@@ -2601,10 +2601,10 @@ def generate_qc_resource_from_rows(
             encoding=encoding,
         )
 
-    def generate_resource_from_iterator(
+    def generate_resource_from_iterable(
         self,
         headers: ListTuple[str],
-        iterator: Iterator[Union[ListTuple, Dict]],
+        iterable: Iterable[Union[ListTuple, Dict]],
         hxltags: Dict[str, str],
         folder: str,
         filename: str,
@@ -2615,7 +2615,7 @@ def generate_resource_from_iterator(
         quickcharts: Optional[Dict] = None,
         encoding: Optional[str] = None,
     ) -> Tuple[bool, Dict]:
-        """Given headers and an iterator, write rows to csv and create
+        """Given headers and an iterable, write rows to csv and create
         resource, adding to it the dataset. The returned dictionary will
         contain the resource in the key resource, headers in the key headers
         and list of rows in the key rows.
@@ -2652,7 +2652,7 @@ def generate_resource_from_iterator(
 
         Args:
             headers (ListTuple[str]): Headers
-            iterator (Iterator[Union[ListTuple,Dict]]): Iterator returning rows
+            iterable (Iterable[Union[ListTuple,Dict]]): Iterable returning rows
             hxltags (Dict[str,str]): Header to HXL hashtag mapping
             folder (str): Folder to which to write file containing rows
             filename (str): Filename of file to write rows
@@ -2738,7 +2738,7 @@ def datecol_function(row):
 
             date_function = datecol_function
 
-        for row in iterator:
+        for row in iterable:
             if date_function is not None:
                 result = date_function(row)
                 if result is None:
@@ -2813,6 +2813,38 @@ def datecol_function(row):
                     retdict["qc_resource"] = resource
         return True, retdict
 
+    def generate_resource_from_iterator(
+        self,
+        headers: ListTuple[str],
+        iterator: Iterator[Union[ListTuple, Dict]],
+        hxltags: Dict[str, str],
+        folder: str,
+        filename: str,
+        resourcedata: Dict,
+        datecol: Optional[Union[int, str]] = None,
+        yearcol: Optional[Union[int, str]] = None,
+        date_function: Optional[Callable[[Dict], Optional[Dict]]] = None,
+        quickcharts: Optional[Dict] = None,
+        encoding: Optional[str] = None,
+    ) -> Tuple[bool, Dict]:
+        warnings.warn(
+            "generate_resource_from_iterator() is deprecated, use generate_resource_from_iterable() instead",
+            DeprecationWarning,
+        )
+        return self.generate_resource_from_iterable(
+            headers,
+            iterator,
+            hxltags,
+            folder,
+            filename,
+            resourcedata,
+            datecol,
+            yearcol,
+            date_function,
+            quickcharts,
+            encoding,
+        )
+
     def download_and_generate_resource(
         self,
         downloader: BaseDownload,
@@ -2898,7 +2930,7 @@ def download_and_generate_resource(
             format="csv",
             **kwargs,
         )
-        return self.generate_resource_from_iterator(
+        return self.generate_resource_from_iterable(
             headers,
             iterator,
             hxltags,