Update pydoclint config. Add docstr formatter

.pre-commit-config.yaml

@@ -19,7 +19,9 @@ repos:
     rev: 0.1.4
     hooks:
       - id: pydoclint
-        args: [--config=pyproject.toml]
+        args:
+          - --quiet  # otherwise prints all checked files...
+          - --config=pyproject.toml
 
   - repo: https://github.com/psf/black
     rev: 23.7.0

cognite/client/_api/assets.py

@@ -260,7 +260,6 @@ def list(
                 To prevent unexpected problems and maximize read throughput, API documentation recommends at most use 10 partitions.
                 When using more than 10 partitions, actual throughout decreases.
                 In future releases of the APIs, CDF may reject requests with more than 10 partitions.
-
             limit (int, optional): Maximum number of assets to return. Defaults to 25. Set to -1, float("inf") or None
                 to return all items.
 

pyproject.toml

@@ -18,6 +18,7 @@ keep-runtime-typing = true
 style = 'google'
 exclude = '\.git|\.tox|tests/|scripts/'
 require-return-section-when-returning-none = true
+arg-type-hints-in-docstring = true
 
 [tool.ruff.per-file-ignores]
 # let scripts use print statements

scripts/custom_checks/docstrings.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import inspect
+import re
+from pathlib import Path
+
+from tests.utils import all_subclasses
+from cognite.client._api_client import APIClient
+
+CWD = Path.cwd()
+
+
+class Param:
+    def __init__(self, line: str):
+        foo = line
+        try:
+            check = line.split(":", 1)[0]
+            if check.count("(") and line[(idx := line.index("(")) - 1] != " ":
+                # User forgot to add a space before first parenthesis:
+                line = f"{line[:idx]} {line[idx:]}"
+
+            if check.count("(") or check.count(")"):
+                # We probably have an annotation:
+                self.var_name, split_line = line.strip().split(" ", 1)
+                self.annot, self.descr = split_line.split(": ", 1)
+                self.annot = self.annot.lstrip("(").rstrip(")")
+
+            elif check.count("(") == check.count(")") == 0:
+                # ...we probably don't:
+                self.var_name, self.descr = line.strip().split(": ", 1)
+                self.annot = ""
+            else:
+                raise ValueError(f"Unable to parse: {line.strip()!r}")
+        except ValueError:
+            print("\n\nORIG STR:", foo)
+            print(self.var_name)
+            raise
+
+    def __repr__(self):
+        return f"Param({self.var_name!s}, {self.annot!s}, {self.descr!s})"
+
+
+def count_indent(s):
+    return re.search("[^ ]", s + "x").start()
+
+
+class DocstrFormatter:
+    NO_ANNOT = object()
+
+    def __init__(self, doc, method):
+        self.original_doc = doc
+        self.doc_annots, self.return_annot = self._extract_annotations(method)
+        if not self.doc_annots:  # Function takes no args
+            self.params = []
+            return
+
+        self.doc_before, self.doc_after, args_indent, parameters = self.parse_doc(doc)
+        self.indentation = args_indent + 4
+        self.params = list(map(Param, parameters))
+
+    def _extract_annotations(self, method):
+        def fix_literal(s):
+            # Example: Union[Literal[('aaa', 'bbb')]] -> Union[Literal["aaa", "bbb"]]
+            if match := re.search(r"Literal\[(\((.*)\))\]", s):
+                return s.replace(match.group(1), match.group(2).replace("'", '"'))
+            return s
+
+        annots = {var: fix_literal(str(annot)) for var, annot in method.__annotations__.items()}
+        return_annot = annots.pop("return", self.NO_ANNOT)
+        return annots, return_annot
+
+    @staticmethod
+    def parse_doc(doc):
+        idx_start, idx_end = None, None
+        args_indent = None
+        start_capture = False
+        parameters = []
+
+        lines = doc.splitlines()
+        if not lines[-1].strip():
+            lines[:-1] = [line.rstrip() for line in lines[:-1]]
+
+        for i, line in enumerate(lines):
+            line_indent = count_indent(line)
+            if start_capture:
+                if line_indent == args_indent or not line.strip():
+                    idx_end = i
+                    break
+                if line_indent > args_indent + 4:
+                    # Assume multilines belong to previous line:
+                    parameters[-1] += f" {line.strip()}"
+                    continue
+                parameters.append(line)
+                continue
+
+            elif "args:" in line.lower():
+                args_indent = line_indent
+                start_capture = True
+                idx_start = i + 1
+        else:
+            # End was not found:
+            idx_end = len(lines)
+
+        return lines[:idx_start], lines[idx_end:], args_indent, parameters
+
+    def docstring_is_correct(self):
+        annots = dict((p.var_name, p.annot) for p in self.params)
+        return (
+            # Takes no args?
+            not self.doc_annots
+            # Do the variables match? ...correct order?
+            or list(self.doc_annots.keys()) == list(annots.keys())
+            # Do the annotations match?
+            and list(self.doc_annots.values()) == list(annots.values())
+        )
+
+    def _create_docstring_param_description(self):
+        whitespace = " " * self.indentation
+        fixed_lines = []
+        doc_annot_dct = dict((p.var_name, p.descr) for p in self.params)
+        for var, annot in self.doc_annots.items():
+            description = doc_annot_dct.get(var, "No description.")
+            fixed_lines.append(f"{whitespace}{var} ({annot}): {description}")
+        return fixed_lines
+
+    def create_docstring(self):
+        fixed_param_description = self._create_docstring_param_description()
+        return "\n".join(self.doc_before + fixed_param_description + self.doc_after)
+
+    def update_py_file(self, cls, attr) -> str:
+        source_code = (path := Path(inspect.getfile(cls))).read_text()
+        new_source = source_code.replace(self.original_doc, self.create_docstring())
+        if source_code == new_source:
+            return f"Couldn't update docstring for '{cls.__name__}.{attr}', please inspect manually"
+
+        with path.open("w") as file:
+            file.write(new_source)
+
+        return f"Fixed docstring for '{cls.__name__}.{attr}'"
+
+
+def get_public_methods(cls):
+    return [
+        (attr, method) for attr in dir(cls)
+        if not attr.startswith("_") and inspect.isfunction((method := getattr(cls, attr)))
+    ]
+
+
+def format_docstrings_for_subclasses(cls) -> list[str]:
+    failed = []
+    for cls in all_subclasses(cls):
+        for attr, method in get_public_methods(cls):
+            if doc := method.__doc__:
+                doc_fmt = DocstrFormatter(doc, method)
+                if not doc_fmt.docstring_is_correct():
+                    if err_msg := doc_fmt.update_py_file(cls, attr):
+                        failed.append(err_msg)
+    return failed
+
+
+def format_docstrings() -> list[str]:
+    # TODO: Add more baseclasses to parse, e.g. CogniteResource:
+    return "\n".join(sum((format_docstrings_for_subclasses(base_cls) for base_cls in [APIClient]), []))

scripts/run_checks.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from scripts.custom_checks.docstrings import format_docstrings
 from scripts.custom_checks.version import (
     changelog_entry_date,
     changelog_entry_version_matches,
@@ -14,6 +15,7 @@ def run_checks() -> list[str | None]:
         changelog_entry_version_matches(),
         changelog_entry_date(),
         version_number_is_increasing(),
+        format_docstrings(),
     ]