Fetch only required fields based on User Query (#83)

* [WIP] feat: Experiments * feat: DataLoader * fix: CursorPaginator Select Fields * fix: deferred_list callback list * fix: support parent link fields * fix: remove deprecated code * fix: removed unused code * feat: Basic Perms for new Resolvers fix: Use get_list in doc-dataloader feat: Basic Perms for new resolvers Co-authored-by: Fahim Ali Zain <[email protected]> Merge-request: ROMMAN-MR-126 Merged-by: Fahim Ali Zain <[email protected]> * feat: Field Level Perms (#66) * feat: Field Level Perms * fix: keywords in field names * fix: DeferredValue support for Mutations (#67) * fix: Reduced no. of iterations in default schema binding * feat: Introduce hook 'doctype_resolver_processors' Co-authored-by: Fahim Ali Zain <[email protected]> Merge-request: ROMMAN-MR-178 Merged-by: Fahim Ali Zain <[email protected]> * feat: Translations Support fix: remove redundant resolver check Merge branch 'ROMMAN-T-289-kick-default-resolver' into ROMMAN-T-481-translations feat: Translations Support Co-authored-by: Fahim Ali Zain <[email protected]> Merge-request: ROMMAN-MR-177 Merged-by: Fahim Ali Zain <[email protected]> * fix: Setup GQLType.doctype resolver manually * feat: implement select field resolver Co-authored-by: Abadulrehman <[email protected]> Merge-request: ROMMAN-MR-196 Merged-by: Fahim Ali Zain <[email protected]> * refactor: check if return type is scalar before link field binded (#70) * fix: get_allowed_fieldnames_for_doctype on plain child-doctype support * fix: default_fields link fields like owner * fix: cache get_allowed_fieldnames_for_doctype at the request level * refactor: query only user requested fields * feat: pre load schema's utility (#78) * [ROMMAN-T-521] GQL Dataloader: Raise Perm Error on GQLNonNull Permlevel Restricted Fields fix: check for GraphQLNonNull fix: use default_field_resolver from graphql fix: refactored perm checks fix: Raise Perm Error on GQLNonNull Permlevel Restricted Fields Co-authored-by: Fahim Ali Zain <[email protected]> Merge-request: ROMMAN-MR-214 Merged-by: Fahim Ali Zain <[email protected]> * use new graphql-sync-dataloaders package (#81) * refactor: use graphql-sync-dataloader package * refactor: update package graphql-sync-dataloader * refactor: fetch required fields * refactor: suppress JMESPathTypeError * refactor: move dedicated functions * refactor: use is_introspection_key wrapper * feat: get fields selected from child table query * feat: get fields selected from doctype dataloader * refactor: support use of aliases ie dataloaders re-used in query * refactor: extract fields from all field nodes * refactor: always query name cursor paginator fields * refactor: query name and parent for child tables * refactor: return of_type * refactor: use merge deep * refactor: remove unused imports * refactor: type error if extra_fields empty * refactor: get_doctype_requested_fields wrapper * refactor: query name always * refactor: cache fields in get_doctype_requested_fields * refactor: add parent doctype kwarg to get_fields_cursor_paginator * refactor: file names --------- Co-authored-by: Fahim Ali Zain <[email protected]> Co-authored-by: Abadulrehman <[email protected]>
leam-tech · Jan 30, 2023 · 3985b3e · 3985b3e
1 parent fd34023
commit 3985b3e
Show file tree

Hide file tree

Showing 10 changed files with 232 additions and 52 deletions.
diff --git a/frappe_graphql/utils/__init__.py b/frappe_graphql/utils/__init__.py
@@ -0,0 +1,5 @@
+from graphql import GraphQLResolveInfo
+
+
+def get_info_path_key(info: GraphQLResolveInfo):
+    return "-".join([p for p in info.path.as_list() if isinstance(p, str)])
diff --git a/frappe_graphql/utils/cursor_pagination.py b/frappe_graphql/utils/cursor_pagination.py
@@ -1,22 +1,24 @@
+import frappe
 import base64
+import contextlib
+from typing import List
 from graphql import GraphQLResolveInfo, GraphQLError
-
-import frappe
-
+from frappe_graphql.utils.introspection import is_introspection_key
+from frappe_graphql.utils.gql_fields import get_field_tree_dict
 from frappe_graphql.utils.permissions import get_allowed_fieldnames_for_doctype
 
 
 class CursorPaginator(object):
     def __init__(
-            self,
-            doctype,
-            filters=None,
-            skip_process_filters=False,
-            count_resolver=None,
-            node_resolver=None,
-            default_sorting_fields=None,
-            default_sorting_direction=None,
-            extra_args=None):
+        self,
+        doctype,
+        filters=None,
+        skip_process_filters=False,
+        count_resolver=None,
+        node_resolver=None,
+        default_sorting_fields=None,
+        default_sorting_direction=None,
+        extra_args=None):
 
         if (not count_resolver) != (not node_resolver):
             frappe.throw(
@@ -151,12 +153,11 @@ def get_data(self, doctype, filters, sorting_fields, sort_dir, limit):
         )
 
     def get_fields_to_fetch(self, doctype, filters, sorting_fields):
-        fieldnames = get_allowed_fieldnames_for_doctype(doctype)
-        return list(set(fieldnames + sorting_fields))
+        return get_paginator_fields(doctype, self.resolve_info, sorting_fields)
 
     def get_sort_args(self, sorting_input=None):
         sort_dir = self.default_sorting_direction if self.default_sorting_direction in (
-        "asc", "desc") else "desc"
+            "asc", "desc") else "desc"
         if not self.default_sorting_fields:
             meta = frappe.get_meta(self.doctype)
             if meta.istable:
@@ -240,27 +241,27 @@ def format_column_name(column):
                 return column
             meta = frappe.get_meta(self.doctype)
             return f"`tab{self.doctype}`.{column}" if column in \
-                meta.get_valid_columns() else column
+                                                      meta.get_valid_columns() else column
 
         def db_escape(v):
             return frappe.db.escape(v)
 
         def _get_cursor_column_condition(operator, column, value, include_equals=False):
             if operator == ">":
                 return format_column_name(column) \
-                    + f" {operator}{'=' if include_equals else ''} " \
-                    + db_escape(value)
+                       + f" {operator}{'=' if include_equals else ''} " \
+                       + db_escape(value)
             else:
                 if value is None:
                     return format_column_name(column) \
-                        + " IS NULL"
+                           + " IS NULL"
                 return "(" \
-                    + format_column_name(column) \
-                    + f" {operator}{'=' if include_equals else ''} " \
-                    + db_escape(value) \
-                    + " OR " \
-                    + format_column_name(column) \
-                    + " IS NULL)"
+                       + format_column_name(column) \
+                       + f" {operator}{'=' if include_equals else ''} " \
+                       + db_escape(value) \
+                       + " OR " \
+                       + format_column_name(column) \
+                       + " IS NULL)"
 
         def _get_cursor_condition(sorting_fields, values):
             """
@@ -277,7 +278,7 @@ def _get_cursor_condition(sorting_fields, values):
 
                 if sub_condition:
                     return f"(({format_column_name(sorting_fields[0])} IS NULL AND {sub_condition})" \
-                        + f" OR {format_column_name(sorting_fields[0])} IS NOT NULL)"
+                           + f" OR {format_column_name(sorting_fields[0])} IS NOT NULL)"
                 return ""
 
             condition = _get_cursor_column_condition(
@@ -322,3 +323,36 @@ def to_cursor(self, row, sorting_fields):
 
     def from_cursor(self, cursor):
         return frappe.parse_json(frappe.safe_decode(base64.b64decode(cursor)))
+
+
+def _get_paginator_node_fields(info: GraphQLResolveInfo):
+    """
+    we know how the structure looks like based on the specs
+    https://relay.dev/graphql/connections.htm
+
+    We are only concerned with the fields we are fetching from the db
+    Note:
+        => We will be avoiding introspection keys
+        => We will not be sending __name as we define these link fields
+    """
+    import jmespath
+    from jmespath.exceptions import JMESPathTypeError
+
+    expression = jmespath.compile("edges.node.keys(@)")
+    fields = get_field_tree_dict(info)
+    with contextlib.suppress(JMESPathTypeError):
+        # maybe the following can be done in jmespath =)
+        return [field.replace('__name', '') for field in expression.search(fields) or [] if
+                not is_introspection_key(field)]
+    return []
+
+
+def get_paginator_fields(doctype: str, info: GraphQLResolveInfo,
+                         extra_fields: List[str] = None, parent_doctype=None):
+    """
+    This can be used in our custom CursorPaginator queries
+    """
+    selected_fields = set(_get_paginator_node_fields(info))
+    selected_fields.add("name")
+    fieldnames = set(get_allowed_fieldnames_for_doctype(doctype, parent_doctype=parent_doctype))
+    return list(set(list(selected_fields.intersection(fieldnames)) + (extra_fields or [])))
diff --git a/frappe_graphql/utils/depth_limit_validator.py b/frappe_graphql/utils/depth_limit_validator.py
@@ -1,8 +1,11 @@
 from frappe import _
-from graphql import (ValidationRule, ValidationContext, DefinitionNode, FragmentDefinitionNode, OperationDefinitionNode,
+from graphql import (ValidationRule, ValidationContext, DefinitionNode, FragmentDefinitionNode,
+                     OperationDefinitionNode,
                      Node, GraphQLError, FieldNode, InlineFragmentNode, FragmentSpreadNode)
 from typing import Optional, Union, Callable, Pattern, List, Dict
 
+from frappe_graphql.utils.introspection import is_introspection_key
+
 IgnoreType = Union[Callable[[str], bool], Pattern, str]
 
 """
@@ -78,7 +81,8 @@ def determine_depth(
     if depth_so_far > max_depth:
         context.report_error(
             GraphQLError(
-                _("'{0}' exceeds maximum operation depth of {1}.").format(operation_name, max_depth),
+                _("'{0}' exceeds maximum operation depth of {1}.").format(operation_name,
+                                                                          max_depth),
                 [node],
             )
         )
@@ -137,14 +141,6 @@ def determine_depth(
         )
 
 
-def is_introspection_key(key):
-    # from: https://spec.graphql.org/June2018/#sec-Schema
-    # > All types and directives defined within a schema must not have a name which
-    # > begins with "__" (two underscores), as this is used exclusively
-    # > by GraphQL’s introspection system.
-    return str(key).startswith("__")
-
-
 def is_ignored(node: FieldNode, ignore: Optional[List[IgnoreType]] = None) -> bool:
     if ignore is None:
         return False

diff --git a/frappe_graphql/utils/gql_fields.py b/frappe_graphql/utils/gql_fields.py
@@ -0,0 +1,112 @@
+from graphql import GraphQLResolveInfo
+
+from mergedeep import merge, Strategy
+
+from frappe_graphql.utils.introspection import is_introspection_key
+from frappe_graphql.utils import get_info_path_key
+from frappe_graphql.utils.permissions import get_allowed_fieldnames_for_doctype
+
+
+def collect_fields(node: dict, fragments: dict):
+    """
+    Recursively collects fields from the AST
+    Inspired from https://gist.github.com/mixxorz/dc36e180d1888629cf33
+
+    Notes:
+        => Please make sure your node and fragments passed have been converted to dicts
+        => Best used in conjunction with `get_allowed_fieldnames_for_doctype()`
+
+    Args:
+        node (dict): A node in the AST
+        fragments (dict): Fragment definitions
+    Returns:
+        A dict mapping each field found, along with their sub fields.
+        {'name': {},
+         'sentimentsPerLanguage': {'id': {},
+                                   'name': {},
+                                   'totalSentiments': {}},
+         'slug': {}}
+    """
+
+    field = {}
+
+    if node.get('selection_set'):
+        for leaf in node['selection_set']['selections']:
+            if leaf['kind'] == 'field':
+                field[leaf['name']['value']] = collect_fields(leaf, fragments)
+            elif leaf['kind'] == 'fragment_spread':
+                field.update(collect_fields(fragments[leaf['name']['value']],
+                                            fragments))
+    return field
+
+
+def get_field_tree_dict(info: GraphQLResolveInfo):
+    """
+    A hierarchical dictionary of the graphql resolver fields nodes merged and returned.
+    Args:
+        info (GraphQLResolveInfo): GraphqlResolver Info
+    Returns:
+        A dict mapping each field found, along with their sub fields.
+        {'name': {},
+         'sentimentsPerLanguage': {'id': {},
+                                   'name': {},
+                                   'totalSentiments': {}},
+         'slug': {}}
+    """
+    fragments = {name: value.to_dict() for name, value in info.fragments.items()}
+    fields = {}
+    for field_node in info.field_nodes:
+        merge(fields, collect_fields(field_node.to_dict(), fragments), strategy=Strategy.ADDITIVE)
+    return fields
+
+
+def get_doctype_requested_fields(
+    doctype: str,
+    info: GraphQLResolveInfo,
+    mandatory_fields: set = None,
+    parent_doctype: str = None
+):
+    """
+    Returns the list of requested fields for the given doctype from a GraphQL query.
+
+    :param doctype: The doctype to retrieve requested fields for.
+    :type doctype: str
+    :param info: The GraphQLResolveInfo object representing information about a
+        resolver's execution.
+    :type info: GraphQLResolveInfo
+    :param mandatory_fields: A set of fields that should always be included in the returned list,
+        even if not requested.
+    :type mandatory_fields: set
+    :param parent_doctype: The doctype of the parent object, if any.
+    :type parent_doctype: str
+    :return: The list of requested fields for the given doctype.
+    :rtype: list of str
+    """
+    p_key = get_info_path_key(info)
+    requested_fields = info.context.get(p_key)
+
+    if requested_fields is not None:
+        return requested_fields
+
+    selected_fields = {
+        key.replace('__name', '')
+        for key in get_field_tree_dict(info).keys()
+        if not is_introspection_key(key)
+    }
+
+    fieldnames = set(get_allowed_fieldnames_for_doctype(
+        doctype=doctype,
+        parent_doctype=parent_doctype
+    ))
+
+    requested_fields = selected_fields.intersection(fieldnames)
+    if mandatory_fields:
+        requested_fields.update(mandatory_fields)
+
+    # send name always..
+    requested_fields.add("name")
+
+    # cache it in context..
+    info.context[p_key] = requested_fields
+
+    return list(requested_fields)
diff --git a/frappe_graphql/utils/introspection.py b/frappe_graphql/utils/introspection.py
@@ -0,0 +1,6 @@
+def is_introspection_key(key):
+    # from: https://spec.graphql.org/June2018/#sec-Schema
+    # > All types and directives defined within a schema must not have a name which
+    # > begins with "__" (two underscores), as this is used exclusively
+    # > by GraphQL’s introspection system.
+    return str(key).startswith("__")
diff --git a/frappe_graphql/utils/resolver/child_tables.py b/frappe_graphql/utils/resolver/child_tables.py
@@ -4,6 +4,8 @@
 
 from .dataloaders import get_child_table_loader
 from .utils import get_frappe_df_from_resolve_info
+from ..gql_fields import get_doctype_requested_fields
+from .. import get_info_path_key
 
 
 def setup_child_table_resolvers(meta: Meta, gql_type: GraphQLType):
@@ -28,5 +30,7 @@ def _child_table_resolver(obj, info: GraphQLResolveInfo, **kwargs):
     return get_child_table_loader(
         child_doctype=df.options,
         parent_doctype=df.parent,
-        parentfield=df.fieldname
+        parentfield=df.fieldname,
+        path=get_info_path_key(info),
+        fields=get_doctype_requested_fields(df.options, info, {"parent"}, df.parent)
     ).load(obj.get("name"))
diff --git a/frappe_graphql/utils/resolver/dataloaders/child_table_loader.py b/frappe_graphql/utils/resolver/dataloaders/child_table_loader.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from typing import List
 
 import frappe
 
@@ -7,9 +8,12 @@
 from .locals import get_loader_from_locals, set_loader_in_locals
 
 
-def get_child_table_loader(child_doctype: str, parent_doctype: str, parentfield: str) \
-        -> FrappeDataloader:
+def get_child_table_loader(child_doctype: str, parent_doctype: str, parentfield: str,
+                           path: str = None, fields: List[str] = None) -> FrappeDataloader:
     locals_key = (child_doctype, parent_doctype, parentfield)
+    if path:
+        # incase alias usage
+        locals_key = locals_key + (path,)
     loader = get_loader_from_locals(locals_key)
     if loader:
         return loader
@@ -18,14 +22,16 @@ def get_child_table_loader(child_doctype: str, parent_doctype: str, parentfield:
         child_doctype=child_doctype,
         parent_doctype=parent_doctype,
         parentfield=parentfield,
+        fields=fields
     ))
     set_loader_in_locals(locals_key, loader)
     return loader
 
 
-def _get_child_table_loader_fn(child_doctype: str, parent_doctype: str, parentfield: str):
+def _get_child_table_loader_fn(child_doctype: str, parent_doctype: str, parentfield: str,
+                               fields: List[str] = None):
     def _inner(keys):
-        fieldnames = get_allowed_fieldnames_for_doctype(
+        fieldnames = fields or get_allowed_fieldnames_for_doctype(
             doctype=child_doctype,
             parent_doctype=parent_doctype
         )

diff --git a/frappe_graphql/utils/resolver/dataloaders/doctype_loader.py b/frappe_graphql/utils/resolver/dataloaders/doctype_loader.py
@@ -5,18 +5,28 @@
 from .locals import get_loader_from_locals, set_loader_in_locals
 
 
-def get_doctype_dataloader(doctype: str) -> FrappeDataloader:
-    loader = get_loader_from_locals(doctype)
+def get_doctype_dataloader(doctype: str, path: str = None,
+                           fields: List[str] = None) -> FrappeDataloader:
+    """
+    Parameters:
+        doctype: the doctype
+        path: pass the graphql info path if your dataloader is used multiple time using aliases
+        fields: fields to fetch
+    """
+    key = doctype
+    if path:
+        key += f"-{path}"
+    loader = get_loader_from_locals(key)
     if loader:
         return loader
 
-    loader = FrappeDataloader(_get_document_loader_fn(doctype=doctype))
-    set_loader_in_locals(doctype, loader)
+    loader = FrappeDataloader(_get_document_loader_fn(doctype=doctype, fields=fields))
+    set_loader_in_locals(key, loader)
     return loader
 
 
-def _get_document_loader_fn(doctype: str):
-    fieldnames = get_allowed_fieldnames_for_doctype(doctype)
+def _get_document_loader_fn(doctype: str, fields: List[str] = None):
+    fieldnames = fields or get_allowed_fieldnames_for_doctype(doctype)
 
     def _load_documents(keys: List[str]):
         docs = frappe.get_list(