From e77272545781beb71cee6adc617949aa2f4fe0fe Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Sun, 6 Jun 2021 18:07:47 +0200 Subject: [PATCH] Cache generated un/structure functions using linecache --- HISTORY.rst | 1 + src/cattr/gen.py | 98 +++++++++++++++++++++++++++++++++++++++-------- tests/test_gen.py | 72 ++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 tests/test_gen.py diff --git a/HISTORY.rst b/HISTORY.rst index e784e81a..ba726fbc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ History ------------------ * Fix ``GenConverter`` mapping structuring for unannotated dicts on Python 3.8. (`#151 `_) +* The source code for generated un/structuring functions is stored in the `linecache` cache, which enables more informative stack traces when un/structuring errors happen using the `GenConverter`. This behavior can optionally be disabled to save memory. 1.7.1 (2021-05-28) ------------------ diff --git a/src/cattr/gen.py b/src/cattr/gen.py index b44baf72..04f1c75c 100644 --- a/src/cattr/gen.py +++ b/src/cattr/gen.py @@ -1,4 +1,6 @@ +import linecache import re +import uuid from dataclasses import is_dataclass from typing import Any, Optional, Type, TypeVar @@ -21,7 +23,13 @@ def override(omit_if_default=None, rename=None): _neutral = AttributeOverride() -def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs): +def make_dict_unstructure_fn( + cl, + converter, + omit_if_default: bool = False, + _cattrs_use_linecache: bool = True, + **kwargs, +): """Generate a specialized dict unstructuring function for an attrs class.""" cl_name = cl.__name__ fn_name = "unstructure_" + cl_name @@ -31,7 +39,7 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs): attrs = adapted_fields(cl) # type: ignore - lines.append(f"def {fn_name}(i):") + lines.append(f"def {fn_name}(instance):") lines.append(" res = {") for a in attrs: attr_name = a.name @@ -50,11 +58,11 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs): is_identity = handler == converter._unstructure_identity if not is_identity: - unstruct_handler_name = f"__cattr_unstruct_handler_{attr_name}" + unstruct_handler_name = f"unstructure_{attr_name}" globs[unstruct_handler_name] = handler - invoke = f"{unstruct_handler_name}(i.{attr_name})" + invoke = f"{unstruct_handler_name}(instance.{attr_name})" else: - invoke = f"i.{attr_name}" + invoke = f"instance.{attr_name}" if d is not attr.NOTHING and ( (omit_if_default and override.omit_if_default is not False) @@ -66,14 +74,18 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs): globs[def_name] = d.factory if d.takes_self: post_lines.append( - f" if i.{attr_name} != {def_name}(i):" + f" if instance.{attr_name} != {def_name}(instance):" ) else: - post_lines.append(f" if i.{attr_name} != {def_name}():") + post_lines.append( + f" if instance.{attr_name} != {def_name}():" + ) post_lines.append(f" res['{kn}'] = {invoke}") else: globs[def_name] = d - post_lines.append(f" if i.{attr_name} != {def_name}:") + post_lines.append( + f" if instance.{attr_name} != {def_name}:" + ) post_lines.append(f" res['{kn}'] = {invoke}") else: @@ -82,10 +94,17 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs): lines.append(" }") total_lines = lines + post_lines + [" return res"] + script = "\n".join(total_lines) - eval(compile("\n".join(total_lines), "", "exec"), globs) + fname = _generate_unique_filename( + cl, "unstructure", reserve=_cattrs_use_linecache + ) + + eval(compile(script, fname, "exec"), globs) fn = globs[fn_name] + if _cattrs_use_linecache: + linecache.cache[fname] = len(script), None, total_lines, fname return fn @@ -110,7 +129,11 @@ def generate_mapping(cl: Type, old_mapping): def make_dict_structure_fn( - cl: Type, converter, _cattrs_forbid_extra_keys: bool = False, **kwargs + cl: Type, + converter, + _cattrs_forbid_extra_keys: bool = False, + _cattrs_use_linecache: bool = True, + **kwargs, ): """Generate a specialized dict structuring function for an attrs class.""" @@ -167,20 +190,20 @@ def make_dict_structure_fn( else: handler = converter.structure - struct_handler_name = f"__cattr_struct_handler_{an}" + struct_handler_name = f"structure_{an}" globs[struct_handler_name] = handler ian = an if (is_dc or an[0] != "_") else an[1:] kn = an if override.rename is None else override.rename - globs[f"__c_t_{an}"] = type + globs[f"type_{an}"] = type if a.default is NOTHING: lines.append( - f" '{ian}': {struct_handler_name}(o['{kn}'], __c_t_{an})," + f" '{ian}': {struct_handler_name}(o['{kn}'], type_{an})," ) else: post_lines.append(f" if '{kn}' in o:") post_lines.append( - f" res['{ian}'] = {struct_handler_name}(o['{kn}'], __c_t_{an})" + f" res['{ian}'] = {struct_handler_name}(o['{kn}'], type_{an})" ) lines.append(" }") if _cattrs_forbid_extra_keys: @@ -196,7 +219,20 @@ def make_dict_structure_fn( total_lines = lines + post_lines + [" return __cl(**res)"] - eval(compile("\n".join(total_lines), "", "exec"), globs) + fname = _generate_unique_filename( + cl, "structure", reserve=_cattrs_use_linecache + ) + script = "\n".join(total_lines) + eval( + compile( + script, + fname, + "exec", + ), + globs, + ) + if _cattrs_use_linecache: + linecache.cache[fname] = len(script), None, total_lines, fname return globs[fn_name] @@ -396,3 +432,35 @@ def make_mapping_structure_fn( fn = globs[fn_name] return fn + + +def _generate_unique_filename(cls, func_name, reserve=True): + """ + Create a "filename" suitable for a function being generated. + """ + unique_id = uuid.uuid4() + extra = "" + count = 1 + + while True: + unique_filename = "".format( + func_name, + cls.__module__, + getattr(cls, "__qualname__", cls.__name__), + extra, + ) + if not reserve: + return unique_filename + # To handle concurrency we essentially "reserve" our spot in + # the linecache with a dummy line. The caller can then + # set this value correctly. + cache_line = (1, None, (str(unique_id),), unique_filename) + if ( + linecache.cache.setdefault(unique_filename, cache_line) + == cache_line + ): + return unique_filename + + # Looks like this spot is taken. Try again. + count += 1 + extra = "-{0}".format(count) diff --git a/tests/test_gen.py b/tests/test_gen.py new file mode 100644 index 00000000..02846543 --- /dev/null +++ b/tests/test_gen.py @@ -0,0 +1,72 @@ +"""Tests for functionality from the gen module.""" +import linecache +from traceback import format_exc + +from attr import define + +from cattr import GenConverter +from cattr.gen import make_dict_structure_fn, make_dict_unstructure_fn + + +def test_structure_linecache(): + """Linecaching for structuring should work.""" + + @define + class A: + a: int + + c = GenConverter() + try: + c.structure({"a": "test"}, A) + except ValueError: + res = format_exc() + assert "'a'" in res + + +def test_unstructure_linecache(): + """Linecaching for unstructuring should work.""" + + @define + class Inner: + a: int + + @define + class Outer: + inner: Inner + + c = GenConverter() + try: + c.unstructure(Outer({})) + except AttributeError: + res = format_exc() + assert "'a'" in res + + +def test_no_linecache(): + """Linecaching should be disableable.""" + + @define + class A: + a: int + + c = GenConverter() + before = len(linecache.cache) + c.structure(c.unstructure(A(1)), A) + after = len(linecache.cache) + + assert after == before + 2 + + @define + class B: + a: int + + before = len(linecache.cache) + c.register_structure_hook( + B, make_dict_structure_fn(B, c, _cattrs_use_linecache=False) + ) + c.register_unstructure_hook( + B, make_dict_unstructure_fn(B, c, _cattrs_use_linecache=False) + ) + c.structure(c.unstructure(B(1)), B) + + assert len(linecache.cache) == before