Merge pull request #421 from rollbar/batched-transform

batched transform

rollbar/__init__.py

@@ -327,6 +327,7 @@ def _get_fastapi_request():
     'request_pool_connections': None,
     'request_pool_maxsize': None,
     'request_max_retries': None,
+    'batch_transforms': False,
 }
 
 _CURRENT_LAMBDA_CONTEXT = None
@@ -341,11 +342,13 @@ def _get_fastapi_request():
 from rollbar.lib.transforms.scrub_redact import REDACT_REF
 
 from rollbar.lib import transforms
+from rollbar.lib import type_info
 from rollbar.lib.transforms.scrub import ScrubTransform
 from rollbar.lib.transforms.scruburl import ScrubUrlTransform
 from rollbar.lib.transforms.scrub_redact import ScrubRedactTransform
 from rollbar.lib.transforms.serializable import SerializableTransform
 from rollbar.lib.transforms.shortener import ShortenerTransform
+from rollbar.lib.transforms.batched import BatchedTransform
 
 
 ## public api
@@ -1082,10 +1085,11 @@ def _add_locals_data(trace_data, exc_info):
 
 
 def _serialize_frame_data(data):
-    for transform in (ScrubRedactTransform(), _serialize_transform):
-        data = transforms.transform(data, transform)
-
-    return data
+    return transforms.transform(
+        data,
+        [ScrubRedactTransform(), _serialize_transform],
+        batch_transforms=SETTINGS['batch_transforms']
+    )
 
 
 def _add_lambda_context_data(data):
@@ -1477,10 +1481,12 @@ def _build_server_data():
 
 
 def _transform(obj, key=None):
-    for transform in _transforms:
-        obj = transforms.transform(obj, transform, key=key)
-
-    return obj
+    return transforms.transform(
+        obj,
+        _transforms,
+        key=key,
+        batch_transforms=SETTINGS['batch_transforms']
+    )
 
 
 def _build_payload(data):

rollbar/lib/transform.py

@@ -0,0 +1,39 @@
+class Transform(object):
+    def default(self, o, key=None):
+        return o
+
+    def transform_circular_reference(self, o, key=None, ref_key=None):
+        # By default, we just perform a no-op for circular references.
+        # Subclasses should implement this method to return whatever representation
+        # for the circular reference they need.
+        return self.default(o, key=key)
+
+    def transform_tuple(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_namedtuple(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_list(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_dict(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_number(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_py2_str(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_py3_bytes(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_unicode(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_boolean(self, o, key=None):
+        return self.default(o, key=key)
+
+    def transform_custom(self, o, key=None):
+        return self.default(o, key=key)

rollbar/lib/transforms/__init__.py

@@ -1,6 +1,23 @@
+try:
+    # Python 3
+    from collections.abc import Iterable
+except ImportError:
+    # Python 2.7
+    from collections import Iterable
+
 from rollbar.lib import (
-    python_major_version, binary_type, string_types, integer_types,
-    number_types, traverse)
+    python_major_version,
+    binary_type,
+    string_types,
+    integer_types,
+    number_types,
+    traverse,
+    type_info,
+)
+# NOTE: Don't remove this import, it would cause a breaking change to the library's API.
+# The `Transform` class was moved out of this file to prevent a cyclical dependency issue.
+from rollbar.lib.transform import Transform
+from rollbar.lib.transforms.batched import BatchedTransform
 
 _ALLOWED_CIRCULAR_REFERENCE_TYPES = [binary_type, bool, type(None)]
 
@@ -17,99 +34,77 @@
 _ALLOWED_CIRCULAR_REFERENCE_TYPES = tuple(_ALLOWED_CIRCULAR_REFERENCE_TYPES)
 
 
-class Transform(object):
-    def default(self, o, key=None):
-        return o
-
-    def transform_circular_reference(self, o, key=None, ref_key=None):
-        # By default, we just perform a no-op for circular references.
-        # Subclasses should implement this method to return whatever representation
-        # for the circular reference they need.
-        return self.default(o, key=key)
-
-    def transform_tuple(self, o, key=None):
-        return self.default(o, key=key)
-
-    def transform_namedtuple(self, o, key=None):
-        return self.default(o, key=key)
-
-    def transform_list(self, o, key=None):
-        return self.default(o, key=key)
-
-    def transform_dict(self, o, key=None):
-        return self.default(o, key=key)
+def transform(obj, transforms, key=None, batch_transforms=False):
+    if isinstance(transforms, Transform):
+        transforms = [transforms]
 
-    def transform_number(self, o, key=None):
-        return self.default(o, key=key)
+    if batch_transforms:
+        transforms = [BatchedTransform(transforms)]
 
-    def transform_py2_str(self, o, key=None):
-        return self.default(o, key=key)
+    for transform in transforms:
+        obj = _transform(obj, transform, key=key)
 
-    def transform_py3_bytes(self, o, key=None):
-        return self.default(o, key=key)
+    return obj
 
-    def transform_unicode(self, o, key=None):
-        return self.default(o, key=key)
 
-    def transform_boolean(self, o, key=None):
-        return self.default(o, key=key)
-
-    def transform_custom(self, o, key=None):
-        return self.default(o, key=key)
-
-
-def transform(obj, transform, key=None):
+def _transform(obj, transform, key=None):
     key = key or ()
 
     def do_transform(type_name, val, key=None, **kw):
-        fn = getattr(transform, 'transform_%s' % type_name, transform.transform_custom)
+        fn = getattr(transform, "transform_%s" % type_name, transform.transform_custom)
         val = fn(val, key=key, **kw)
 
         return val
 
     if python_major_version() < 3:
+
         def string_handler(s, key=None):
             if isinstance(s, str):
-                return do_transform('py2_str', s, key=key)
+                return do_transform("py2_str", s, key=key)
             elif isinstance(s, unicode):
-                return do_transform('unicode', s, key=key)
+                return do_transform("unicode", s, key=key)
+
     else:
+
         def string_handler(s, key=None):
             if isinstance(s, bytes):
-                return do_transform('py3_bytes', s, key=key)
+                return do_transform("py3_bytes", s, key=key)
             elif isinstance(s, str):
-                return do_transform('unicode', s, key=key)
+                return do_transform("unicode", s, key=key)
 
     def default_handler(o, key=None):
         if isinstance(o, bool):
-            return do_transform('boolean', o, key=key)
+            return do_transform("boolean", o, key=key)
 
         # There is a quirk in the current version (1.1.6) of the enum
         # backport enum34 which causes it to not have the same
         # behavior as Python 3.4+. One way to identify IntEnums is that
         # they are instances of numbers but not number types.
         if isinstance(o, number_types):
             if type(o) not in number_types:
-                return do_transform('custom', o, key=key)
+                return do_transform("custom", o, key=key)
             else:
-                return do_transform('number', o, key=key)
+                return do_transform("number", o, key=key)
 
-        return do_transform('custom', o, key=key)
+        return do_transform("custom", o, key=key)
 
     handlers = {
-        'string_handler': string_handler,
-        'tuple_handler': lambda o, key=None: do_transform('tuple', o, key=key),
-        'namedtuple_handler': lambda o, key=None: do_transform('namedtuple', o, key=key),
-        'list_handler': lambda o, key=None: do_transform('list', o, key=key),
-        'set_handler': lambda o, key=None: do_transform('set', o, key=key),
-        'mapping_handler': lambda o, key=None: do_transform('dict', o, key=key),
-        'circular_reference_handler': lambda o, key=None, ref_key=None:
-            do_transform('circular_reference', o, key=key, ref_key=ref_key),
-        'default_handler': default_handler,
-        'allowed_circular_reference_types': _ALLOWED_CIRCULAR_REFERENCE_TYPES
+        "string_handler": string_handler,
+        "tuple_handler": lambda o, key=None: do_transform("tuple", o, key=key),
+        "namedtuple_handler": lambda o, key=None: do_transform(
+            "namedtuple", o, key=key
+        ),
+        "list_handler": lambda o, key=None: do_transform("list", o, key=key),
+        "set_handler": lambda o, key=None: do_transform("set", o, key=key),
+        "mapping_handler": lambda o, key=None: do_transform("dict", o, key=key),
+        "circular_reference_handler": lambda o, key=None, ref_key=None: do_transform(
+            "circular_reference", o, key=key, ref_key=ref_key
+        ),
+        "default_handler": default_handler,
+        "allowed_circular_reference_types": _ALLOWED_CIRCULAR_REFERENCE_TYPES,
     }
 
     return traverse.traverse(obj, key=key, **handlers)
 
 
-__all__ = ['transform', 'Transform']
+__all__ = ["transform", "Transform"]

rollbar/lib/transforms/batched.py

@@ -0,0 +1,88 @@
+from rollbar.lib.transform import Transform
+from rollbar.lib import (
+    python_major_version,
+    number_types,
+    type_info,
+)
+
+
+def do_transform(transform, type_name, val, key=None, **kw):
+    fn = getattr(transform, "transform_%s" % type_name, transform.transform_custom)
+    val = fn(val, key=key, **kw)
+
+    return val
+
+
+if python_major_version() < 3:
+
+    def string_handler(transform, s, key=None):
+        if isinstance(s, str):
+            return do_transform(transform, "py2_str", s, key=key)
+        elif isinstance(s, unicode):
+            return do_transform(transform, "unicode", s, key=key)
+
+else:
+
+    def string_handler(transform, s, key=None):
+        if isinstance(s, bytes):
+            return do_transform(transform, "py3_bytes", s, key=key)
+        elif isinstance(s, str):
+            return do_transform(transform, "unicode", s, key=key)
+
+
+def default_handler(transform, o, key=None):
+    if isinstance(o, bool):
+        return do_transform(transform, "boolean", o, key=key)
+
+    # There is a quirk in the current version (1.1.6) of the enum
+    # backport enum34 which causes it to not have the same
+    # behavior as Python 3.4+. One way to identify IntEnums is that
+    # they are instances of numbers but not number types.
+    if isinstance(o, number_types):
+        if type(o) not in number_types:
+            return do_transform(transform, "custom", o, key=key)
+        else:
+            return do_transform(transform, "number", o, key=key)
+
+    return do_transform(transform, "custom", o, key=key)
+
+
+handlers = {
+    type_info.STRING: string_handler,
+    type_info.TUPLE: lambda transform, o, key=None: do_transform(
+        transform, "tuple", o, key=key
+    ),
+    type_info.NAMEDTUPLE: lambda transform, o, key=None: do_transform(
+        transform, "namedtuple", o, key=key
+    ),
+    type_info.LIST: lambda transform, o, key=None: do_transform(
+        transform, "list", o, key=key
+    ),
+    type_info.SET: lambda transform, o, key=None: do_transform(
+        transform, "set", o, key=key
+    ),
+    type_info.MAPPING: lambda transform, o, key=None: do_transform(
+        transform, "dict", o, key=key
+    ),
+    type_info.CIRCULAR: lambda transform, o, key=None, ref_key=None: do_transform(
+        transform, "circular_reference", o, key=key, ref_key=ref_key
+    ),
+    type_info.DEFAULT: default_handler,
+}
+
+
+class BatchedTransform(Transform):
+    def __init__(self, transforms):
+        super(BatchedTransform, self).__init__()
+        self._transforms = transforms
+
+    def default(self, o, key=None):
+        for transform in self._transforms:
+            node_type = type_info.get_type(o)
+            handler = handlers.get(node_type, handlers.get(type_info.DEFAULT))
+            o = handler(transform, o, key=key)
+
+        return o
+
+
+__all__ = ["BatchedTransform"]

rollbar/lib/transforms/scrub.py

@@ -1,7 +1,7 @@
 import random
 
 from rollbar.lib import build_key_matcher, text
-from rollbar.lib.transforms import Transform
+from rollbar.lib.transform import Transform
 
 
 class ScrubTransform(Transform):

rollbar/lib/transforms/serializable.py

@@ -6,7 +6,7 @@
     undecodable_object_label, unencodable_object_label)
 from rollbar.lib import iteritems, python_major_version, text
 
-from rollbar.lib.transforms import Transform
+from rollbar.lib.transform import Transform
 
 
 class SerializableTransform(Transform):

rollbar/lib/transforms/shortener.py

@@ -12,7 +12,7 @@
 from rollbar.lib import (
     integer_types, iteritems, key_in, number_types, reprlib, sequence_types,
     string_types, text)
-from rollbar.lib.transforms import Transform
+from rollbar.lib.transform import Transform
 
 
 _type_name_mapping = {