Updating PreparererStep, creating PreparerMapping. CR changes to DataCleaner.

foreshadow/cleaners/data_cleaner.py

@@ -4,6 +4,7 @@
 import pandas as pd
 from sklearn.base import BaseEstimator, TransformerMixin
 
+from foreshadow.core import logging
 from foreshadow.core.preparerstep import PreparerStep
 from foreshadow.exceptions import InvalidDataFrame
 from foreshadow.metrics.internals import avg_col_regex, regex_rows
@@ -19,15 +20,14 @@
 class DataCleaner(PreparerStep):
     """Determine and perform best data cleaning step."""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, **kwargs):
         """Define the single step for DataCleaner, using SmartCleaner.
 
         Args:
-            *args: args to PreparerStep constructor.
             **kwargs: kwargs to PreparerStep constructor.
 
         """
-        super().__init__(*args, use_single_pipeline=True, **kwargs)
+        super().__init__(use_single_pipeline=True, **kwargs)
 
     def get_mapping(self, X):
         """Return the mapping of transformations for the DataCleaner step.
@@ -47,15 +47,37 @@ def get_mapping(self, X):
                 ]
                 for c in X
             ],
-            X=X,
+            cols=X.columns,
+        )
+
+    def __repr__(self):
+        """Return string representation of this object with parent params.
+
+        Returns:
+            See above.
+
+        """
+        r = super().__repr__()
+        preparer_params = self._preparer_params()
+        preparer_params = {p: getattr(self, p, None) for p in preparer_params}
+        preparer_print = ", ".join(
+            ["{}={}".format(k, v) for k, v in preparer_params.items()]
         )
+        return r[:-1] + preparer_print + ")"
 
 
 class SmartCleaner(SmartTransformer):
     """Intelligently decide which cleaning function should be applied."""
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
+    def __init__(
+        self,  # manually adding as otherwise get_params won't see it.
+        check_wrapped=False,
+        **kwargs
+    ):
+        self.single_input = True  # all transformers under this only accept
+        # 1 column. This is how DynamicPipeline knows this.
+        # get_params then set_params, it may be in kwargs already
+        super().__init__(check_wrapped=check_wrapped, **kwargs)
 
     def pick_transformer(self, X, y=None, **fit_params):
         """Get best transformer for a given column.
@@ -79,22 +101,44 @@ def pick_transformer(self, X, y=None, **fit_params):
         best_score = 0
         best_cleaner = None
         for cleaner, name in cleaners:
-            cleaner = cleaner(column_sharer=self.column_sharer, name=name)
+            cleaner = cleaner(column_sharer=self.column_sharer)
             score = cleaner.metric_score(X)
             if score > best_score:
                 best_score = score
                 best_cleaner = cleaner
-
         if best_cleaner is None:
             return NoTransform(column_sharer=self.column_sharer)
         return best_cleaner
 
+    def resolve(self, X, *args, **kwargs):
+        """Resolve the underlying concrete transformer.
+
+        Sets self.column_sharer with the domain tag.
+
+        Args:
+            X: input DataFrame
+            *args: args to super
+            **kwargs: kwargs to super
+
+        Returns:
+            Return from super.
+
+        """
+        ret = super().resolve(X, *args, **kwargs)
+        if self.column_sharer is not None:
+            self.column_sharer[
+                "domain", X.columns[0]
+            ] = self.transformer.__class__.__name__
+        else:
+            logging.debug("column_sharer was None")
+        return ret
+
 
 class SmartFlatten(SmartTransformer):
     """Smartly determine how to flatten an input DataFrame."""
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
+    def __init__(self, check_wrapped=True, **kwargs):
+        super().__init__(check_wrapped=check_wrapped, **kwargs)
 
     def pick_transformer(self, X, y=None, **fit_params):
         """Get best transformer for a given column.
@@ -122,16 +166,38 @@ def pick_transformer(self, X, y=None, **fit_params):
         best_score = 0
         best_flattener = None
         for flattener, name in flatteners:
-            flattener = flattener(column_sharer=self.column_sharer, name=name)
+            flattener = flattener(column_sharer=self.column_sharer)
             score = flattener.metric_score(X)
             if score > best_score:
                 best_score = score
                 best_flattener = flattener
-
         if best_flattener is None:
             return NoTransform(column_sharer=self.column_sharer)
         return best_flattener
 
+    def resolve(self, X, *args, **kwargs):
+        """Resolve the underlying concrete transformer.
+
+        Sets self.column_sharer with the domain tag.
+
+        Args:
+            X: input DataFrame
+            *args: args to super
+            **kwargs: kwargs to super
+
+        Returns:
+            Return from super.
+
+        """
+        ret = super().resolve(X, *args, **kwargs)
+        if self.column_sharer is not None:
+            self.column_sharer[
+                "domain", X.columns[0]
+            ] = self.transformer.__class__.__name__
+        else:
+            logging.debug("column_sharer was None")
+        return ret
+
 
 class BaseCleaner(BaseEstimator, TransformerMixin):
     """Base class for any Cleaner Transformer."""
@@ -142,6 +208,7 @@ def __init__(
         output_columns=None,
         confidence_computation=None,
         default=lambda x: x,
+        column_sharer=None,
     ):
         """Construct any cleaner/flattener.
 
@@ -157,6 +224,8 @@ def __init__(
                 subclass's metric computation. This implies an OVR model.
             default: Function that returns the default value for a row if
                 the transformation failed. Accepts the row as input.
+            column_sharer: An instance of 'foreshadow.core.ColumnSharer' to
+                be used to share information between PreparerSteps.
 
         Raises:
             ValueError: If not a list, int, or None specifying expected
@@ -170,6 +239,7 @@ def __init__(
         self.output_columns = output_columns
         self.transformations = transformations
         self.confidence_computation = {regex_rows: 0.8, avg_col_regex: 0.2}
+        self.column_sharer = column_sharer
         if confidence_computation is not None:
             self.confidence_computation = confidence_computation
 
@@ -289,25 +359,33 @@ def transform(self, X, y=None):
         # access single column as series and apply the list of
         # transformations to each row in the series.
         if any(
-            [isinstance(out[i], (list, tuple)) for i in range(out.shape[0])]
+            [
+                isinstance(out.iloc[i], (list, tuple))
+                for i in range(out.shape[0])
+            ]
         ):  # out are lists == new columns
             if not all(
-                [len(out[0]) == len(out[i]) for i in range(len(out[0]))]
+                [
+                    len(out.iloc[0]) == len(out.iloc[i])
+                    for i in range(len(out.iloc[0]))
+                ]
             ):
                 raise InvalidDataFrame(
                     "length of lists: {}, returned not of same value.".format(
-                        [out[i] for i in range(len(out[0]))]
+                        [out.iloc[i] for i in range(len(out[0]))]
                     )
                 )
             columns = self.output_columns
             if columns is None:
-                columns = [X.columns[0] + str(c) for c in range(len(out[0]))]
+                columns = [
+                    X.columns[0] + str(c) for c in range(len(out.iloc[0]))
+                ]
                 # by default, pandas would have given a unique integer to
                 # each column, instead, we keep the previous column name and
                 # add that integer.
             X = pd.DataFrame([*out.values], columns=columns)
         elif any(
-            [isinstance(out[i], (dict)) for i in range(out.shape[0])]
+            [isinstance(out.iloc[i], (dict)) for i in range(out.shape[0])]
         ):  # out are dicts ==  named new columns
             all_keys = dict()
             for row in out:

foreshadow/cleaners/internals/__init__.py

@@ -33,5 +33,5 @@ def _get_classes():
     return classes
 
 
-classes = _get_modules(_get_classes(), globals(), __name__)
+classes = _get_modules(_get_classes(), globals(), __name__, wrap=False)
 __all__ = classes

foreshadow/cleaners/internals/datetimes.py

@@ -16,15 +16,14 @@ def _split_to_new_cols(t):
 
     """
     delimiters = "[-/]"
-    regex = r"^.*(([\d]{4})%s([\d]{2})%s([\d]{2})).*$" % (
-        delimiters,
-        delimiters,
+    regex = r"^.*(([\d]{{4}}){delim}([\d]{{2}}){delim}([\d]{{2}})).*$".format(
+        delim=delimiters
     )
     text = str(t)
     res = re.search(regex, text)
     if res is not None:
         res = sum([len(range(reg[0], reg[1])) for reg in res.regs[1:2]])
-        texts = [re.sub(regex, r"\%d" % i, text) for i in range(2, 5)]
+        texts = [re.sub(regex, r"\{}".format(i), text) for i in range(2, 5)]
     else:
         texts = t
         res = 0
@@ -38,7 +37,7 @@ class YYYYMMDDDateCleaner(BaseCleaner):
 
     """
 
-    def __init__(self):
+    def __init__(self, column_sharer=None):
         transformations = [_split_to_new_cols]
 
         def make_list_of_three(x):
@@ -55,4 +54,6 @@ def make_list_of_three(x):
             return [x, "", ""]
 
         default = make_list_of_three
-        super().__init__(transformations, default=default)
+        super().__init__(
+            transformations, default=default, column_sharer=column_sharer
+        )

foreshadow/cleaners/internals/drop.py

@@ -35,9 +35,9 @@ class DropCleaner(BaseCleaner):
 
     """
 
-    def __init__(self):
+    def __init__(self, column_sharer=None):
         transformations = [drop_transform]
-        super().__init__(transformations)
+        super().__init__(transformations, column_sharer=column_sharer)
 
     def transform(self, X, y=None):
         """Clean string columns.

foreshadow/cleaners/internals/financial_cleaner.py

@@ -34,6 +34,6 @@ class DollarFinancialCleaner(BaseCleaner):
 
     """
 
-    def __init__(self):
+    def __init__(self, column_sharer=None):
         transformations = [financial_transform]
-        super().__init__(transformations)
+        super().__init__(transformations, column_sharer=column_sharer)

foreshadow/cleaners/internals/json_flattener.py

@@ -47,7 +47,7 @@ def json_flatten(text):
     try:
         ret = json.loads(text)
         matched = len(text)
-    except json.JSONDecodeError:
+    except (json.JSONDecodeError, TypeError):
         pass  # didn't match.
     return ret, matched
 
@@ -59,6 +59,6 @@ class StandardJsonFlattener(BaseCleaner):
 
     """
 
-    def __init__(self):
+    def __init__(self, column_sharer=None):
         transformations = [json_flatten]
-        super().__init__(transformations)
+        super().__init__(transformations, column_sharer=column_sharer)

foreshadow/config.py

@@ -50,7 +50,7 @@ def get_config(base):
 
 
 def reset_config():
-    """Reset internal configuration
+    """Reset internal configuration.
 
     Note:
         This is useful in an IDLE setting when the configuration file might

foreshadow/core/logging.py

@@ -278,6 +278,24 @@ def info(*args, **kwargs):
     return log(*args, **kwargs)
 
 
+def warning(*args, **kwargs):
+    """Log warning message.
+
+    Manually overriding so that this method is explicitly a part of this
+    module.
+
+    Args:
+        *args: To logging.info
+        **kwargs: To logging.info
+
+    Returns:
+        logging.info
+
+    """
+    log = _wrap_log(_log, "warning")
+    return log(*args, **kwargs)
+
+
 def log_and_gui(level, msg, gui_details, gui_schema, *args, **kwargs):
     """Log msg to gui at specific level and write gui_details under gui_schema.