Add mimetype renderer (#195)

* Rename vegafusion-jupyter renderer to vegafusion-widget
Rename vf.jupyter.enable to vf.jupyter.enable_widget()

* Add initial vegafusion-inline transformer and vegafusion-mime renderer

* Move mime renderer to vegafusion package
Add top-level vf.enable_mime()/enable_widget() methods

* Add local timezone configuration

* Add eval_transforms function that returns the result of evaluating the transforms on a Chart

* Don't use editable install in build:dev to get nbextension working for development

* Add row_limit to pre_transform_values

* Add html, svg, and png render mime types

* Test html mimetype

* Add transformed_dtypes

* Initial support for row/column encoding facets

* Lookup Altair's vegalite version dynamically

* Thread embed options through to mime renderer

* Merge embed options and fix bundle embed metadata

* work around to avoid white background in dark theme

* Support custom vegalite-to-vega compiler plugins

* Add better error message in local_tz if vl-convert is not available

* Install vl-convert-python in CI

* Derive Debug for Plan

* For selection stores with inline data, keep values on both client and server

* Fix "dispatch dropped without returning error" test error

Sharing a reqwest Client across tokio runtimes can result in "dispatch dropped without returning error" error. This removes the lazy_static client and reconstructs the client on demand

* Check client-only-vars in determining if signal is supported

.github/workflows/build_test.yml

@@ -341,6 +341,7 @@ jobs:
           python -m pip install vegafusion_python*${{ matrix.options[4] }}*${{ matrix.options[3] }}*.whl
           python -m pip install vegafusion-*.whl
           python -m pip install vegafusion_jupyter*.whl
+          python -m pip install vl-convert-python
       - name: Test vegafusion
         working-directory: python/vegafusion/
         run: pytest

python/vegafusion-jupyter/package.json

@@ -32,7 +32,7 @@
     "url": "https://github.com/jonmmease/vegafusion"
   },
   "scripts": {
-    "build:dev": "npm run build:lib && npm run build:nbextension && npm run build:labextension:dev && pip install --force-reinstall --no-deps -e .",
+    "build:dev": "npm run build:lib && npm run build:nbextension && npm run build:labextension:dev && pip install --force-reinstall --no-deps .",
     "build:prod": "npm run clean && npm run build:lib && npm run build:nbextension && npm run build:labextension",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",

python/vegafusion-jupyter/tests/test_altair_mocks.py

@@ -46,28 +46,28 @@
 ```
 """
 
-vegafusion_feather_markdown_template = r"""
+vegafusion_widget_feather_markdown_template = r"""
 ```python
 import altair as alt
-import vegafusion_jupyter as vf
-vf.enable()
+import vegafusion as vf
+vf.enable_widget()
 ```
 
 ```python
 {code}
 ```
 
 ```python
-assert(alt.renderers.active == "vegafusion-jupyter")
+assert(alt.renderers.active == "vegafusion-widget")
 assert(alt.data_transformers.active == 'vegafusion-feather')
 ```
 """
 
-vegafusion_default_markdown_template = r"""
+vegafusion_widget_default_markdown_template = r"""
 ```python
 import altair as alt
-import vegafusion_jupyter as vf
-vf.enable()
+import vegafusion as vf
+vf.enable_widget()
 alt.data_transformers.enable("default");
 ```
 
@@ -76,11 +76,27 @@
 ```
 
 ```python
-assert(alt.renderers.active == "vegafusion-jupyter")
+assert(alt.renderers.active == "vegafusion-widget")
 assert(alt.data_transformers.active == 'default')
 ```
 """
 
+vegafusion_mime_markdown_template = r"""
+```python
+import altair as alt
+import vegafusion as vf
+vf.enable_mime(mimetype="html", embed_options={'actions': False})
+```
+
+```python
+{code}
+```
+
+```python
+assert(alt.renderers.active == "vegafusion-mime")
+assert(alt.data_transformers.active == 'vegafusion-inline')
+```
+"""
 
 def setup_module(module):
     """ setup any state specific to the execution of the given module."""
@@ -265,13 +281,15 @@ def test_altair_mock(mock_name, img_tolerance, delay):
 
     mock_code = mock_path.read_text()
     altair_markdown = altair_markdown_template.replace("{code}", mock_code)
-    vegafusion_arrow_markdown = vegafusion_feather_markdown_template.replace("{code}", mock_code)
-    vegafusion_default_markdown = vegafusion_default_markdown_template.replace("{code}", mock_code)
+    vegafusion_arrow_markdown = vegafusion_widget_feather_markdown_template.replace("{code}", mock_code)
+    vegafusion_default_markdown = vegafusion_widget_default_markdown_template.replace("{code}", mock_code)
+    vegafusion_mime_markdown = vegafusion_mime_markdown_template.replace("{code}", mock_code)
 
     # Use jupytext to convert markdown to an ipynb file
     altair_notebook = jupytext.read(io.StringIO(altair_markdown), fmt="markdown")
     vegafusion_arrow_notebook = jupytext.read(io.StringIO(vegafusion_arrow_markdown), fmt="markdown")
     vegafusion_default_notebook = jupytext.read(io.StringIO(vegafusion_default_markdown), fmt="markdown")
+    vegafusion_mime_notebook = jupytext.read(io.StringIO(vegafusion_mime_markdown), fmt="markdown")
 
     # Create selenium Chrome instance
     chrome_opts = webdriver.ChromeOptions()
@@ -296,26 +314,38 @@ def test_altair_mock(mock_name, img_tolerance, delay):
             chrome_driver, vegafusion_arrow_notebook, name + "_vegafusion_feather", actions, delay
         )
         vegafusion_default_imgs = export_image_sequence(
-            chrome_driver, vegafusion_default_notebook, name + "_vegafusion", actions, delay
+            chrome_driver, vegafusion_default_notebook, name + "_vegafusion_widget", actions, delay
+        )
+        vegafusion_mime_imgs = export_image_sequence(
+            chrome_driver, vegafusion_mime_notebook, name + "_vegafusion_mime", actions, delay
         )
 
         for i in range(len(altair_imgs)):
             altair_img = altair_imgs[i]
             vegafusion_arrow_img = vegafusion_arrow_imgs[i]
             vegafusion_default_img = vegafusion_default_imgs[i]
+            vegafusion_mime_img = vegafusion_mime_imgs[i]
 
             assert altair_img.shape == vegafusion_arrow_img.shape, "Size mismatch with Arrow data transformer"
             assert altair_img.shape == vegafusion_default_img.shape, "Size mismatch with default data transformer"
+            assert altair_img.shape == vegafusion_mime_img.shape, "Size mismatch with mime renderer"
 
             similarity_arrow_value = ssim(altair_img, vegafusion_arrow_img, channel_axis=2)
             similarity_default_value = ssim(altair_img, vegafusion_default_img, channel_axis=2)
+            similarity_mime_value = ssim(altair_img, vegafusion_mime_img, channel_axis=2)
 
             print(f"({i}) similarity_arrow_value={similarity_arrow_value}")
             print(f"({i}) similarity_default_value={similarity_default_value}")
+            print(f"({i}) similarity_mime_value={similarity_mime_value}")
 
             assert similarity_arrow_value >= img_tolerance, f"Similarity failed with Arrow data transformer on image {i}"
             assert similarity_default_value >= img_tolerance, f"Similarity failed with default data transformer on image {i}"
 
+            # Allow slightly more image tolerance for mime renderer as floating point differences may
+            # be introduced by pre-transform process
+            mime_image_tolerance = img_tolerance * 0.99
+            assert similarity_mime_value >= mime_image_tolerance, f"Similarity failed with mime renderer on image {i}"
+
     finally:
         voila_proc.kill()
         chrome_driver.close()

python/vegafusion-jupyter/vegafusion_jupyter/__init__.py

@@ -11,6 +11,7 @@
 from .widget import VegaFusionWidget
 from ._version import __version__
 
+
 def enable(
         download_source_link=None,
         debounce_wait=30,
@@ -27,7 +28,7 @@ def enable(
     # will be registered
     import vegafusion.transformer
     alt.renderers.enable(
-        'vegafusion-jupyter',
+        'vegafusion-widget',
         debounce_wait=debounce_wait,
         debounce_max_wait=debounce_max_wait,
         download_source_link=download_source_link,
@@ -39,7 +40,7 @@ def enable(
 
 def disable():
     """
-    Disable the VegaFusion data transformer and renderer so that Charts
+    Disable the VegaFusion data transformers and renderers so that Charts
     are not displayed using VegaFusion
 
     Equivalent to
@@ -52,8 +53,8 @@ def disable():
 
     This does not affect the behavior of VegaFusionWidget
     """
-    alt.renderers.enable('default')
-    alt.data_transformers.enable('default')
+    from vegafusion import disable
+    disable()
 
 
 def _jupyter_labextension_paths():

python/vegafusion-jupyter/vegafusion_jupyter/renderer.py

@@ -20,5 +20,4 @@ def vegafusion_renderer(spec, **widget_options):
     display(widget)
     return {'text/plain': ""}
 
-
-alt.renderers.register('vegafusion-jupyter', vegafusion_renderer)
+alt.renderers.register('vegafusion-widget', vegafusion_renderer)

python/vegafusion-jupyter/vegafusion_jupyter/widget.py

@@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs):
             else:
                 data_transformer_opts = dict()
 
-            with alt.renderers.enable("vegafusion-jupyter"):
+            with alt.renderers.enable("vegafusion-widget"):
                 with alt.data_transformers.enable("vegafusion-feather", **data_transformer_opts):
                     # Temporarily enable the vegafusion renderer and transformer so
                     # that we use them even if they are not enabled globally
@@ -73,7 +73,7 @@ def __init__(self, *args, **kwargs):
             kwargs["spec"] = json.dumps(kwargs["spec"], indent=2)
 
         # If vegafusion renderer is already enabled, use the configured debounce options as the default
-        if alt.renderers.active == "vegafusion-jupyter":
+        if alt.renderers.active == "vegafusion-widget":
             # Use configured debounce options, if any
             renderer_opts = alt.renderers.options
             for opt in ["debounce_wait", "debounce_max_wait", "download_source_link"]:

python/vegafusion/vegafusion/__init__.py

@@ -5,10 +5,14 @@
 # Please consult the license documentation provided alongside
 # this program the details of the active license.
 from .runtime import runtime
-from .transformer import to_feather
-from ._version import __version__
+from .transformer import to_feather, get_inline_datasets_for_spec
+from .local_tz import set_local_tz, get_local_tz
+from . import renderer
+from .compilers import vegalite_compilers
+import altair as alt
 
-# Import subpackages
+from ._version import __version__
+# Import optional subpackages
 try:
     import vegafusion.jupyter
 except ImportError:
@@ -18,3 +22,77 @@
     import vegafusion.embed
 except ImportError:
     pass
+
+
+def altair_vl_version(vl_convert=False):
+    """
+    Get Altair's preferred Vega-Lite version
+
+    :param vl_convert: If True, return a version string compatible with vl_convert
+        (e.g. v4_17 rather than 4.17.0)
+    :return: str with Vega-Lite version
+    """
+    from altair.vegalite.v4 import SCHEMA_VERSION
+    if vl_convert:
+        # Compute VlConvert's vl_version string (of the form 'v5_2')
+        # from SCHEMA_VERSION (of the form 'v5.2.0')
+        return "_".join(SCHEMA_VERSION.split(".")[:2])
+    else:
+        # Return full version without leading v
+        return SCHEMA_VERSION.rstrip("v")
+
+
+def enable_mime(mimetype="html", embed_options=None):
+    """
+    Enable the VegaFusion data transformer and renderer so that all Charts
+    are displayed using VegaFusion.
+
+    This isn't necessary in order to use the VegaFusionWidget directly
+
+    :param mimetype: Mime type. One of:
+        - "html" (default)
+        - "vega"
+        - "svg"
+        - "png": Note: the PNG renderer can be quite slow for charts with lots of marks
+    :param embed_options: dict (optional)
+        Dictionary of options to pass to the vega-embed. Default
+        entry is {'mode': 'vega'}.
+    """
+    # Import vegafusion.transformer so that vegafusion-inline transform
+    # will be registered
+    alt.renderers.enable('vegafusion-mime', mimetype=mimetype, embed_options=embed_options)
+    alt.data_transformers.enable('vegafusion-inline')
+
+
+def enable_widget(
+    download_source_link=None,
+    debounce_wait=30,
+    debounce_max_wait=60,
+    data_dir="_vegafusion_data"
+):
+    from vegafusion.jupyter import enable
+    enable(
+        download_source_link=download_source_link,
+        debounce_wait=debounce_wait,
+        debounce_max_wait=debounce_max_wait,
+        data_dir=data_dir
+    )
+
+
+def disable():
+    """
+    Disable the VegaFusion data transformers and renderers so that Charts
+    are not displayed using VegaFusion
+
+    Equivalent to
+
+    ```python
+    import altair as alt
+    alt.renderers.enable('default')
+    alt.data_transformers.enable('default')
+    ```
+
+    This does not affect the behavior of VegaFusionWidget
+    """
+    alt.renderers.enable('default')
+    alt.data_transformers.enable('default')

python/vegafusion/vegafusion/compilers.py

@@ -0,0 +1,29 @@
+from altair.utils.plugin_registry import PluginRegistry
+from typing import Callable
+
+
+VegaLiteCompilerType = Callable[..., dict]
+
+
+class VegaLiteCompilerRegistry(PluginRegistry[VegaLiteCompilerType]):
+    pass
+
+
+vegalite_compilers = VegaLiteCompilerRegistry()
+
+
+def vl_convert_compiler(vegalite_spec) -> dict:
+    try:
+        import vl_convert as vlc
+    except ImportError:
+        raise ImportError(
+            "The vl-convert Vega-Lite compiler requires the vl-convert-python package"
+        )
+
+    from . import altair_vl_version
+    vega_spec = vlc.vegalite_to_vega(vegalite_spec, vl_version=altair_vl_version(vl_convert=True))
+    return vega_spec
+
+
+vegalite_compilers.register("vl-convert", vl_convert_compiler)
+vegalite_compilers.enable("vl-convert")

python/vegafusion/vegafusion/local_tz.py

@@ -0,0 +1,41 @@
+__tz_config = dict(local_tz=None)
+
+
+def get_local_tz():
+    """
+    Get the named local timezone that the VegaFusion mimetype renderer
+    will use for calculations.
+
+    Defaults to the kernel's local timezone as determined by vl-convert.
+
+    Has no effect on VegaFusionWidget, which always uses the
+    browser's local timezone
+
+    :return: named timezone string
+    """
+    if __tz_config["local_tz"] is None:
+        # Fall back to getting local_tz from vl-convert if not set
+        try:
+            import vl_convert as vlc
+            __tz_config["local_tz"] = vlc.get_local_tz()
+        except ImportError:
+            raise ImportError(
+                "vl-convert is not installed and so the local system timezone cannot be determined.\n"
+                "Either install the vl-convert-python package or set the local timezone manually using\n"
+                "the vegafusion.set_local_tz function"
+            )
+
+    return __tz_config["local_tz"]
+
+
+def set_local_tz(local_tz):
+    """
+    Set the named local timezone that the VegaFusion mimetype renderer
+    will use for calculations.
+
+    Has no effect on VegaFusionWidget, which always uses the
+    browser's local timezone
+
+    :param local_tz: Named local timezone (e.g. "America/New_York")
+    """
+    __tz_config["local_tz"] = local_tz