Convert how_to

holoviz · Apr 19, 2023 · 072351b · 072351b
1 parent be74c92
commit 072351b
Show file tree

Hide file tree

Showing 2 changed files with 118 additions and 207 deletions.
diff --git a/scripts/generate_panelite_content.json b/scripts/generate_panelite_content.json
@@ -1,200 +1,49 @@
 {
-    "developer_guide/Developing_Custom_Models.ipynb": [],
-    "gallery/apis/stocks_altair.ipynb": ["panel"],
-    "gallery/apis/stocks_hvplot.ipynb": ["panel", "hvplot"],
-    "gallery/apis/stocks_matplotlib.ipynb": ["panel"],
-    "gallery/apis/stocks_plotly.ipynb": ["panel", "plotly"],
-    "gallery/components/CanvasDraw.ipynb": ["panel"],
-    "gallery/components/LeafletHeatMap.ipynb": ["panel"],
-    "gallery/components/MaterialUI.ipynb": ["panel"],
-    "gallery/components/VuePdbInput.ipynb": ["panel", "pyodide-http"],
-    "gallery/demos/attractors.ipynb": [],
-    "gallery/demos/gapminders.ipynb": [],
-    "gallery/demos/glaciers.ipynb": [],
-    "gallery/demos/nyc_taxi.ipynb": [],
-    "gallery/demos/portfolio-optimizer.ipynb": [],
-    "gallery/demos/VTKInteractive.ipynb": ["panel", "pyvista"],
-    "gallery/demos/VTKSlicer.ipynb": ["panel", "holoviews", "scipy"],
-    "gallery/demos/VTKWarp.ipynb": ["panel"],
-    "gallery/dynamic/dynamic_plot_layout.ipynb": ["panel", "holoviews"],
-    "gallery/dynamic/dynamic_timeseries_image_analysis.ipynb": ["panel", "holoviews", "hvplot"],
-    "gallery/dynamic/dynamic_ui.ipynb": ["panel"],
-    "gallery/dynamic/dynamic_widget_values.ipynb": ["panel"],
-    "gallery/external/DataTable.ipynb": ["panel", "pandas"],
-    "gallery/external/deck.gl.ipynb": ["panel"],
-    "gallery/external/Folium.ipynb": ["panel", "pandas", "folium"],
-    "gallery/featured/gapminders.ipynb": ["panel", "altair", "plotly", "hvplot"],
-    "gallery/featured/glaciers.ipynb": ["panel", "holoviews", "colorcet", "hvplot"],
-    "gallery/featured/nyc_deckgl.ipynb": ["panel", "fastparquet", "pyodide-http"],
-    "gallery/featured/penguin_crossfilter.ipynb": ["panel", "holoviews", "hvplot"],
-    "gallery/featured/penguin_kmeans.ipynb": ["panel", "altair"],
-    "gallery/featured/portfolio_optimizer.ipynb": ["panel", "holoviews", "hvplot", "scipy"],
-    "gallery/featured/windturbines.ipynb": ["panel", "hvplot", "holoviews", "fastparquet", "pyodide-http"],
-    "gallery/layout/distribution_tabs.ipynb": ["panel", "holoviews"],
-    "gallery/layout/dynamic_tabs.ipynb": ["panel", "altair", "vega-datasets", "holoviews", "hvplot", "matplotlib", "plotly"],
-    "gallery/layout/plot_with_columns.ipynb": ["panel", "hvplot"],
-    "gallery/links/bokeh_property_editor.ipynb": ["panel"],
-    "gallery/links/deck_gl_json_editor.ipynb": ["panel"],
-    "gallery/links/holoviews_glyph_link.ipynb": ["panel", "holoviews"],
-    "gallery/links/plotly_link.ipynb": ["panel", "plotly"],
-    "gallery/links/vega_heatmap_link.ipynb": ["panel"],
-    "gallery/param/action_button.ipynb": ["panel"],
-    "gallery/param/deck_gl_global_power_plants.ipynb": ["pandas", "panel", "pydeck", "pyodide-http"],
-    "gallery/param/download_upload_csv.ipynb": ["panel", "plotly"],
-    "gallery/param/loading_indicator.ipynb": ["panel", "holoviews"],
-    "gallery/param/param_subobjects.ipynb": ["panel"],
-    "gallery/param/precedence.ipynb": ["panel"],
-    "gallery/param/reactive_plots.ipynb": ["panel", "hvplot"],
-    "gallery/param/reactive_tables.ipynb": ["panel", "pandas"],
-    "gallery/simple/altair_brushing.ipynb": ["panel", "altair"],
-    "gallery/simple/altair_choropleth.ipynb": ["panel", "altair", "vega-datasets"],
-    "gallery/simple/clifford_interact.ipynb": ["panel", "datashader", "colorcet"],
-    "gallery/simple/color_speech_recognition.ipynb": ["panel"],
-    "gallery/simple/deckgl_game_of_life.ipynb": ["panel"],
-    "gallery/simple/defer_data_load.ipynb": ["panel", "hvplot"],
-    "gallery/simple/file_download_examples.ipynb": ["panel", "XlsxWriter", "fastparquet", "pyodide-http"],
-    "gallery/simple/hvplot_explorer.ipynb": ["panel", "hvplot", "scipy"],
-    "gallery/simple/iris_kmeans.ipynb": ["panel", "hvplot", "scikit-learn"],
-    "gallery/simple/loading_spinner.ipynb": ["panel", "holoviews"],
-    "gallery/simple/random_number_generator.ipynb": ["panel"],
-    "gallery/simple/save_filtered_df.ipynb": ["panel", "pandas"],
-    "gallery/simple/sync_location.ipynb": ["panel"],
-    "gallery/simple/temperature_distribution.ipynb": ["panel", "hvplot", "holoviews", "scipy"],
-    "gallery/simple/xgboost_classifier.ipynb": ["panel", "pandas", "scikit-learn", "xgboost"],
-    "gallery/streaming/hardware_automation.ipynb": ["panel", "holoviews"],
-    "gallery/streaming/streaming_bokeh.ipynb": ["panel"],
-    "gallery/streaming/streaming_indicator.ipynb": ["panel"],
-    "gallery/streaming/streaming_perspective.ipynb": ["panel"],
-    "gallery/streaming/streaming_tabulator.ipynb": ["panel"],
-    "gallery/streaming/streaming_videostream.ipynb": ["panel", "pillow", "scikit-image"],
-    "gallery/styles/MatplotlibStyle.ipynb": ["panel"],
-    "gallery/styles/PlotlyStyle.ipynb": ["panel", "plotly"],
-    "gallery/styles/SeabornStyle.ipynb": ["panel", "seaborn"],
-    "gallery/styles/VegaAltairStyle.ipynb": ["panel", "altair", "vega-datasets"],
-    "gallery/templates/material_template.ipynb": ["panel", "holoviews"],
-    "gallery/viz/GraphViz.ipynb": ["panel", "graphviz"],
-    "gallery/viz/NetworkX.ipynb": ["panel", "networkx", "pygraphviz"],
-    "reference/global/Notifications.ipynb": ["panel"],
-    "reference/indicators/BooleanStatus.ipynb": ["panel"],
-    "reference/indicators/Dial.ipynb": ["panel"],
-    "reference/indicators/Gauge.ipynb": ["panel"],
-    "reference/indicators/LinearGauge.ipynb": ["panel"],
-    "reference/indicators/LoadingSpinner.ipynb": ["panel"],
-    "reference/indicators/Number.ipynb": ["panel"],
-    "reference/indicators/Progress.ipynb": ["panel"],
-    "reference/indicators/Tqdm.ipynb": ["panel"],
-    "reference/indicators/Trend.ipynb": ["panel"],
-    "reference/layouts/Accordion.ipynb": ["panel"],
-    "reference/layouts/Card.ipynb": ["panel"],
-    "reference/layouts/Column.ipynb": ["panel"],
-    "reference/layouts/Divider.ipynb": ["panel"],
-    "reference/layouts/FlexBox.ipynb": ["panel"],
-    "reference/layouts/GridBox.ipynb": ["panel"],
-    "reference/layouts/GridSpec.ipynb": ["panel", "holoviews"],
-    "reference/layouts/GridStack.ipynb": ["panel", "holoviews"],
-    "reference/layouts/Row.ipynb": ["panel"],
-    "reference/layouts/Swipe.ipynb": ["panel"],
-    "reference/layouts/Tabs.ipynb": ["panel"],
-    "reference/layouts/WidgetBox.ipynb": ["panel"],
-    "reference/panes/Alert.ipynb": ["panel"],
-    "reference/panes/Audio.ipynb": ["panel", "scipy"],
-    "reference/panes/Bokeh.ipynb": ["panel"],
-    "reference/panes/DataFrame.ipynb": ["panel", "streamz"],
-    "reference/panes/DeckGL.ipynb": ["panel", "pydeck"],
-    "reference/panes/ECharts.ipynb": ["panel", "pyecharts"],
-    "reference/panes/Folium.ipynb": ["panel", "folium"],
-    "reference/panes/GIF.ipynb": ["panel"],
-    "reference/panes/HoloViews.ipynb": ["panel", "holoviews", "hvplot", "matplotlib", "plotly", "scipy"],
-    "reference/panes/HTML.ipynb": ["panel"],
-    "reference/panes/Image.ipynb": ["panel"],
-    "reference/panes/IPyWidget.ipynb": ["panel", "ipywidgets", "ipyvolume", "ipyleaflet"],
-    "reference/panes/JPG.ipynb": ["panel"],
-    "reference/panes/JSON.ipynb": ["panel"],
-    "reference/panes/LaTeX.ipynb": ["panel"],
-    "reference/panes/Markdown.ipynb": ["panel"],
-    "reference/panes/Matplotlib.ipynb": ["panel", "matplotlib", "ipywidgets", "ipympl"],
-    "reference/panes/Param.ipynb": ["panel", "hvplot"],
-    "reference/panes/PDF.ipynb": ["panel"],
-    "reference/panes/Perspective.ipynb": ["panel"],
-    "reference/panes/Plotly.ipynb": ["panel", "plotly"],
-    "reference/panes/PNG.ipynb": ["panel"],
-    "reference/panes/Reacton.ipynb": ["panel", "reacton", "pandas", "jupyter_bokeh"],
-    "reference/panes/Str.ipynb": ["panel"],
-    "reference/panes/Streamz.ipynb": ["panel", "streamz", "altair", "pandas"],
-    "reference/panes/SVG.ipynb": ["panel"],
-    "reference/panes/Vega.ipynb": ["panel", "altair", "vega_datasets"],
-    "reference/panes/Video.ipynb": ["panel"],
-    "reference/panes/Vizzu.ipynb": ["panel"],
-    "reference/panes/VTK.ipynb": ["panel", "vtk", "pyvista"],
-    "reference/panes/VTKJS.ipynb": ["panel", "pyodide-http"],
-    "reference/panes/VTKVolume.ipynb": ["panel", "pyvista"],
-    "reference/templates/Bootstrap.ipynb": ["panel", "holoviews"],
-    "reference/templates/FastGridTemplate.ipynb": ["panel", "holoviews"],
-    "reference/templates/FastListTemplate.ipynb": ["panel", "holoviews"],
-    "reference/templates/GoldenLayout.ipynb": ["panel", "holoviews"],
-    "reference/templates/Material.ipynb": ["panel", "holoviews"],
-    "reference/templates/React.ipynb": ["panel", "holoviews"],
-    "reference/templates/Vanilla.ipynb": ["panel", "holoviews"],
-    "reference/widgets/ArrayInput.ipynb": ["panel"],
-    "reference/widgets/AutocompleteInput.ipynb": ["panel"],
-    "reference/widgets/Button.ipynb": ["panel"],
-    "reference/widgets/Checkbox.ipynb": ["panel"],
-    "reference/widgets/CheckBoxGroup.ipynb": ["panel"],
-    "reference/widgets/CheckButtonGroup.ipynb": ["panel"],
-    "reference/widgets/CodeEditor.ipynb": ["panel"],
-    "reference/widgets/ColorPicker.ipynb": ["panel"],
-    "reference/widgets/CrossSelector.ipynb": ["panel"],
-    "reference/widgets/DataFrame.ipynb": ["panel"],
-    "reference/widgets/DatePicker.ipynb": ["panel"],
-    "reference/widgets/DateRangeSlider.ipynb": ["panel"],
-    "reference/widgets/DateSlider.ipynb": ["panel"],
-    "reference/widgets/DatetimeInput.ipynb": ["panel"],
-    "reference/widgets/DatetimePicker.ipynb": ["panel"],
-    "reference/widgets/DatetimeRangeInput.ipynb": ["panel"],
-    "reference/widgets/DatetimeRangePicker.ipynb": ["panel"],
-    "reference/widgets/DatetimeRangeSlider.ipynb": ["panel"],
-    "reference/widgets/Debugger.ipynb": ["panel"],
-    "reference/widgets/DiscretePlayer.ipynb": ["panel"],
-    "reference/widgets/DiscreteSlider.ipynb": ["panel"],
-    "reference/widgets/EditableFloatSlider.ipynb": ["panel"],
-    "reference/widgets/EditableIntSlider.ipynb": ["panel"],
-    "reference/widgets/EditableRangeSlider.ipynb": ["panel"],
-    "reference/widgets/FileDownload.ipynb": ["panel", "pandas"],
-    "reference/widgets/FileInput.ipynb": ["panel"],
-    "reference/widgets/FileSelector.ipynb": ["panel"],
-    "reference/widgets/FloatInput.ipynb": ["panel"],
-    "reference/widgets/FloatSlider.ipynb": ["panel"],
-    "reference/widgets/IntInput.ipynb": ["panel"],
-    "reference/widgets/IntRangeSlider.ipynb": ["panel"],
-    "reference/widgets/IntSlider.ipynb": ["panel"],
-    "reference/widgets/JSONEditor.ipynb": ["panel"],
-    "reference/widgets/LiteralInput.ipynb": ["panel"],
-    "reference/widgets/MenuButton.ipynb": ["panel"],
-    "reference/widgets/MultiChoice.ipynb": ["panel"],
-    "reference/widgets/MultiSelect.ipynb": ["panel"],
-    "reference/widgets/PasswordInput.ipynb": ["panel"],
-    "reference/widgets/Player.ipynb": ["panel"],
-    "reference/widgets/RadioBoxGroup.ipynb": ["panel"],
-    "reference/widgets/RadioButtonGroup.ipynb": ["panel"],
-    "reference/widgets/RangeSlider.ipynb": ["panel"],
-    "reference/widgets/Select.ipynb": ["panel"],
-    "reference/widgets/SpeechToText.ipynb": ["panel"],
-    "reference/widgets/StaticText.ipynb": ["panel"],
-    "reference/widgets/Switch.ipynb": ["panel"],
-    "reference/widgets/Tabulator.ipynb": ["panel", "pandas"],
-    "reference/widgets/Terminal.ipynb": ["panel"],
-    "reference/widgets/TextAreaInput.ipynb": ["panel"],
-    "reference/widgets/TextEditor.ipynb": ["panel"],
-    "reference/widgets/TextInput.ipynb": ["panel"],
-    "reference/widgets/TextToSpeech.ipynb": ["panel"],
-    "reference/widgets/Toggle.ipynb": ["panel"],
-    "reference/widgets/ToggleGroup.ipynb": ["panel"],
-    "reference/widgets/VideoStream.ipynb": ["panel"],
-    "user_guide/Async_and_Concurrency.ipynb": ["panel"],
-    "user_guide/Custom_Components.ipynb": ["panel"],
-    "user_guide/Display_and_Export.ipynb": ["panel"],
-    "user_guide/Performance_and_Debugging.ipynb": ["panel", "aiohttp"],
-    "user_guide/Session_State_and_Callbacks.ipynb": ["croniter"],
-    "user_guide/Templates.ipynb": ["panel", "holoviews"]
-
+  "gallery/altair_brushing.ipynb": ["altair"],
+  "gallery/gapminders.ipynb": ["altair", "plotly", "hvplot"],
+  "gallery/glaciers.ipynb": ["holoviews", "colorcet", "hvplot"],
+  "gallery/hvplot_explorer.ipynb": ["hvplot", "scipy"],
+  "gallery/iris_kmeans.ipynb": ["hvplot", "scikit-learn"],
+  "gallery/nyc_deckgl.ipynb": ["fastparquet", "pyodide-http"],
+  "gallery/penguin_crossfilter.ipynb": ["holoviews", "hvplot"],
+  "gallery/penguin_kmeans.ipynb": ["altair"],
+  "gallery/portfolio_optimizer.ipynb": ["holoviews", "hvplot", "scipy"],
+  "gallery/streaming_videostream.ipynb": ["pillow", "scikit-image"],
+  "gallery/VTKInteractive.ipynb": ["pyvista"],
+  "gallery/VTKSlicer.ipynb": ["holoviews", "scipy"],
+  "gallery/windturbines.ipynb": ["hvplot", "holoviews", "fastparquet", "pyodide-http"],
+  "gallery/xgboost_classifier.ipynb": ["pandas", "scikit-learn", "xgboost"],
+  "how_to/apis/examples/stocks_hvplot.ipynb": ["hvplot"],
+  "how_to/apis/examples/stocks_plotly.ipynb": ["plotly"],
+  "how_to/custom_components/plot_viewer.ipynb": ["hvplot"],
+  "how_to/io/download_simple.ipynb": ["XlsxWriter", "fastparquet"],
+  "how_to/layout/examples/dynamic_tabs.ipynb": ["altair", "vega-datasets", "holoviews", "hvplot", "matplotlib", "plotly"],
+  "how_to/links/examples/holoviews_glyph_link.ipynb": ["holoviews"],
+  "how_to/links/examples/plotly_link.ipynb": ["plotly"],
+  "how_to/param/examples/loading.ipynb": ["holoviews"],
+  "reference/layouts/GridSpec.ipynb": ["holoviews"],
+  "reference/layouts/GridStack.ipynb": ["holoviews"],
+  "reference/panes/Audio.ipynb": ["scipy"],
+  "reference/panes/DataFrame.ipynb": ["streamz"],
+  "reference/panes/DeckGL.ipynb": ["pydeck"],
+  "reference/panes/ECharts.ipynb": ["pyecharts"],
+  "reference/panes/Folium.ipynb": ["folium"],
+  "reference/panes/HoloViews.ipynb": ["holoviews", "hvplot", "matplotlib", "plotly", "scipy"],
+  "reference/panes/IPyWidget.ipynb": ["ipywidgets", "ipyvolume", "ipyleaflet"],
+  "reference/panes/Matplotlib.ipynb": ["matplotlib", "ipywidgets", "ipympl"],
+  "reference/panes/Param.ipynb": ["hvplot"],
+  "reference/panes/Plotly.ipynb": ["plotly"],
+  "reference/panes/Reacton.ipynb": ["reacton", "pandas", "jupyter_bokeh"],
+  "reference/panes/Streamz.ipynb": ["streamz", "altair", "pandas"],
+  "reference/panes/Vega.ipynb": ["altair", "vega_datasets"],
+  "reference/panes/VTK.ipynb": ["vtk", "pyvista"],
+  "reference/panes/VTKJS.ipynb": ["pyodide-http"],
+  "reference/panes/VTKVolume.ipynb": ["pyvista"],
+  "reference/templates/Bootstrap.ipynb": ["holoviews"],
+  "reference/templates/FastGridTemplate.ipynb": ["holoviews"],
+  "reference/templates/FastListTemplate.ipynb": ["holoviews"],
+  "reference/templates/GoldenLayout.ipynb": ["holoviews"],
+  "reference/templates/Material.ipynb": ["holoviews"],
+  "reference/templates/React.ipynb": ["holoviews"],
+  "reference/templates/Vanilla.ipynb": ["holoviews"]
 }
diff --git a/scripts/generate_panelite_content.py b/scripts/generate_panelite_content.py
@@ -14,7 +14,8 @@
 
 PANEL_BASE = pathlib.Path(__file__).parent.parent
 EXAMPLES_DIR = PANEL_BASE / 'examples'
-DEFAULT_DEPENDENCIES = ["panel"]
+DOC_DIR = PANEL_BASE / 'doc'
+BASE_DEPENDENCIES = ["panel", "pyodide-http"]
 
 # Add piplite command to notebooks
 with open(PANEL_BASE/"scripts"/"generate_panelite_content.json", "r", encoding="utf8") as file:
@@ -24,19 +25,19 @@ class DependencyNotFound(Exception):
     """Raised if a dependency cannot be found"""
 
 def _notebook_key(nbpath: pathlib.Path):
-    return str(nbpath).replace("\\", "/").split("examples/")[-1]
+    nbpath = str(nbpath).replace(os.path.sep, "/")
+
+    return nbpath.split("examples/")[-1]
 
 def _get_dependencies(nbpath: pathlib.Path):
     key = _notebook_key(nbpath)
-    try:
-        dependencies = DEPENDENCIES[key]
-    except KeyError as ex:
-        print(f"WARNING: Could not find the dependencies for '{key}'. Please add them")
-        dependencies = DEFAULT_DEPENDENCIES
-    return dependencies
+    dependencies = DEPENDENCIES.get(key, [])
+    if dependencies is None:
+        return []
+    return BASE_DEPENDENCIES + dependencies
 
 def _to_piplite_install_code(dependencies):
-    dependencies = [f"'{dependency}'" for dependency in dependencies]
+    dependencies = [repr(dep) for dep in dependencies]
     return f"import piplite\nawait piplite.install([{', '.join(dependencies)}])"
 
 def _get_install_code_cell(dependencies):
@@ -81,6 +82,66 @@ def _get_info_markdown_cell(nbpath):
     del info['id']
     return info
 
+def convert_md_to_nb(
+    filehandle, supported_syntax=('{pyodide}',)
+) -> str:
+    """
+    Extracts Panel application code from a Markdown file.
+    """
+    nb = nbformat.v4.new_notebook()
+    cells = nb['cells']
+    inblock = False
+    block_opener = None
+    markdown, code = [], []
+    while True:
+        line = filehandle.readline()
+        if not line:
+            # EOF
+            break
+
+        lsline = line.lstrip()
+        if inblock:
+            if lsline.startswith(block_opener):
+                inblock = False
+                code_cell = nbformat.v4.new_code_cell(source=''.join(code))
+                code.clear()
+                cells.append(code_cell)
+            else:
+                code.append(line)
+        elif lsline.startswith("```"):
+            num_leading_backticks = len(lsline) - len(lsline.lstrip("`"))
+            block_opener = '`'*num_leading_backticks
+            syntax = line.strip()[num_leading_backticks:]
+            if syntax in supported_syntax:
+                if markdown:
+                    md_cell = nbformat.v4.new_markdown_cell(source=''.join(markdown))
+                    markdown.clear()
+                    nb['cells'].append(md_cell)
+                inblock = True
+            else:
+                markdown.append(line)
+        else:
+            markdown.append(line)
+    return nb
+
+def convert_howto():
+    mds = list(DOC_DIR.glob('how_to/**/*.md'))
+    for md in mds:
+        out = (PANEL_BASE / 'lite/files') / md.relative_to(DOC_DIR).with_suffix('.ipynb')
+        out.parent.mkdir(parents=True, exist_ok=True)
+        if md.suffix != '.md':
+            continue
+        with open(md, encoding="utf-8") as mdf:
+            nb = convert_md_to_nb(mdf)
+        dependencies = _get_dependencies(md)
+        if dependencies:
+            install = _get_install_code_cell(dependencies=dependencies)
+            nb['cells'].insert(0, install)
+            info = _get_info_markdown_cell(md)
+            nb['cells'].insert(0, info)
+        with open(out, 'w', encoding='utf-8') as fout:
+            nbformat.write(nb, fout)
+
 def copy_examples():
     nbs = list(EXAMPLES_DIR.glob('*/*/*.ipynb')) + list(EXAMPLES_DIR.glob('*/*.*'))
     for nb in nbs:
@@ -144,6 +205,7 @@ def download_sample_data():
         _download_file(s3, filename, data_dir, progress=False)
 
 if __name__=="__main__":
+    convert_howto()
     copy_examples()
     copy_assets()
     download_sample_data()