From cbde3a9faecc0fe6138675ec3565e6446bd9eaa8 Mon Sep 17 00:00:00 2001 From: Winston Chang Date: Fri, 29 Sep 2023 13:39:40 -0500 Subject: [PATCH 01/20] Basic quarto extension functionality --- shiny/__init__.py | 2 + shiny/_app.py | 18 +- shiny/_main.py | 11 ++ shiny/quarto/__init__.py | 8 + shiny/quarto/_ipynb.py | 205 +++++++++++++++++++++++ shiny/quarto/_output.py | 8 + shiny/render/_render.py | 11 +- shiny/render/transformer/_transformer.py | 5 + 8 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 shiny/quarto/__init__.py create mode 100644 shiny/quarto/_ipynb.py create mode 100644 shiny/quarto/_output.py diff --git a/shiny/__init__.py b/shiny/__init__.py index 12c95b494..269fb3c6e 100644 --- a/shiny/__init__.py +++ b/shiny/__init__.py @@ -5,6 +5,7 @@ from ._shinyenv import is_pyodide as _is_pyodide # User-facing subpackages that should be available on `from shiny import *` +from . import quarto from . import reactive from . import render from .session import ( @@ -37,6 +38,7 @@ __all__ = ( # public sub-packages + "quarto", "reactive", "render", "session", diff --git a/shiny/_app.py b/shiny/_app.py index 4715a2a93..3ec491717 100644 --- a/shiny/_app.py +++ b/shiny/_app.py @@ -3,6 +3,7 @@ import copy import os import secrets +from pathlib import Path from typing import Any, Callable, Optional, cast import starlette.applications @@ -89,7 +90,7 @@ def server(input: Inputs, output: Outputs, session: Session): def __init__( self, - ui: Tag | TagList | Callable[[Request], Tag | TagList], + ui: Tag | TagList | Callable[[Request], Tag | TagList] | Path, server: Optional[Callable[[Inputs, Outputs, Session], None]], *, static_assets: Optional["str" | "os.PathLike[str]"] = None, @@ -146,6 +147,12 @@ def _server(inputs: Inputs, outputs: Outputs, session: Session): raise TypeError("App UI cannot be a coroutine function") # Dynamic UI: just store the function for later self.ui = cast("Callable[[Request], Tag | TagList]", ui) + elif isinstance(ui, Path): + if not ui.is_absolute(): + raise ValueError("Path to UI must be absolute") + with open(ui, "r") as f: + page_html = f.read() + self.ui = {"html": page_html, "dependencies": []} else: # Static UI: render the UI now and save the results self.ui = self._render_page( @@ -371,8 +378,13 @@ def _render_page(self, ui: Tag | TagList, lib_prefix: str) -> RenderedHTML: return rendered -def is_uifunc(x: Tag | TagList | Callable[[Request], Tag | TagList]): - if isinstance(x, Tag) or isinstance(x, TagList) or not callable(x): +def is_uifunc(x: Path | Tag | TagList | Callable[[Request], Tag | TagList]): + if ( + isinstance(x, Path) + or isinstance(x, Tag) + or isinstance(x, TagList) + or not callable(x) + ): return False else: return True diff --git a/shiny/_main.py b/shiny/_main.py index f53e51392..303c3cf77 100644 --- a/shiny/_main.py +++ b/shiny/_main.py @@ -480,6 +480,17 @@ def static_assets(command: str) -> None: raise click.UsageError(f"Unknown command: {command}") +@main.command(help="""Convert an ipynb to py file.""") +@click.argument("file", type=str) +def convert(file: str) -> None: + shiny.quarto.convert_ipynb_to_py(file) + + +@main.command(help="""Get Shiny's HTML dependencies as JSON.""") +def get_shiny_deps() -> None: + print(shiny.quarto.get_shiny_deps()) + + class ReloadArgs(TypedDict): reload: NotRequired[bool] reload_includes: NotRequired[list[str]] diff --git a/shiny/quarto/__init__.py b/shiny/quarto/__init__.py new file mode 100644 index 000000000..3e1e3c518 --- /dev/null +++ b/shiny/quarto/__init__.py @@ -0,0 +1,8 @@ +from ._ipynb import convert_ipynb_to_py, get_shiny_deps +from ._output import output_shim + +__all__ = ( + "convert_ipynb_to_py", + "get_shiny_deps", + "output_shim", +) diff --git a/shiny/quarto/_ipynb.py b/shiny/quarto/_ipynb.py new file mode 100644 index 000000000..13b60577b --- /dev/null +++ b/shiny/quarto/_ipynb.py @@ -0,0 +1,205 @@ +"""Tools for parsing ipynb files.""" +from __future__ import annotations + +from pathlib import Path + +__all__ = ( + "convert_ipynb_to_py", + "get_shiny_deps", +) + +from typing import TYPE_CHECKING, Literal, cast + +from .._typing_extensions import NotRequired, TypedDict +from ..html_dependencies import jquery_deps, shiny_deps + +# from htmltools import HTMLDependency + + +class NbCellCodeOutputStream(TypedDict): + output_type: Literal["stream"] + name: Literal["stdout", "stderr"] + text: list[str] + + +class NbCellCodeOutputDisplayData(TypedDict): + output_type: Literal["display_data"] + metadata: dict[str, object] + data: dict[str, object] + + +class NbCellCodeOutputExecuteResult(TypedDict): + output_type: Literal["execute_result"] + execution_count: int + metadata: dict[str, object] + data: dict[str, object] + + +NbCellCodeOutput = ( + NbCellCodeOutputStream | NbCellCodeOutputDisplayData | NbCellCodeOutputExecuteResult +) + + +class NbCellCode(TypedDict): + cell_type: Literal["code"] + execution_count: int | None + id: str + metadata: dict[str, object] + source: str | list[str] + outputs: list[NbCellCodeOutput] + + +class NbCellMarkdown(TypedDict): + cell_type: Literal["markdown"] + metadata: dict[str, object] + source: str | list[str] + + +class NbCellRaw(TypedDict): + cell_type: Literal["raw"] + metadata: dict[str, object] + source: str | list[str] + + +NbCell = NbCellCode | NbCellMarkdown | NbCellRaw + + +class Ipynb(TypedDict): + cells: list[NbCell] + metadata: dict[str, object] + nbformat: int + nbformat_minor: int + + +def convert_ipynb_to_py(file: str | Path) -> None: + """Parse an ipynb file.""" + import json + + file = Path(file) + + with open(file, "r") as f: + nb = cast(Ipynb, json.load(f)) + + cells = nb["cells"] + + code_cell_sources: list[str] = [] + + for cell in cells: + if cell["cell_type"] != "code": + continue + + if "skip" in cell["metadata"] and cell["metadata"]["skip"] is True: + continue + + code_cell_sources.append( + " " + + " ".join(cell["source"]) + + "\n\n # ============================\n" + ) + + app_content = f""" +from pathlib import Path +from shiny import App, Inputs, Outputs, Session, ui + +def server(input: Inputs, output: Outputs, session: Session) -> None: +{ "".join(code_cell_sources) } + +app = App( + Path(__file__).parent / "{ file.with_suffix(".html").name }", + server, + static_assets=Path(__file__).parent, +) + """ + + # print(app_content) + + with open("app.py", "w") as f: + f.write(app_content) + + +# ============================================================================= +# HTML Dependency types +# ============================================================================= +class QuartoHtmlDepItem(TypedDict): + name: str + path: str + attribs: NotRequired[dict[str, str]] + + +class QuartoHtmlDepServiceworkerItem(TypedDict): + source: str + destination: str + + +class QuartoHtmlDependency(TypedDict): + name: str + version: NotRequired[str] + scripts: NotRequired[list[str | QuartoHtmlDepItem]] + stylesheets: NotRequired[list[str | QuartoHtmlDepItem]] + resources: NotRequired[list[QuartoHtmlDepItem]] + meta: NotRequired[dict[str, str]] + serviceworkers: NotRequired[list[QuartoHtmlDepServiceworkerItem]] + + +if TYPE_CHECKING: + from htmltools._core import ScriptItem, StylesheetItem + + +def get_shiny_deps() -> str: + import json + + deps = [ + jquery_deps().as_dict(lib_prefix=None, include_version=False), + shiny_deps().as_dict(lib_prefix=None, include_version=False), + ] + + # Convert from htmltools format to quarto format + for dep in deps: + if "script" in dep: + dep["scripts"] = [htmltools_to_quarto_script(s) for s in dep.pop("script")] + if "stylesheet" in dep: + dep["stylesheets"] = [ + htmltools_to_quarto_stylesheet(s) for s in dep.pop("stylesheet") + ] + + del dep["meta"] + + return json.dumps(deps, indent=2) + + +_shared_dir = (Path(__file__) / ".." / "..").resolve() / "www" / "shared" + + +def htmltools_to_quarto_script(dep: ScriptItem) -> QuartoHtmlDepItem: + if dep["src"].startswith("shiny"): + src = str(remove_first_dir(Path(dep["src"]))) + else: + src = str(Path(dep["src"])) + # NOTE: htmltools always prepends a directory after .as_dict() is called, so we'll + # remove the first directory part. + # src = str(remove_first_dir(Path(dep["src"]))) + + return { + "name": src, + "path": str(_shared_dir / src), + } + + +def htmltools_to_quarto_stylesheet(dep: StylesheetItem) -> QuartoHtmlDepItem: + src = str(remove_first_dir(Path(dep["href"]))) + return { + "name": src, + "path": str(_shared_dir / src), + } + + +def remove_first_dir(p: Path) -> Path: + """Remove the first directory from a Path""" + parts = p.parts + + # If there's only one part (just a filename), return it as is + if len(parts) == 1: + return p + + # Otherwise, skip the first directory and reconstruct the path + return Path(parts[1]).joinpath(*parts[2:]) diff --git a/shiny/quarto/_output.py b/shiny/quarto/_output.py new file mode 100644 index 000000000..497d7cce0 --- /dev/null +++ b/shiny/quarto/_output.py @@ -0,0 +1,8 @@ +"""Shim for @output""" +from __future__ import annotations + +__all__ = ("output_shim",) + + +def output_shim(x: object) -> object: + return x diff --git a/shiny/render/_render.py b/shiny/render/_render.py index 00beb901f..6bee649c8 100644 --- a/shiny/render/_render.py +++ b/shiny/render/_render.py @@ -89,7 +89,16 @@ def text( -------- ~shiny.ui.output_text """ - return TextTransformer(_fn) + text_transformer_fn = TextTransformer(_fn) + + def _output_ui_() -> str: + from .. import ui + + return ui.output_text_verbatim(_fn.__name__, placeholder=True)._repr_html_() + + text_transformer_fn._repr_html_ = _output_ui_ + + return text_transformer_fn # ====================================================================================== diff --git a/shiny/render/transformer/_transformer.py b/shiny/render/transformer/_transformer.py index f206a8221..6d9bf956e 100644 --- a/shiny/render/transformer/_transformer.py +++ b/shiny/render/transformer/_transformer.py @@ -219,6 +219,8 @@ def __init__( *, value_fn: ValueFn[IT], transform_fn: TransformFn[IT, P, OT], + # TODO: Type should be Tag | TagChild | TagList ... + output_fn: Optional[Callable[[], object]] = None, params: TransformerParams[P], ) -> None: """ @@ -314,6 +316,8 @@ def __init__( value_fn: ValueFnSync[IT], transform_fn: TransformFn[IT, P, OT], params: TransformerParams[P], + # TODO: Type should be Tag | TagChild | TagList ... + output_fn: Optional[Callable[[], object]] = None, ) -> None: if is_async_callable(value_fn): raise TypeError( @@ -323,6 +327,7 @@ def __init__( super().__init__( value_fn=value_fn, transform_fn=transform_fn, + output_fn=output_fn, params=params, ) From 7e360252ebe5a7142e026667ab862b21c677dc9e Mon Sep 17 00:00:00 2001 From: Winston Chang Date: Tue, 3 Oct 2023 13:28:22 -0500 Subject: [PATCH 02/20] Inject HTML dependencies using placeholder --- shiny/_app.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/shiny/_app.py b/shiny/_app.py index 3ec491717..8f5b3ec67 100644 --- a/shiny/_app.py +++ b/shiny/_app.py @@ -11,7 +11,14 @@ import starlette.middleware import starlette.routing import starlette.websockets -from htmltools import HTMLDependency, HTMLDocument, RenderedHTML, Tag, TagList +from htmltools import ( + HTMLDependency, + HTMLDocument, + HTMLTextDocument, + RenderedHTML, + Tag, + TagList, +) from starlette.requests import Request from starlette.responses import HTMLResponse, JSONResponse, Response from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -150,9 +157,9 @@ def _server(inputs: Inputs, outputs: Outputs, session: Session): elif isinstance(ui, Path): if not ui.is_absolute(): raise ValueError("Path to UI must be absolute") - with open(ui, "r") as f: - page_html = f.read() - self.ui = {"html": page_html, "dependencies": []} + + self.ui = self._render_page_from_file(ui, lib_prefix=self.lib_prefix) + else: # Static UI: render the UI now and save the results self.ui = self._render_page( @@ -377,6 +384,22 @@ def _render_page(self, ui: Tag | TagList, lib_prefix: str) -> RenderedHTML: self._ensure_web_dependencies(rendered["dependencies"]) return rendered + def _render_page_from_file(self, file: Path, lib_prefix: str) -> RenderedHTML: + with open(file, "r") as f: + page_html = f.read() + + doc = HTMLTextDocument( + page_html, + deps=[require_deps(), jquery_deps(), shiny_deps()], + deps_replace_pattern='', + ) + + rendered = doc.render(lib_prefix=lib_prefix) + self._ensure_web_dependencies(rendered["dependencies"]) + + # TODO: scan for inlined