From f4e708d48be82b55cd00ec831abf153b026c2253 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Mon, 27 Jul 2020 13:57:30 -0400 Subject: [PATCH 1/7] examples: Add Binder-compatible Jupyter notebook example --- .binder/.dockerignore | 1 + .binder/Dockerfile | 58 ++++++++++++ .binder/README.md | 30 +++++++ .dockerignore | 1 + docs/examples/notebook/example.ipynb | 128 +++++++++++++++++++++++++++ 5 files changed, 218 insertions(+) create mode 100644 .binder/.dockerignore create mode 100644 .binder/Dockerfile create mode 100644 .binder/README.md create mode 120000 .dockerignore create mode 100644 docs/examples/notebook/example.ipynb diff --git a/.binder/.dockerignore b/.binder/.dockerignore new file mode 100644 index 00000000..b43bf86b --- /dev/null +++ b/.binder/.dockerignore @@ -0,0 +1 @@ +README.md diff --git a/.binder/Dockerfile b/.binder/Dockerfile new file mode 100644 index 00000000..42df500b --- /dev/null +++ b/.binder/Dockerfile @@ -0,0 +1,58 @@ +# -*- mode: dockerfile -*- +# vi: set ft=dockerfile : + +FROM ubuntu:18.04 + +ARG NB_USER=jovyan +ARG NB_UID=1000 +ARG NB_GID=100 +EXPOSE 7000/tcp +EXPOSE 8888/tcp + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew \ + -o Dpkg::Use-Pty=0 \ + jupyter-notebook \ + locales \ + python3-pip \ + python3-setuptools \ + wget \ + && rm -rf /var/lib/apt/lists/* \ + && locale-gen en_US.UTF-8 + +# Install later version of castxml. +# See README here: https://github.com/CastXML/CastXMLSuperbuild/tree/75ec9ef4ad48ddab605627d783bfdee57fd7bcbf +# This is v0.3.4 for Linux: https://data.kitware.com/#item/5ee7eb659014a6d84ec1f25c +RUN wget https://data.kitware.com/api/v1/file/5ee7eb659014a6d84ec1f25e/download -O /tmp/castxml.tar.gz \ + && tar xfz /tmp/castxml.tar.gz -C /usr/local --strip-components 1 \ + && rm /tmp/castxml.tar.gz + +# Install common C++ libraries for experimenting. These are not necessary to +# use pygccxml itself. +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew \ + -o Dpkg::Use-Pty=0 \ + libeigen3-dev \ + libstdc++-7-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -d "/home/$NB_USER" -G $NB_GID -mU -s /bin/bash "$NB_USER" +ENV HOME="/home/$NB_USER" \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + SHELL=/bin/bash \ + USER="$NB_USER" + +WORKDIR $HOME +RUN mkdir pygccxml +COPY ["/", "pygccxml/"] +RUN chown -R $NB_UID:$NB_GID \ + "$HOME/pygccxml" +USER "$NB_USER" +RUN pip3 --no-cache-dir install -e ./pygccxml +CMD ["jupyter", "notebook", "--ip", "0.0.0.0", "pygccxml/docs/examples/notebook/example.ipynb"] diff --git a/.binder/README.md b/.binder/README.md new file mode 100644 index 00000000..8a479284 --- /dev/null +++ b/.binder/README.md @@ -0,0 +1,30 @@ +# Docker Image for Binder + + + +*Note that due to Binder conventions, this directory MUST always be in the root +of the repository and named either `binder` or `.binder`. This image is NOT +intended for use by most developers or users.* + +To create a Docker image and run a Docker container similar to those used by +[Binder](https://mybinder.org) for local debugging purposes, execute the +following `pull`, `build`, and `run` commands from the top level of this Git +repository: + +```bash +cd pygccxml +docker build -f .binder/Dockerfile -t binder . +docker run --rm -it --name mybinder -p 8888:8888 binder +``` + +Copy and paste the URL (including the login token) that is displayed in the +terminal into the web browser of your choice. + +To stop the running container, simply exit it from the terminal with Ctrl+C. + +*Note*: If you want to test the Docker image with the current source tree +(without copying, so you can modify source files), add the arguments +`-v ${PWD}:/home/jovyan/pygccxml` to mount it directly. diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/docs/examples/notebook/example.ipynb b/docs/examples/notebook/example.ipynb new file mode 100644 index 00000000..245aa619 --- /dev/null +++ b/docs/examples/notebook/example.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Example pygccxml notebook on Binder\n", + "\n", + "Running this notebook on Binder allows you to execute the code online.\n", + "\n", + "Please note that provisioning may take about 1-2 minutes.\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "The following example shows an example usage of `pygccxml` on Ubuntu\n", + "Bionic, from within a Docker container, for a simple toy C++ API that\n", + "uses both `std::vector` and `Eigen::Matrix<>`. The code is defined inline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pygccxml import declarations\n", + "from pygccxml import utils\n", + "from pygccxml import parser" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Find out the c++ parser. This should resolve to the castxml\n", + "# version installed in Docker.\n", + "generator_path, generator_name = utils.find_xml_generator()\n", + "\n", + "# Configure the xml generator\n", + "config = parser.xml_generator_configuration_t(\n", + " xml_generator_path=generator_path,\n", + " xml_generator=generator_name,\n", + " include_paths=[\"/usr/include/eigen3\"],\n", + " # TODO(eric.cousineau): Why is `compiler_path` necessary?\n", + " compiler_path=generator_path,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "code = r\"\"\"\n", + "#include \n", + "\n", + "#include \n", + "\n", + "namespace ns {\n", + "\n", + "template \n", + "class ExampleClass {\n", + "public:\n", + " std::vector make_std_vector() const;\n", + " Eigen::Matrix make_matrix3();\n", + "};\n", + "\n", + "// Analyze concrete instantiations of the given class.\n", + "extern template class ExampleClass;\n", + "extern template class ExampleClass;\n", + "\n", + "} // namespace ns\n", + "\"\"\"\n", + "\n", + "(global_ns,) = parser.parse_string(code, config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "ns = global_ns.namespace(\"ns\")\n", + "declarations.print_declarations([ns])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve an instantiation and show template parameters.\n", + "cls, = ns.classes('ExampleClass')\n", + "declarations.templates.split(cls.name)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 88637f987b535f5a41f0f365ec8043dc9c8bf1ff Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Fri, 31 Jul 2020 13:03:36 -0400 Subject: [PATCH 2/7] TO BE SQUASHED jupyterlab --- .binder/Dockerfile | 8 ++++---- setup.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.binder/Dockerfile b/.binder/Dockerfile index 42df500b..537b6243 100644 --- a/.binder/Dockerfile +++ b/.binder/Dockerfile @@ -14,7 +14,6 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends \ -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew \ -o Dpkg::Use-Pty=0 \ - jupyter-notebook \ locales \ python3-pip \ python3-setuptools \ @@ -46,7 +45,8 @@ ENV HOME="/home/$NB_USER" \ LANGUAGE=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 \ SHELL=/bin/bash \ - USER="$NB_USER" + USER="$NB_USER" \ + PATH="/home/$NB_USER/.local/bin:/usr/local/bin:/usr/bin:/bin" WORKDIR $HOME RUN mkdir pygccxml @@ -54,5 +54,5 @@ COPY ["/", "pygccxml/"] RUN chown -R $NB_UID:$NB_GID \ "$HOME/pygccxml" USER "$NB_USER" -RUN pip3 --no-cache-dir install -e ./pygccxml -CMD ["jupyter", "notebook", "--ip", "0.0.0.0", "pygccxml/docs/examples/notebook/example.ipynb"] +RUN pip3 --no-cache-dir install -e ./pygccxml[examples] +CMD ["jupyter", "lab", "--ip", "0.0.0.0", "pygccxml/docs/examples/notebook/example.ipynb"] diff --git a/setup.py b/setup.py index a647e238..6c5b039d 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,9 @@ "sphinx", "sphinx_rtd_theme", } +requirements_examples = { + "jupyterlab", +} setup(name="pygccxml", version=version, @@ -38,6 +41,7 @@ extras_require={ "test": list(requirements_test), "docs": list(requirements_docs), + "examples": list(requirements_examples), }, classifiers=[ "Development Status :: 5 - Production/Stable", From 608f224005f4f7800f0d5b39fdc856329bd85c32 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Thu, 19 Nov 2020 12:01:09 -0500 Subject: [PATCH 3/7] try jupyterlab --- .binder/Dockerfile | 3 +++ README.rst | 14 ++++++++++++++ docs/examples/notebook/example.ipynb | 9 ++++----- setup.py | 1 + 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.binder/Dockerfile b/.binder/Dockerfile index 537b6243..5efd4ba5 100644 --- a/.binder/Dockerfile +++ b/.binder/Dockerfile @@ -1,6 +1,9 @@ # -*- mode: dockerfile -*- # vi: set ft=dockerfile : +# TODO(eric.cousineau): Figure out how to make JupyterLab work with this setup: +# https://github.com/binder-examples/jupyterlab + FROM ubuntu:18.04 ARG NB_USER=jovyan diff --git a/README.rst b/README.rst index 167c3b4d..69e26aeb 100644 --- a/README.rst +++ b/README.rst @@ -33,9 +33,23 @@ Documentation and examples -------------------------- The documentation can be found `here `_, examples can be found `here `_. +You can also run an example JupyterLab Notebook using Binder, or view it using +``nbviewer``: + +.. + Developers: See `.binder/README.md` for more information. + +.. image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/EricCousineau-TRI/pygccxml/feature-py-notebook-example?urlpath=lab/tree/docs/examples/notebook/ + :alt: Binder +.. image:: https://img.shields.io/badge/view%20on-nbviewer-brightgreen.svg + :target: https://nbviewer.jupyter.org/github/EricCousineau-TRI/pygccxml/tree/feature-py-notebook-example/docs/examples/notebook/ + :alt: nbviewer If you want to know more about the API provided by pygccxml, read the `query interface `_ document or the `API documentation `_. + + A `FAQ `_ is also available and may answer some of your questions. License diff --git a/docs/examples/notebook/example.ipynb b/docs/examples/notebook/example.ipynb index 245aa619..b7605c29 100644 --- a/docs/examples/notebook/example.ipynb +++ b/docs/examples/notebook/example.ipynb @@ -6,13 +6,13 @@ "source": [ "# Example pygccxml notebook on Binder\n", "\n", - "Running this notebook on Binder allows you to execute the code online.\n", + "Running this notebook on Binder allows you to execute the code online:", + "\n", + " \n", + "\n", "\n", "Please note that provisioning may take about 1-2 minutes.\n", "\n", - "\n", - " \n", - "\n", "\n", "The following example shows an example usage of `pygccxml` on Ubuntu\n", "Bionic, from within a Docker container, for a simple toy C++ API that\n", @@ -45,7 +45,6 @@ " xml_generator_path=generator_path,\n", " xml_generator=generator_name,\n", " include_paths=[\"/usr/include/eigen3\"],\n", - " # TODO(eric.cousineau): Why is `compiler_path` necessary?\n", " compiler_path=generator_path,\n", ")" ] diff --git a/setup.py b/setup.py index 6c5b039d..4539a321 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ } requirements_examples = { "jupyterlab", + "jupyterlab_launcher", } setup(name="pygccxml", From 5998a72af7bd2645ad330222835fef86333f9af9 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Thu, 19 Nov 2020 12:24:18 -0500 Subject: [PATCH 4/7] give up on JupyterLab; leave as TODO --- .binder/Dockerfile | 5 +++++ README.rst | 2 +- docs/examples/notebook/example.ipynb | 2 +- setup.py | 1 - 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.binder/Dockerfile b/.binder/Dockerfile index 5efd4ba5..34c14a6b 100644 --- a/.binder/Dockerfile +++ b/.binder/Dockerfile @@ -4,6 +4,11 @@ # TODO(eric.cousineau): Figure out how to make JupyterLab work with this setup: # https://github.com/binder-examples/jupyterlab +# TODO(eric.cousineau): See if it's easier to use a conda-based workflow, or a +# simpler Docker base image, to use Eigen headers, rather than doing a custom +# Docker image: +# https://mybinder.readthedocs.io/en/latest/using/config_files.html + FROM ubuntu:18.04 ARG NB_USER=jovyan diff --git a/README.rst b/README.rst index 69e26aeb..cf2dc59b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ You can also run an example JupyterLab Notebook using Binder, or view it using Developers: See `.binder/README.md` for more information. .. image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/EricCousineau-TRI/pygccxml/feature-py-notebook-example?urlpath=lab/tree/docs/examples/notebook/ + :target: https://mybinder.org/v2/gh/EricCousineau-TRI/pygccxml/feature-py-notebook-example?urlpath=tree/pygccxml/docs/examples/notebook/ :alt: Binder .. image:: https://img.shields.io/badge/view%20on-nbviewer-brightgreen.svg :target: https://nbviewer.jupyter.org/github/EricCousineau-TRI/pygccxml/tree/feature-py-notebook-example/docs/examples/notebook/ diff --git a/docs/examples/notebook/example.ipynb b/docs/examples/notebook/example.ipynb index b7605c29..68c31f25 100644 --- a/docs/examples/notebook/example.ipynb +++ b/docs/examples/notebook/example.ipynb @@ -7,7 +7,7 @@ "# Example pygccxml notebook on Binder\n", "\n", "Running this notebook on Binder allows you to execute the code online:", - "\n", + "\n", " \n", "\n", "\n", diff --git a/setup.py b/setup.py index 4539a321..6c5b039d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ } requirements_examples = { "jupyterlab", - "jupyterlab_launcher", } setup(name="pygccxml", From 54f2cda6a658c3d7fc1b05379d127f9c9316c8e7 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Thu, 18 Feb 2021 09:06:11 -0500 Subject: [PATCH 5/7] address review --- .binder/Dockerfile | 17 ++++++++--------- .binder/README.md | 11 ++++++----- setup.py | 20 ++++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.binder/Dockerfile b/.binder/Dockerfile index 34c14a6b..86c1c86b 100644 --- a/.binder/Dockerfile +++ b/.binder/Dockerfile @@ -25,17 +25,9 @@ RUN export DEBIAN_FRONTEND=noninteractive \ locales \ python3-pip \ python3-setuptools \ - wget \ && rm -rf /var/lib/apt/lists/* \ && locale-gen en_US.UTF-8 -# Install later version of castxml. -# See README here: https://github.com/CastXML/CastXMLSuperbuild/tree/75ec9ef4ad48ddab605627d783bfdee57fd7bcbf -# This is v0.3.4 for Linux: https://data.kitware.com/#item/5ee7eb659014a6d84ec1f25c -RUN wget https://data.kitware.com/api/v1/file/5ee7eb659014a6d84ec1f25e/download -O /tmp/castxml.tar.gz \ - && tar xfz /tmp/castxml.tar.gz -C /usr/local --strip-components 1 \ - && rm /tmp/castxml.tar.gz - # Install common C++ libraries for experimenting. These are not necessary to # use pygccxml itself. RUN export DEBIAN_FRONTEND=noninteractive \ @@ -56,11 +48,18 @@ ENV HOME="/home/$NB_USER" \ USER="$NB_USER" \ PATH="/home/$NB_USER/.local/bin:/usr/local/bin:/usr/bin:/bin" +# Upgrade pip to use newer indices for castxml. +# WARNING: Never upgrade a distribution `pip` on a host system using sudo! +# We are only doing this for a transient Docker image. For a host system, use a +# virtualenv to upgrade pip. +RUN pip3 --no-cache-dir install -U pip + WORKDIR $HOME RUN mkdir pygccxml COPY ["/", "pygccxml/"] RUN chown -R $NB_UID:$NB_GID \ "$HOME/pygccxml" USER "$NB_USER" +RUN pip3 --no-cache-dir install castxml RUN pip3 --no-cache-dir install -e ./pygccxml[examples] -CMD ["jupyter", "lab", "--ip", "0.0.0.0", "pygccxml/docs/examples/notebook/example.ipynb"] +CMD ["jupyter", "notebook", "--ip", "0.0.0.0", "pygccxml/docs/examples/notebook/example.ipynb"] diff --git a/.binder/README.md b/.binder/README.md index 8a479284..698eaf9c 100644 --- a/.binder/README.md +++ b/.binder/README.md @@ -9,19 +9,20 @@ https://github.com/RobotLocomotion/drake/tree/dc2a9394d/.binder of the repository and named either `binder` or `.binder`. This image is NOT intended for use by most developers or users.* +These instructions are for running the image locally. For Binder itself, you +should only need to visit the link from the root-level README. + To create a Docker image and run a Docker container similar to those used by [Binder](https://mybinder.org) for local debugging purposes, execute the -following `pull`, `build`, and `run` commands from the top level of this Git -repository: +following `build` and `run` commands from the top level of this Git repository: ```bash -cd pygccxml docker build -f .binder/Dockerfile -t binder . docker run --rm -it --name mybinder -p 8888:8888 binder ``` -Copy and paste the URL (including the login token) that is displayed in the -terminal into the web browser of your choice. +For the URLs printed, only open the `127.0.0.1:8888` URL (including the login +token) in a web browser on your host system. To stop the running container, simply exit it from the terminal with Ctrl+C. diff --git a/setup.py b/setup.py index 6c5b039d..0501b1d9 100644 --- a/setup.py +++ b/setup.py @@ -9,18 +9,18 @@ version = utils.find_version("../pygccxml/__init__.py") -requirements_test = { +requirements_test = [ "coverage", "coveralls", "pycodestyle", -} -requirements_docs = { +] +requirements_docs = [ "sphinx", "sphinx_rtd_theme", -} -requirements_examples = { - "jupyterlab", -} +] +requirements_examples = [ + "notebook", +] setup(name="pygccxml", version=version, @@ -39,9 +39,9 @@ "pygccxml.parser", "pygccxml.utils"], extras_require={ - "test": list(requirements_test), - "docs": list(requirements_docs), - "examples": list(requirements_examples), + "test": requirements_test, + "docs": requirements_docs, + "examples": requirements_examples, }, classifiers=[ "Development Status :: 5 - Production/Stable", From 3117ee1f08e4b33d4ec6322c6390dbbec63e2e1f Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Sat, 20 Feb 2021 10:44:09 -0500 Subject: [PATCH 6/7] address review --- .binder/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.binder/README.md b/.binder/README.md index 698eaf9c..fb71df09 100644 --- a/.binder/README.md +++ b/.binder/README.md @@ -27,5 +27,6 @@ token) in a web browser on your host system. To stop the running container, simply exit it from the terminal with Ctrl+C. *Note*: If you want to test the Docker image with the current source tree -(without copying, so you can modify source files), add the arguments -`-v ${PWD}:/home/jovyan/pygccxml` to mount it directly. +(without copying, so you can modify source files), insert the arguments +`-v "${PWD}:/home/jovyan/pygccxml"` before the image name (`binder`) to mount it +directly. This will *not* act on any changes to `./setup.py`. From edfa8d886fc211fbf7aa6ce41358071b367c8095 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Mon, 22 Feb 2021 16:41:21 -0500 Subject: [PATCH 7/7] address review --- .binder/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.binder/README.md b/.binder/README.md index fb71df09..d9f2a820 100644 --- a/.binder/README.md +++ b/.binder/README.md @@ -28,5 +28,6 @@ To stop the running container, simply exit it from the terminal with Ctrl+C. *Note*: If you want to test the Docker image with the current source tree (without copying, so you can modify source files), insert the arguments -`-v "${PWD}:/home/jovyan/pygccxml"` before the image name (`binder`) to mount it -directly. This will *not* act on any changes to `./setup.py`. +`-v "${PWD}:/home/jovyan/pygccxml"` to `docker run`, before the image name +(`binder`), to mount it directly. This will *not* act on any changes to +`./setup.py`.