feature/desktop app (#295)

* Create @stlite/desktop by copying the code from toy-app

* Add electron and set browserslist only with the latest ver of electron in @stlite/desktop

* Set up start and build npm scripts for electron with a minimum files for electron

* Introduce stlite for dev env

* Set packageJSON.homepage for path resolution of the dynamic imports

* Fix CSP setting

* [WIP] Download wheels from remote and mount them on emfs to install

* Configure hot-reloading electron

* Create an IPC, localWheelFiles.read()

* Use path.resolve() in mainWindow.loadFile()

* Fix the sample Python script to use st.text_input

* Load the wheels from the bundled file at the local FS

* Organize NPM scripts and app window

* Save and restore a snapshot

* Revert "Load the wheels from the bundled file at the local FS"

This reverts commit cc187a1.

* Fix craco.config.js to use emotion plugin and fix the CSP for <img>, <video>, and <audio> to work

* Refactoring

* Set up electron-builder

* Fix StliteKernelOptions.wheels to be optional

* Delete build/pypi manually as a workaround

* Download the pyodide files and bundle them to the production release

* Refactoring

* Rename the snapshot file name implying it is only for site-packages

* Bundle the user code and data in the build directory and the app loads it at runtime

* Extend dump_snapshot.ts to copy the app dir and configure the requirements

* Add a comment

* Remove wheelUrls option

* Add comments

* Remove web-vitals

* Remove index.css

* Remove manifest.json and logo*.png

* Fix the title tag

* Remove favicon.ico

* Remove some meta tags unnecessary for Electron

* Remove robots.txt

* Fix the test command

* Configure the CSP for st.map and some components

* Refactoring

* Refactoring

* Set tsconfigJSON.exclude.

* Update README.md

* Set up eslint and prettier

* Set up the CI workflow to run tests on @stlite/desktop

.github/workflows/main.yml

@@ -10,6 +10,7 @@ jobs:
       tornado-e2e: ${{ steps.filter.outputs.tornado-e2e }}
       mountable: ${{ steps.filter.outputs.mountable }}
       sharing-common: ${{ steps.filter.outputs.sharing-common }}
+      desktop: ${{ steps.filter.outputs.desktop }}
     steps:
     - uses: actions/checkout@v3
     - uses: dorny/paths-filter@v2
@@ -27,6 +28,8 @@ jobs:
             - 'packages/mountable/**/*'
           sharing-common:
             - 'packages/sharing-common/**/*'
+          desktop:
+            - 'packages/desktop/**/*'
 
   test-stlite-kernel:
     needs: changes
@@ -198,6 +201,38 @@ jobs:
         yarn check:prettier
     - run: yarn test
 
+  test-desktop:
+    needs: changes
+    if: ${{ needs.changes.outputs.desktop == 'true' }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [16.x]
+
+    defaults:
+      run:
+        working-directory: packages/desktop
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'yarn'
+    - run: yarn install --frozen-lockfile
+    - name: Lint
+      run: |
+        yarn check:eslint
+        yarn check:prettier
+    - run: yarn typecheck
+    - run: yarn test
+
   build-mountable:
     if: ${{ ! failure() }}  # This job should run even if the depending jobs are skipped, but not when those jobs failed: https://qiita.com/abetomo/items/d9ede7dbeeb24f723fc5#%E8%A8%AD%E5%AE%9A%E4%BE%8B4
     needs: [test-stlite-kernel, test-tornado-e2e, test-mountable]

package.json

@@ -12,8 +12,12 @@
   "devDependencies": {
     "@lerna-lite/cli": "^1.11.0",
     "chokidar-cli": "^3.0.0",
+    "concurrently": "^7.4.0",
+    "cross-env": "^7.0.3",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",
-    "prettier": "2.7.1"
+    "prettier": "2.7.1",
+    "rimraf": "^3.0.2",
+    "wait-on": "^6.0.1"
   }
 }

packages/desktop/.env

@@ -0,0 +1,4 @@
+# For Content-Security-Policy compatibility.
+# Ref: https://drag13.io/posts/react-inline-runtimer-chunk/index.html
+INLINE_RUNTIME_CHUNK=false
+IMAGE_INLINE_SIZE_LIMIT=0

packages/desktop/.gitignore

@@ -0,0 +1,25 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+!.env

packages/desktop/.lintstagedrc.json

@@ -0,0 +1,4 @@
+{
+  "*.{js,jsx,ts,tsx}": "eslint --cache --fix",
+  "*.{js,jsx,ts,tsx,css,md,json}": "prettier --write"
+}

packages/desktop/.prettierignore

@@ -0,0 +1,7 @@
+# Ignore artifacts:
+build
+dist
+coverage
+
+# Ignore all HTML files:
+*.html

packages/desktop/README.md

@@ -0,0 +1,53 @@
+## stlite Electron app
+
+This project has been initialized with Create React App v4.0.3, which version is compatible with the Streamlit frontend.
+So the directory structure and configurations are following it.
+The React app in `./src` will be used as a frontend app of the Electron app, running in the renderer process.
+
+Upon it, the source code for the Electron main process has been added at `./electron` that will be built into the same directory as the React app, `./build`.
+
+This project structure is based on the following references.
+
+- [Quick Start - Electron](https://www.electronjs.org/docs/latest/tutorial/quick-start)
+- [Create React App(typescript)をベースに electron 環境を構築する](https://zenn.dev/niwaringo/articles/af693596ef948e)
+- [Electron + React + TypeScript の開発環境構築 (webpack 編)](https://zenn.dev/sprout2000/articles/5d7b350c2e85bc)
+
+## Workflow to build the app
+
+### Build the base app
+
+```sh
+yarn build
+```
+
+This command builds the Electron app including both the main process (from `./electron`) and the renderer process (from `./src` with CRA) into `./build` directory.
+
+At this point, the built app **does not contain the user code and data for Streamlit like `streamlit_app.py`**.
+We will bundle it at the next step.
+
+### Inject the user code and data and the installed requirements snapshot
+
+```
+./bin/dump_snapshot.ts <app source directory> [--requirements <requirement1> <requirement2> ... <requirementN>]
+```
+
+This command will do 2 things;
+
+1. Copy the `<app source directory>` into `./build`, which will be loaded as a Streamlit app at runtime.
+2. Create a temporary Pyodide environment, install the requirements there, create the snapshot file containing the installed files, and put the file into `./build`.
+
+The Electron app built in the previous step will load these files and serve the Streamlit app at runtime.
+
+In other words, we can replace the Streamlit app just by re-running `./bin/dump_snapshot.ts`, without re-building the app here with `yarn build`.
+
+Now the `./build` is ready to be packaged.
+At this point, `yarn serve` can be used to preview the app.
+
+### Package the final Electron app
+
+```
+yarn dist
+```
+
+Create a distributable package using `electron-builder`.
+This set-up is just following [its tutorial](https://www.electron.build/#quick-setup-guide)

packages/desktop/bin/dump_snapshot.ts

@@ -0,0 +1,129 @@
+#!/usr/bin/env yarn ts-node
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import path from "path";
+import fsPromises from "fs/promises";
+import fsExtra from "fs-extra";
+import fetch from "node-fetch";
+import { loadPyodide, PyodideInterface } from "pyodide";
+
+// @ts-ignore
+global.fetch = fetch; // The global `fetch()` is necessary for micropip.install() to load the remote packages.
+
+async function installLocalWheel(pyodide: PyodideInterface, localPath: string) {
+  console.log(`Install the local wheel ${localPath}`);
+
+  const data = await fsPromises.readFile(localPath);
+  const emfsPath = "/tmp/" + path.basename(localPath);
+  pyodide.FS.writeFile(emfsPath, data);
+
+  const micropip = pyodide.pyimport("micropip");
+  const requirement = `emfs:${emfsPath}`;
+  console.log(`Install ${requirement}`);
+  await micropip.install.callKwargs(requirement, { keep_going: true });
+}
+
+interface CreateSitePackagesSnapshotOptions {
+  localWheelPaths: {
+    tornado: string;
+    pyarrow: string;
+    streamlit: string;
+  };
+  requirements: string[];
+  saveTo: string;
+}
+
+async function createSitePackagesSnapshot(
+  options: CreateSitePackagesSnapshotOptions
+) {
+  const pyodide = await loadPyodide();
+
+  await pyodide.loadPackage(["micropip"]);
+
+  await installLocalWheel(pyodide, options.localWheelPaths.tornado);
+  await installLocalWheel(pyodide, options.localWheelPaths.pyarrow);
+  await installLocalWheel(pyodide, options.localWheelPaths.streamlit);
+
+  console.log(
+    `Install the requirements ${JSON.stringify(options.requirements)}`
+  );
+  const micropip = pyodide.pyimport("micropip");
+  await micropip.install.callKwargs(options.requirements, { keep_going: true });
+
+  console.log("Archive the site-packages director(y|ies)");
+  const archiveFilePath = "/tmp/site-packages-snapshot.tar.gz";
+  await pyodide.runPythonAsync(`
+    import tarfile
+    import site
+
+    site_packages_dirs = site.getsitepackages()
+
+    tar_file_name = '${archiveFilePath}'
+    with tarfile.open(tar_file_name, mode='w:gz') as gzf:
+        for site_packages in site_packages_dirs:
+            gzf.add(site_packages)
+  `);
+
+  console.log("Extract the archive file from EMFS");
+  const archiveBin = pyodide.FS.readFile(archiveFilePath);
+
+  console.log("Save the archive file");
+  await fsPromises.writeFile(options.saveTo, archiveBin);
+}
+
+interface CopyHomeDirectoryOptions {
+  sourceDir: string;
+  saveTo: string;
+}
+async function copyHomeDirectory(options: CopyHomeDirectoryOptions) {
+  await fsExtra.ensureDir(options.sourceDir);
+  return fsExtra.copy(options.sourceDir, options.saveTo);
+}
+
+yargs(hideBin(process.argv))
+  .command(
+    "* <appHomeDirSource>",
+    "Put the user code and data and the snapshot of the required packages into the build artifact.",
+    () => {},
+    (argv) => {
+      console.info(argv);
+    }
+  )
+  .positional("appHomeDirSource", {
+    describe:
+      "The source directory of the user code and data that will be mounted in the Pyodide file system at app runtime",
+    type: "string",
+    demandOption: true,
+  })
+  .options("requirements", {
+    array: true,
+    type: "string",
+    alias: "r",
+    default: [],
+  })
+  .parseAsync()
+  .then(async (args) => {
+    await createSitePackagesSnapshot({
+      localWheelPaths: {
+        pyarrow: path.resolve(
+          __dirname,
+          "../../stlite-kernel/py/stlite-pyarrow//dist/stlite_pyarrow-0.1.0-py3-none-any.whl"
+        ),
+        tornado: path.resolve(
+          __dirname,
+          "../../stlite-kernel/py/tornado/dist/tornado-6.2-py3-none-any.whl"
+        ),
+        streamlit: path.resolve(
+          __dirname,
+          "../../../streamlit/lib/dist/streamlit-1.12.0-py2.py3-none-any.whl"
+        ),
+      },
+      requirements: args.requirements,
+      saveTo: "build/site-packages-snapshot.tar.gz", // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
+    });
+    await copyHomeDirectory({
+      sourceDir: args.appHomeDirSource,
+      saveTo: "./build/streamlit_app", // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
+    });
+  });

packages/desktop/bin/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "commonjs",
+    "outDir": ".",
+    "rootDir": ".",
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  },
+  "include": ["."],
+  "exclude": []
+}