dev/upgrade streamlit 1.18.1 (#482)

* Update the streamlit submodule to v1.18.1 * Update the wheel file name * Update react-scripts to 5.0.1 * Replace path with path-browserify in the worker, that were automatically polyfilled by Webpack4 but it isnt' with v5 * Remove worker-loader that has been deprecated for Webpack5 * Update dependencies and craco config in @stlite/sharing adapting to [email protected] * Update dependencies and craco config in @stlite/desktop adapting to [email protected] * Set streamlit-browser dependency version * [WIP] Override mountable/* with files ejected from a newly scaffolded package with CRA5.0.1 * Customize the build config and dependency list for @stlite/mountable upon the ejected files * Update dependency versions following the streamlit-browser package * Fix typing in sharing-editor * Update mountable/DEVELOPMENT.md * Update yarn.lock * Rename the websocket endpoint path following streamlit/streamlit#5534 * Fix packages/kernel/py/e2etests/tests/test_httpserver.py * Fix the CI to run test-tornado-e2e anytime * Fix tornado-e2e tests * Fix CI * Replace @st.experimental_memo with @st.cache_data in the samples * Rename the file upload endpoint path following streamlit/streamlit#5534 * Fix BROWSER envvar value for CRA5 * Fix to avoid forbidden access to LocalStorage due to #476 * Add streamlit-browser@^1.18.1 to the deps list of @stlite/mountable * Fix comments in packages/mountable/config/webpack.config.js * Remove the type declarations for inline raw-loader import * Move the deps list from dependencies to devDependencies in @stlite/mountable * Introduce a hack to handle cross-origin policy of worker creation * Set local package name for dump_artifacts.ts * Fix CSP
whitphx · Feb 17, 2023 · cd556ee · cd556ee
1 parent ac25c06
commit cd556ee
Show file tree

Hide file tree

Showing 56 changed files with 7,178 additions and 8,512 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -7,7 +7,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       kernel: ${{ steps.filter.outputs.kernel }}
-      tornado-e2e: ${{ steps.filter.outputs.tornado-e2e }}
+      # tornado-e2e: ${{ steps.filter.outputs.tornado-e2e }}
+      tornado-e2e: true  # This step does not detect changes in the `streamlit` submodule that is needed to trigger the test-tornado-e2e job (https://github.com/dorny/paths-filter/issues/143), so skip checking and make it always return true as a workaround.
       mountable: ${{ steps.filter.outputs.mountable }}
       sharing-editor: ${{ steps.filter.outputs.sharing-editor }}
       sharing-common: ${{ steps.filter.outputs.sharing-common }}
@@ -21,10 +22,10 @@ jobs:
           kernel:
             - 'packages/kernel/**/*'
             # - '!packages/kernel/py/**/*'  # Not supported by paths-filter now: https://github.com/dorny/paths-filter/issues/106
-          tornado-e2e:
-            - 'packages/kernel/py/tornado'
-            - 'packages/kernel/py/e2etests/**/*'
-            - 'streamlit/**/*'
+          # tornado-e2e:  # We run this job anytime. See above.
+          #   - 'packages/kernel/py/tornado'
+          #   - 'packages/kernel/py/e2etests/**/*'
+          #   - 'streamlit/**/*'
           mountable:
             - 'packages/mountable/**/*'
           sharing-editor:
@@ -107,6 +108,13 @@ jobs:
     - name: Create virtualenv
       run: python -m venv .venv
 
+    - name: Install dependencies
+      shell: bash
+      run: |
+        . .venv/bin/activate
+        cd packages/kernel/py/e2etests
+        poetry install
+
     # Set up the /streamlit submodule
     ## Set up apt packages. Ref: https://github.com/streamlit/streamlit/wiki/Contributing#ubuntu
     - name: Install Streamlit build dependencies
@@ -117,26 +125,26 @@ jobs:
         cd streamlit
         make mini-init
 
-    - name: Install dependencies
-      shell: bash
-      run: poetry install
-      working-directory: packages/kernel/py/e2etests
-
     - name: Run linter and code formatter to the test code module
-      working-directory: packages/kernel/py/e2etests
       run: |
+        . .venv/bin/activate
+        cd packages/kernel/py/e2etests
         poetry run black . --check
         poetry run isort . --check
         poetry run flake8
 
     - name: Run mypy
-      working-directory: packages/kernel/py/e2etests
-      run: poetry run mypy .
+      run: |
+        . .venv/bin/activate
+        cd packages/kernel/py/e2etests
+        poetry run mypy .
 
     - name: Run pytest
       shell: bash
-      run: poetry run python -m pytest -v tests
-      working-directory: packages/kernel/py/e2etests
+      run: |
+        . .venv/bin/activate
+        cd packages/kernel/py/e2etests
+        poetry run python -m pytest -v tests
 
   test-mountable:
     needs: changes

diff --git a/Makefile b/Makefile
@@ -9,7 +9,7 @@ kernel := packages/kernel/dist/*
 pyarrow_wheel := packages/kernel/py/stlite-pyarrow/dist/stlite_pyarrow-0.1.0-py3-none-any.whl
 tornado_wheel := packages/kernel/py/tornado/dist/tornado-6.2-py3-none-any.whl
 streamlit_proto := streamlit/frontend/src/autogen
-streamlit_wheel := packages/kernel/py/streamlit/lib/dist/streamlit-1.17.0-py2.py3-none-any.whl
+streamlit_wheel := packages/kernel/py/streamlit/lib/dist/streamlit-1.18.1-py2.py3-none-any.whl
 
 .PHONY: all
 all: init mountable sharing sharing-editor
@@ -124,4 +124,4 @@ $(streamlit_wheel): $(VENV) $(streamlit_proto) streamlit/lib/streamlit/**/*.py s
 	cd streamlit && \
 	$(MAKE) distribution
 	mkdir -p `dirname $(streamlit_wheel)`
-	cp streamlit/lib/dist/streamlit-1.17.0-py2.py3-none-any.whl $(streamlit_wheel)
+	cp streamlit/lib/dist/streamlit-1.18.1-py2.py3-none-any.whl $(streamlit_wheel)
diff --git a/packages/common-react/package.json b/packages/common-react/package.json
@@ -14,17 +14,17 @@
     "check:prettier": "prettier --check ."
   },
   "dependencies": {
-    "streamlit-browser": "^1.12.0"
+    "streamlit-browser": "^1.18.1"
   },
   "devDependencies": {
-    "@typescript-eslint/eslint-plugin": "^5.46.1",
-    "@typescript-eslint/parser": "^5.46.1",
-    "eslint": "^8.21.0",
-    "eslint-plugin-react": "^7.31.11",
+    "@typescript-eslint/eslint-plugin": "^5.49.0",
+    "@typescript-eslint/parser": "^5.49.0",
+    "eslint": "^8.33.0",
+    "eslint-plugin-react": "^7.32.1",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-toastify": "^9.1.1",
-    "typescript": "^4.6.4"
+    "typescript": "^4.9.4"
   },
   "peerDependencies": {
     "react": "^17.0.2",

diff --git a/packages/common-react/src/toastify-components/theme.ts b/packages/common-react/src/toastify-components/theme.ts
@@ -1,16 +1,28 @@
-import { LocalStore } from "streamlit-browser/src/lib/storageUtils";
+import {
+  LocalStore,
+  localStorageAvailable,
+} from "streamlit-browser/src/lib/storageUtils";
 import { darkTheme } from "streamlit-browser/src/theme/themeConfigs";
 
-export function isDarkTheme() {
+function isSystemDarkTheme(): boolean {
+  // Detect the system color mode. Ref: https://stackoverflow.com/a/57795495/13103190
+  return (
+    window.matchMedia &&
+    window.matchMedia("(prefers-color-scheme: dark)").matches
+  );
+}
+
+export function isDarkTheme(): boolean {
+  if (!localStorageAvailable()) {
+    return isSystemDarkTheme();
+  }
+
   // Ref: https://github.com/streamlit/streamlit/blob/1.12.0/frontend/src/theme/utils.ts#L544
   const cachedThemeStr = window.localStorage.getItem(LocalStore.ACTIVE_THEME);
   if (!cachedThemeStr) {
-    // Detect the system color mode Ref: https://stackoverflow.com/a/57795495/13103190
-    return (
-      window.matchMedia &&
-      window.matchMedia("(prefers-color-scheme: dark)").matches
-    );
+    return isSystemDarkTheme();
   }
+
   const { name } = JSON.parse(cachedThemeStr);
 
   return name === darkTheme.name;

diff --git a/packages/common/package.json b/packages/common/package.json
@@ -19,10 +19,10 @@
     "check:prettier": "prettier --check ."
   },
   "devDependencies": {
-    "@typescript-eslint/eslint-plugin": "^5.33.0",
-    "@typescript-eslint/parser": "^5.33.0",
-    "eslint": "^8.21.0",
-    "typescript": "^4.6.4",
+    "@typescript-eslint/eslint-plugin": "^5.49.0",
+    "@typescript-eslint/parser": "^5.49.0",
+    "eslint": "^8.33.0",
+    "typescript": "^4.9.4",
     "vitest": "^0.21.1"
   }
 }
diff --git a/packages/desktop/bin/dump_artifacts.ts b/packages/desktop/bin/dump_artifacts.ts
@@ -100,7 +100,7 @@ async function createSitePackagesSnapshot(
       pyodide,
       path.join(
         stliteKernelPyDir,
-        "streamlit/lib/dist/streamlit-1.17.0-py2.py3-none-any.whl"
+        "streamlit/lib/dist/streamlit-1.18.1-py2.py3-none-any.whl"
       )
     );
   } else {

diff --git a/packages/desktop/craco.config.js b/packages/desktop/craco.config.js
@@ -3,17 +3,7 @@ const HtmlWebpackPlugin = require("html-webpack-plugin");
 
 module.exports = {
   babel: {
-    plugins: [
-      "@emotion",
-      // Stlite: This specific syntax plugin is needed since the syntax was started being used in the upstream codebase (https://github.com/streamlit/streamlit/pull/5913/files#diff-845917f3a07167e741db444532fae1e083d5b9f84ac8e8e38d3a34818a311ad8R242).
-      // With the browserslist setting in the upstream project, this plugin is automatically chosen by @babel/preset-env and the syntax is transpiled nicely,
-      // however, this `desktop` package has a different browserslist optimized for the Electron runtime
-      // and it leads to an error maybe because of CRA's bug.
-      // So we had to specify this single plugin here explicitly.
-      // See https://github.com/whitphx/stlite/issues/471 for the details.
-      // TODO: When CRA is updated to v5, the bug should be gone away so this config can be removed.
-      "@babel/plugin-proposal-logical-assignment-operators",
-    ],
+    plugins: ["@emotion"],
     loaderOptions: {
       cacheDirectory: true,
     },
@@ -37,8 +27,9 @@ module.exports = {
         "script-src 'wasm-unsafe-eval' 'unsafe-eval'",
         // style-src is necessary because of emotion. In dev, style-loader with injectType=styleTag is also the reason.
         "style-src 'self' 'unsafe-inline'",
-        // The worker is inlined as blob: https://github.com/whitphx/stlite/blob/v0.7.1/packages/stlite-kernel/src/kernel.ts#L16
-        "worker-src blob:",
+        // - 'self': The stlite kernel worker is bundled as a separate file via Webpack 5's worker feature.
+        // - blob: : Some third party libraries such as Mapbox used in st.map() create workers from blob.
+        "worker-src 'self' blob:",
         // For <script /> tag permissions.
         // - 'self': The main scripts
         // - 'unsafe-inline': Allow the inline scripts from custom components
@@ -73,9 +64,11 @@ module.exports = {
       ]
         .filter(Boolean)
         .join("; ");
-      htmlWebpackPlugin.options.meta["Content-Security-Policy"] = {
-        "http-equiv": "Content-Security-Policy",
-        content: csp,
+      htmlWebpackPlugin.userOptions.meta = {
+        "Content-Security-Policy": {
+          "http-equiv": "Content-Security-Policy",
+          content: csp,
+        },
       };
 
       // Let Babel compile outside of src/.
@@ -94,6 +87,12 @@ module.exports = {
       };
 
       /* To build Streamlit. These configs are copied from streamlit/frontend/craco.config.js */
+      webpackConfig.resolve.mainFields = ["module", "main"];
+      // Webpack 5 requires polyfills. We don't need them, so resolve to an empty module
+      webpackConfig.resolve.fallback ||= {};
+      webpackConfig.resolve.fallback.tty = false;
+      webpackConfig.resolve.fallback.os = false;
+
       // Apache Arrow uses .mjs
       webpackConfig.module.rules.push({
         include: /node_modules/,
@@ -109,16 +108,15 @@ module.exports = {
       // then we don't obtain the expected result.
       // So we turn off Asset Modules here by setting `type: 'javascript/auto'`.
       // See https://webpack.js.org/guides/asset-modules/
-      // TODO: Enable when using Webpack 5.
-      // webpackConfig.module.rules.push({
-      //   test: /\.whl$/i,
-      //   use: [
-      //     {
-      //       loader: 'file-loader',
-      //     }
-      //   ],
-      //   type: 'javascript/auto'
-      // })
+      webpackConfig.module.rules.push({
+        test: /\.whl$/i,
+        use: [
+          {
+            loader: "file-loader",
+          },
+        ],
+        type: "javascript/auto",
+      });
 
       return webpackConfig;
     },

diff --git a/packages/desktop/package.json b/packages/desktop/package.json
@@ -56,14 +56,14 @@
     "yargs": "^17.5.1"
   },
   "devDependencies": {
-    "@craco/craco": "^6.1.2",
+    "@craco/craco": "^7.0.0",
     "@stlite/common": "^0.25.0",
     "@stlite/common-react": "^0.25.0",
     "@stlite/kernel": "^0.25.0",
-    "@testing-library/react": "^11.2.7",
-    "@testing-library/user-event": "^13.1.9",
-    "@types/jest": "^26.0.19",
-    "@types/node": "^14.14.16",
+    "@testing-library/react": "^12.1.4",
+    "@testing-library/user-event": "^14.0.0",
+    "@types/jest": "^27.4.3",
+    "@types/node": "^16.18.12",
     "@types/react": "^17.0.7",
     "@types/react-dom": "^17.0.5",
     "@types/yargs": "^17.0.13",
@@ -74,10 +74,11 @@
     "raw-loader": "^4.0.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
-    "react-scripts": "4.0.3",
+    "react-scripts": "5.0.1",
     "react-toastify": "^9.1.1",
+    "streamlit-browser": "^1.18.1",
     "ts-node": "^10.9.1",
-    "typescript": "^4.6.3"
+    "typescript": "^4.9.4"
   },
   "///": "build.productName is necessary because electron-builder uses the package name for its purpose but the scoped name including '@' makes a problem: https://github.com/electron-userland/electron-builder/issues/3230",
   "build": {

diff --git a/packages/kernel/package.json b/packages/kernel/package.json
@@ -20,23 +20,22 @@
   },
   "devDependencies": {
     "@testing-library/react": "^12",
-    "@typescript-eslint/eslint-plugin": "^5.33.0",
-    "@typescript-eslint/parser": "^5.33.0",
+    "@types/path-browserify": "^1.0.0",
+    "@typescript-eslint/eslint-plugin": "^5.49.0",
+    "@typescript-eslint/parser": "^5.49.0",
     "blob-polyfill": "^7.0.20220408",
-    "eslint": "^8.21.0",
+    "eslint": "^8.33.0",
     "eslint-plugin-react": "^7.30.1",
     "eslint-plugin-react-hooks": "^4.6.0",
     "pyodide": "0.22.1",
-    "typescript": "^4.6.4",
+    "typescript": "^4.9.4",
     "vitest": "^0.21.1"
   },
   "dependencies": {
     "@jupyterlab/coreutils": "^5.4.2",
     "@lumino/coreutils": "^1.12.0",
     "form-data-encoder": "^2.0.1",
-    "streamlit-browser": "^1.12.0"
-  },
-  "peerDependencies": {
-    "worker-loader": "^3.0.8"
+    "path-browserify": "^1.0.1",
+    "streamlit-browser": "^1.18.1"
   }
 }
diff --git a/packages/kernel/py/e2etests/pyproject.toml b/packages/kernel/py/e2etests/pyproject.toml
@@ -11,7 +11,6 @@ python = "^3.10"
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"
 requests = "^2.28.1"
-streamlit = "1.10.0"
 black = "^22.6.0"
 flake8 = "^5.0.4"
 isort = "^5.10.1"
@@ -30,3 +29,8 @@ profile = "black"
 
 [tool.mypy]
 check_untyped_defs = true
+mypy_path = "../tornado"
+
+[[tool.mypy.overrides]]
+module = ["tornado", "tornado.*"]
+ignore_errors = true
diff --git a/packages/kernel/py/e2etests/tests/conftest.py b/packages/kernel/py/e2etests/tests/conftest.py
@@ -13,6 +13,9 @@
 TORNADO_PYTHONPATH = TORNADO_SUBMODULE
 sys.path.insert(0, str(TORNADO_PYTHONPATH))
 
+PYARROW_DIR = HERE / "../../stlite-pyarrow"
+PYARROW_PYTHONPATH = PYARROW_DIR
+sys.path.insert(0, str(PYARROW_PYTHONPATH))
 
 sys.modules["pyodide"] = Mock()
 sys.modules["pyodide.ffi"] = Mock()