From d5d22a09fb567402e8c3609b00cf048bd812938f Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Mon, 8 Jan 2024 15:00:13 +0100 Subject: [PATCH 01/26] Add stlite modifications --- packages/common-react/tsconfig.json | 6 +- packages/desktop/package.json | 4 +- packages/kernel/package.json | 2 +- .../stlite-server/stlite_server/bootstrap.py | 26 +++ .../py/stlite-server/stlite_server/handler.py | 3 + .../py/stlite-server/stlite_server/server.py | 12 +- .../upload_file_request_handler.py | 31 ++- packages/kernel/src/index.ts | 2 - packages/kernel/src/kernel.ts | 6 + .../lib/ConnectionManager.ts | 193 ------------------ .../lib/FileUploadClient.ts | 162 --------------- packages/kernel/src/types.ts | 1 + packages/kernel/src/worker.ts | 53 +++-- packages/mountable/src/options.ts | 2 + requirements.dev.txt | 2 +- 15 files changed, 109 insertions(+), 396 deletions(-) delete mode 100644 packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts delete mode 100644 packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts diff --git a/packages/common-react/tsconfig.json b/packages/common-react/tsconfig.json index 1b5ab51cd..0028dd6bf 100644 --- a/packages/common-react/tsconfig.json +++ b/packages/common-react/tsconfig.json @@ -29,9 +29,9 @@ "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ "paths": { - "src/theme": ["../../streamlit/frontend/src/theme"], - "src/theme/*": ["../../streamlit/frontend/src/theme/*"], - "src/lib/*": ["../../streamlit/frontend/src/lib/*"], + "theme/src": ["../../streamlit/frontend/lib/src/theme"], + "theme/src/*": ["../../streamlit/frontend/lib/src/theme/*"], + "lib/src/*": ["../../streamlit/frontend/lib/src/*"], } /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 92dd13b50..8767ea993 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -19,7 +19,7 @@ "eject": "craco eject", "start:electron": "tsc -p electron -w", "build:electron": "tsc -p electron", - "build:pyodide": "curl -L https://github.com/pyodide/pyodide/releases/download/0.23.3/pyodide-core-0.23.3.tar.bz2 | tar xj -C ./build --files-from=./pyodide-files.txt", + "build:pyodide": "curl -L https://github.com/pyodide/pyodide/releases/download/0.24.1/pyodide-core-0.24.1.tar.bz2 | tar xj -C ./build --files-from=./pyodide-files.txt", "build:bin": "./scripts/build_bin.js && sed -i'' -e '1 s/^#!.*$/#!\\/usr\\/bin\\/env node/' ./bin/*.js", "typecheck": "yarn tsc --noEmit -p electron", "start": "concurrently \"cross-env BROWSER=none yarn start:web\" \"wait-on http://localhost:3000 && yarn start:electron\" \"wait-on http://localhost:3000 && tsc -p electron && cross-env NODE_ENV=\"development\" electron .\"", @@ -53,7 +53,7 @@ "dependencies": { "fs-extra": "^10.1.0", "node-fetch": "2", - "pyodide": "0.23.3", + "pyodide": "0.24.1", "yargs": "^17.5.1" }, "devDependencies": { diff --git a/packages/kernel/package.json b/packages/kernel/package.json index feb825abc..882c9da82 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -26,7 +26,7 @@ "eslint": "^8.33.0", "eslint-plugin-react": "^7.30.1", "eslint-plugin-react-hooks": "^4.6.0", - "pyodide": "0.23.3", + "pyodide": "0.24.1", "typescript": "^4.9.4", "vitest": "^0.21.1" }, diff --git a/packages/kernel/py/stlite-server/stlite_server/bootstrap.py b/packages/kernel/py/stlite-server/stlite_server/bootstrap.py index 6ecccd862..7db70ecb2 100644 --- a/packages/kernel/py/stlite-server/stlite_server/bootstrap.py +++ b/packages/kernel/py/stlite-server/stlite_server/bootstrap.py @@ -123,6 +123,30 @@ def _on_pages_changed(_path: str) -> None: allow_nonexistent=True, ) +def _fix_altair(): + """Fix an issue with Altair and the mocked pyarrow module of stlite.""" + try: + from altair.utils import _importers + + def _pyarrow_available(): + return False + + _importers.pyarrow_available = _pyarrow_available + + def _import_pyarrow_interchange(): + raise ImportError("Pyarrow is not available in stlite.") + + _importers.import_pyarrow_interchange = _import_pyarrow_interchange + except: + pass + +def _fix_requests(): + try: + import pyodide_http + pyodide_http.patch_all() # Patch all libraries + except ImportError: + # pyodide_http is not installed. No need to do anything. + pass def prepare( main_script_path: str, @@ -135,6 +159,8 @@ def prepare( """ _fix_sys_path(main_script_path) _fix_matplotlib_crash() + _fix_altair() + _fix_requests() _fix_sys_argv(main_script_path, args) _fix_pydeck_mapbox_api_warning() _install_pages_watcher(main_script_path) diff --git a/packages/kernel/py/stlite-server/stlite_server/handler.py b/packages/kernel/py/stlite-server/stlite_server/handler.py index 7fb35446a..2b35e2a50 100644 --- a/packages/kernel/py/stlite-server/stlite_server/handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/handler.py @@ -21,3 +21,6 @@ def get(self, request: Request) -> Response | Awaitable[Response]: def post(self, request: Request) -> Response | Awaitable[Response]: return Response(status_code=405, headers={}, body="") + + def delete(self, request: Request) -> Response | Awaitable[Response]: + return Response(status_code=405, headers={}, body="") diff --git a/packages/kernel/py/stlite-server/stlite_server/server.py b/packages/kernel/py/stlite-server/stlite_server/server.py index 473e63ec2..23e07d0c5 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server.py +++ b/packages/kernel/py/stlite-server/stlite_server/server.py @@ -11,6 +11,10 @@ from streamlit.runtime import Runtime, RuntimeConfig, SessionClient from streamlit.runtime.memory_media_file_storage import MemoryMediaFileStorage from streamlit.runtime.runtime_util import serialize_forward_msg +from streamlit.runtime.memory_uploaded_file_manager import MemoryUploadedFileManager +from streamlit.web.cache_storage_manager_config import ( + create_default_cache_storage_manager, +) from .component_request_handler import ComponentRequestHandler from .handler import RequestHandler @@ -22,8 +26,9 @@ LOGGER = logging.getLogger(__name__) # These route definitions are copied from the original impl at https://github.com/streamlit/streamlit/blob/1.18.1/lib/streamlit/web/server/server.py#L81-L89 # noqa: E501 +UPLOAD_FILE_ENDPOINT: Final = "/_stcore/upload_file" MEDIA_ENDPOINT: Final = "/media" -STREAM_ENDPOINT: Final = r"_stcore/stream" +STREAM_ENDPOINT: Final = "_stcore/stream" HEALTH_ENDPOINT: Final = r"(?:healthz|_stcore/health)" @@ -34,12 +39,15 @@ def __init__(self, main_script_path: str, command_line: str | None) -> None: self._main_script_path = main_script_path self._media_file_storage = MemoryMediaFileStorage(MEDIA_ENDPOINT) + self.uploaded_file_mgr = MemoryUploadedFileManager(UPLOAD_FILE_ENDPOINT) self._runtime = Runtime( RuntimeConfig( script_path=main_script_path, command_line=command_line, media_file_storage=self._media_file_storage, + uploaded_file_manager=self.uploaded_file_mgr, + cache_storage_manager=create_default_cache_storage_manager(), ), ) @@ -152,7 +160,7 @@ def receive_http( on_response(404, {}, b"No handler found") return method_name = method.lower() - if method_name not in ("get", "post"): + if method_name not in ("get", "post", "delete"): on_response(405, {}, b"Now allowed") return handler_method = getattr(handler, method_name, None) diff --git a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py index 09c6af9b4..1203d9e3f 100644 --- a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py @@ -1,14 +1,15 @@ import logging from typing import Callable, Dict, List +import re from streamlit.runtime.uploaded_file_manager import UploadedFileManager, UploadedFileRec from .handler import Request, RequestHandler, Response from .httputil import HTTPFile, parse_body_arguments -# /_stcore/upload_file/(optional session id)/(optional widget id) +# /_stcore/upload_file/(optional session id)/(optional file id) UPLOAD_FILE_ROUTE = ( - r"/_stcore/upload_file/?(?P[^/]*)?/?(?P[^/]*)?" + r"/_stcore/upload_file/(?P[^/]+)/(?P[^/]+)" ) LOGGER = logging.getLogger(__name__) @@ -61,8 +62,11 @@ def post(self, request: Request, **kwargs) -> Response: ) try: - session_id = self._require_arg(args, "sessionId") - widget_id = self._require_arg(args, "widgetId") + path_args = re.match(UPLOAD_FILE_ROUTE, request.path) + session_id = path_args.group('session_id') + file_id = path_args.group('file_id') + # session_id = self._require_arg(args, "sessionId") + # file_id = self._require_arg(args, "fileId") if not self._is_active_session(session_id): raise Exception(f"Invalid session_id: '{session_id}'") @@ -78,7 +82,7 @@ def post(self, request: Request, **kwargs) -> Response: for file in flist: uploaded_files.append( UploadedFileRec( - id=0, + file_id=file_id, name=file.filename, type=file.content_type, data=file.body, @@ -92,10 +96,17 @@ def post(self, request: Request, **kwargs) -> Response: body=f"Expected 1 file, but got {len(uploaded_files)}", ) - added_file = self._file_mgr.add_file( - session_id=session_id, widget_id=widget_id, file=uploaded_files[0] + self._file_mgr.add_file( + session_id=session_id, file=uploaded_files[0] ) + return Response(status_code=204, headers={}, body="") - # Return the file_id to the client. (The client will parse - # the string back to an int.) - return Response(status_code=200, headers={}, body=str(added_file.id)) + def delete(self, request: Request, **kwargs): + """Delete file request handler.""" + + path_args = re.match(UPLOAD_FILE_ROUTE, request.path) + session_id = path_args.group('session_id') + file_id = path_args.group('file_id') + + self._file_mgr.remove_file(session_id=session_id, file_id=file_id) + self.set_status(204) diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index a8ec3acf4..028123615 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -1,5 +1,3 @@ export * from "./kernel"; -export * from "./streamlit-replacements/lib/ConnectionManager"; -export * from "./streamlit-replacements/lib/FileUploadClient"; export * from "./react-helpers"; export * from "./types"; diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index 066a2f196..fc94d620f 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -107,6 +107,11 @@ export interface StliteKernelOptions { */ streamlitConfig?: StreamlitConfig; + /** + * If true, the loading toasts are disabled. + */ + disableLoadingToasts?: boolean; + onProgress?: (message: string) => void; onLoad?: () => void; @@ -186,6 +191,7 @@ export class StliteKernel { archives: options.archives, requirements: options.requirements, pyodideUrl: options.pyodideUrl, + disableLoadingToasts: options.disableLoadingToasts, wheels, mountedSitePackagesSnapshotFilePath: options.mountedSitePackagesSnapshotFilePath, diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts deleted file mode 100644 index 523c8c641..000000000 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ /dev/null @@ -1,193 +0,0 @@ -// Mimic https://github.com/streamlit/streamlit/blob/1.9.0/frontend/src/lib/ConnectionManager.ts -// and WebsocketConnection. - -import type { ReactNode } from "react" - -import { BackMsg, ForwardMsg } from "@streamlit/lib/src/proto" -import type { IAllowedMessageOriginsResponse } from "@streamlit/lib/src/hostComm/types" -import type { BaseUriParts } from "@streamlit/lib/src/util/UriUtil" - -import type { StliteKernel } from "../../kernel" -import { ConnectionState } from "@streamlit/app/src/connection/ConnectionState" -import type { SessionInfo } from "@streamlit/lib/src/SessionInfo" -import { ensureError } from "@streamlit/lib/src/util/ErrorHandling" -import { DUMMY_BASE_HOSTNAME, DUMMY_BASE_PORT } from "../../consts" - -interface MessageQueue { - [index: number]: any -} - -interface Props { - /** - * Kernel object to connect to. - */ - kernel: StliteKernel - - /** The app's SessionInfo instance */ - sessionInfo: SessionInfo - - /** - * Function to be called when we receive a message from the server. - */ - onMessage: (message: ForwardMsg) => void - - /** - * Function to be called when the connection errors out. - */ - onConnectionError: (errNode: ReactNode) => void - - /** - * Called when our ConnectionState is changed. - */ - connectionStateChanged: (connectionState: ConnectionState) => void - - /** - * Function to set the list of origins that this app should accept - * cross-origin messages from (if in a relevant deployment scenario). - */ - setAllowedOriginsResp: (resp: IAllowedMessageOriginsResponse) => void -} - -export class ConnectionManager { - private readonly props: Props - - private connectionState: ConnectionState = ConnectionState.INITIAL - - constructor(props: Props) { - this.props = props - - this.props.kernel.onWebSocketMessage((payload) => { - if (typeof payload === "string") { - console.error("Unexpected payload type.") - return - } - this.handleMessage(payload) - }) - - this.props.kernel.loaded.then(() => { - console.log("The kernel has been loaded. Start connecting.") - this.props.setAllowedOriginsResp(this.props.kernel.allowedOriginsResp) - this.connect() - }) - } - - /** - * Indicates whether we're connected to the server. - */ - public isConnected(): boolean { - return this.connectionState === ConnectionState.CONNECTED - } - - /** - * Return the BaseUriParts for the server we're connected to, - * if we are connected to a server. - */ - public getBaseUriParts(): BaseUriParts | undefined { - if (this.connectionState === ConnectionState.CONNECTED) { - // The existence of this property became necessary for multi-page apps: https://github.com/streamlit/streamlit/pull/4698/files#diff-e56cb91573ddb6a97ecd071925fe26504bb5a65f921dc64c63e534162950e1ebR967-R975 - // so here a dummy BaseUriParts object is returned. - // The host and port are set as dummy values that are invalid as a URL - // in order to avoid unexpected accesses to external resources, - // while the basePath is representing the actual info. - return { - host: DUMMY_BASE_HOSTNAME, - port: DUMMY_BASE_PORT, - // When a new session starts, a page name for multi-page apps (a relative path to the app root url) is calculated based on this `basePath` - // then a `rerunScript` BackMsg is sent to the server with `pageName` (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L1064) - // and `window.history.pushState` is called (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L665). - basePath: this.props.kernel.basePath, - } - } - return undefined - } - - public sendMessage(obj: BackMsg): void { - if (this.isConnected()) { - this.props.kernel.sendWebSocketMessage( - BackMsg.encode(BackMsg.create(obj)).finish() - ) - } else { - // Don't need to make a big deal out of this. Just print to console. - console.error(`Cannot send message when server is disconnected: ${obj}`) - } - } - - /** - * To guarantee packet transmission order, this is the index of the last - * dispatched incoming message. - */ - private lastDispatchedMessageIndex = -1 - - /** - * And this is the index of the next message we receive. - */ - private nextMessageIndex = 0 - - /** - * This dictionary stores received messages that we haven't sent out yet - * (because we're still decoding previous messages) - */ - private readonly messageQueue: MessageQueue = {} - - /** - * No-op in stlite. - */ - public incrementMessageCacheRunCount(): void { - // no-op. - // Because caching is disabled in stlite. See https://github.com/whitphx/stlite/issues/495 - } - - private async handleMessage(data: ArrayBuffer): Promise { - // Assign this message an index. - const messageIndex = this.nextMessageIndex - this.nextMessageIndex += 1 - - const encodedMsg = new Uint8Array(data) - const msg = ForwardMsg.decode(encodedMsg) - - // stlite doesn't handle caches. - if (msg.type === "refHash") { - throw new Error(`Unexpected cache reference message.`) - } - - this.messageQueue[messageIndex] = msg - - // Dispatch any pending messages in the queue. This may *not* result - // in our just-decoded message being dispatched: if there are other - // messages that were received earlier than this one but are being - // downloaded, our message won't be sent until they're done. - while (this.lastDispatchedMessageIndex + 1 in this.messageQueue) { - const dispatchMessageIndex = this.lastDispatchedMessageIndex + 1 - this.props.onMessage(this.messageQueue[dispatchMessageIndex]) - delete this.messageQueue[dispatchMessageIndex] - this.lastDispatchedMessageIndex = dispatchMessageIndex - } - } - - private async connect(): Promise { - const WEBSOCKET_STREAM_PATH = "_stcore/stream" // The original is defined in streamlit/frontend/src/lib/WebsocketConnection.tsx - - try { - await this.props.kernel.connectWebSocket("/" + WEBSOCKET_STREAM_PATH) - this.setConnectionState(ConnectionState.CONNECTED) - } catch (e) { - const err = ensureError(e) - console.error(err.message) - this.setConnectionState(ConnectionState.DISCONNECTED_FOREVER, err.message) - } - } - - private setConnectionState = ( - connectionState: ConnectionState, - errMsg?: string - ): void => { - if (this.connectionState !== connectionState) { - this.connectionState = connectionState - this.props.connectionStateChanged(connectionState) - } - - if (errMsg || connectionState === ConnectionState.DISCONNECTED_FOREVER) { - this.props.onConnectionError(errMsg || "unknown") - } - } -} diff --git a/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts b/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts deleted file mode 100644 index 555258d32..000000000 --- a/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * @license - * Copyright 2018-2022 Streamlit Inc. - * Copyright 2022 Yuichiro Tachibana - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { CancelToken } from "axios" -import type { SessionInfo } from "@streamlit/lib/src/SessionInfo" -import _ from "lodash" -import type { BaseUriParts } from "@streamlit/lib/src/util/UriUtil" -import { FormDataEncoder, FormDataLike } from "form-data-encoder" -import { isValidFormId } from "@streamlit/lib/src/util/utils" -import type { StliteKernel } from "../../kernel" - -/** Common widget protobuf fields that are used by the FileUploadClient. */ -interface WidgetInfo { - id: string - formId: string -} - -interface Props { - /** The app's SessionInfo instance. */ - sessionInfo: SessionInfo - getServerUri: () => BaseUriParts | undefined - csrfEnabled: boolean - formsWithPendingRequestsChanged: (formIds: Set) => void -} - -/** - * Handles uploading files to the server. - */ -export class FileUploadClient { - private readonly sessionInfo: SessionInfo - - /** - * Map of . Updated whenever - * a widget in a form creates are completes a request. - */ - private readonly formsWithPendingRequests = new Map() - - /** - * Called when the set of forms that have pending file requests changes. - */ - private readonly pendingFormUploadsChanged: (formIds: Set) => void - - public constructor(props: Props) { - this.pendingFormUploadsChanged = props.formsWithPendingRequestsChanged - this.sessionInfo = props.sessionInfo - } - - private kernel: StliteKernel | undefined - public setKernel(kernel: StliteKernel) { - this.kernel = kernel - } - - /** - * Upload a file to the server. It will be associated with this browser's sessionID. - * - * @param widget: the FileUploader widget that's doing the upload. - * @param file: the files to upload. - * @param onUploadProgress: an optional function that will be called repeatedly with progress events during the upload. - * @param cancelToken: an optional axios CancelToken that can be used to cancel the in-progress upload. - * - * @return a Promise that resolves with the file's unique ID, as assigned by the server. - */ - public async uploadFile( - widget: WidgetInfo, - file: File, - onUploadProgress?: (progressEvent: any) => void, // TODO - cancelToken?: CancelToken // TODO - ): Promise { - const form = new FormData() - form.append("sessionId", this.sessionInfo.current.sessionId) - form.append("widgetId", widget.id) - form.append(file.name, file) - - this.offsetPendingRequestCount(widget.formId, 1) - - const encoder = new FormDataEncoder(form as unknown as FormDataLike) - const bodyBlob = new Blob(encoder as unknown as BufferSource[], { - type: encoder.contentType, - }) - - return bodyBlob.arrayBuffer().then((body) => { - if (this.kernel == null) { - throw new Error("Kernel not ready") - } - - return this.kernel - .sendHttpRequest({ - method: "POST", - path: "/_stcore/upload_file", - body, - headers: { ...encoder.headers }, - }) - .then((response) => { - const text = new TextDecoder().decode(response.body) - if (typeof text === "number") { - return text - } - if (typeof text === "string") { - const parsed = parseInt(text, 10) - if (!Number.isNaN(parsed)) { - return parsed - } - } - - throw new Error( - `Bad uploadFile response: expected a number but got '${text}'` - ) - }) - .finally(() => this.offsetPendingRequestCount(widget.formId, -1)) - }) - } - - private getFormIdSet(): Set { - return new Set(this.formsWithPendingRequests.keys()) - } - - private offsetPendingRequestCount(formId: string, offset: number): void { - if (offset === 0) { - return - } - - if (!isValidFormId(formId)) { - return - } - - const curCount = this.formsWithPendingRequests.get(formId) ?? 0 - const newCount = curCount + offset - if (newCount < 0) { - throw new Error( - `Can't offset pendingRequestCount below 0 (formId=${formId}, curCount=${curCount}, offset=${offset})` - ) - } - - const prevWidgetIds = this.getFormIdSet() - - if (newCount === 0) { - this.formsWithPendingRequests.delete(formId) - } else { - this.formsWithPendingRequests.set(formId, newCount) - } - - const newWidgetIds = this.getFormIdSet() - if (!_.isEqual(newWidgetIds, prevWidgetIds)) { - this.pendingFormUploadsChanged(newWidgetIds) - } - } -} diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index ca88cb69b..f2b83d8b7 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -51,6 +51,7 @@ export interface WorkerInitialData { }; mountedSitePackagesSnapshotFilePath?: string; streamlitConfig?: StreamlitConfig; + disableLoadingToasts?: boolean; } /** diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index fbf39ff05..5153e74ff 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -56,7 +56,7 @@ async function initPyodide( } const DEFAULT_PYODIDE_URL = - "https://cdn.jsdelivr.net/pyodide/v0.23.3/full/pyodide.js"; + "https://cdn.jsdelivr.net/pyodide/v0.24.1/full/pyodide.js"; /** * Load Pyodided and initialize the interpreter. @@ -77,10 +77,12 @@ async function loadPyodideAndPackages() { mountedSitePackagesSnapshotFilePath, pyodideUrl = DEFAULT_PYODIDE_URL, streamlitConfig, + disableLoadingToasts, } = await initDataPromiseDelegate.promise; - postProgressMessage("Loading Pyodide."); - + if (!disableLoadingToasts) { + postProgressMessage("⚙️ Loading Pyodide..."); + } console.debug("Loading Pyodide"); pyodide = await initPyodide(pyodideUrl, { stdout: console.log, @@ -89,7 +91,8 @@ async function loadPyodideAndPackages() { console.debug("Loaded Pyodide"); // Mount files - postProgressMessage("Mounting files."); + // postProgressMessage("Mounting files."); + console.debug("Mounting files."); await Promise.all( Object.keys(files).map(async (path) => { const file = files[path]; @@ -111,7 +114,8 @@ async function loadPyodideAndPackages() { ); // Unpack archives - postProgressMessage("Unpacking archives."); + // postProgressMessage("Unpacking archives."); + console.debug("Unpacking archives."); await Promise.all( archives.map(async (archive) => { let buffer: Parameters[0]; @@ -134,7 +138,10 @@ async function loadPyodideAndPackages() { if (mountedSitePackagesSnapshotFilePath) { // Restore the site-packages director(y|ies) from the mounted snapshot file. - postProgressMessage("Restoring the snapshot."); + if (!disableLoadingToasts) { + postProgressMessage("Restoring the snapshot..."); + } + // console.debug("Restoring the snapshot."); await pyodide.runPythonAsync(`import tarfile, shutil, site`); @@ -152,18 +159,20 @@ async function loadPyodideAndPackages() { `); console.debug("Restored the snapshot"); - postProgressMessage("Mocking some packages."); + // postProgressMessage("Mocking some packages."); console.debug("Mock pyarrow"); mockPyArrow(pyodide); console.debug("Mocked pyarrow"); } - - // NOTE: It's important to install the requirements before loading the streamlit package + + // NOTE: It's important to install the requirements before loading the streamlit package // because it allows users to specify the versions of Streamlit's dependencies via requirements.txt // before these versions are automatically resolved by micropip when installing Streamlit. // Also, this must be after restoring the snapshot because the snapshot may contain the site-packages. if (requirements.length > 0) { - postProgressMessage("Installing the requirements."); + if (!disableLoadingToasts) { + postProgressMessage("📦 Processing dependencies..."); + } console.debug("Installing the requirements:", requirements); verifyRequirements(requirements); // Blocks the not allowed wheel URL schemes. await pyodide.loadPackage("micropip"); @@ -173,18 +182,19 @@ async function loadPyodideAndPackages() { } if (wheels) { - postProgressMessage("Installing streamlit and its dependencies."); + if (!disableLoadingToasts) { + postProgressMessage("🎛 Installing Streamlit..."); + } console.debug("Loading stlite-server, and streamlit"); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); - await micropip.install(["altair<5.2.0"]); // Altair>=5.2.0 checks PyArrow version and emits an error (https://github.com/altair-viz/altair/pull/3160) await micropip.install.callKwargs([wheels.stliteServer], { keep_going: true, }); await micropip.install.callKwargs([wheels.streamlit], { keep_going: true }); console.debug("Loaded stlite-server, and streamlit"); - postProgressMessage("Mocking some packages."); + // postProgressMessage("Mocking some packages."); console.debug("Mock pyarrow"); mockPyArrow(pyodide); console.debug("Mocked pyarrow"); @@ -197,7 +207,7 @@ async function loadPyodideAndPackages() { importlib.invalidate_caches() `); - postProgressMessage("Loading streamlit package."); + // postProgressMessage("Loading streamlit package."); console.debug("Loading the Streamlit package"); // Importing the `streamlit` module takes most of the time, // so we first run this step independently for clearer logs and easy exec-time profiling. @@ -208,7 +218,7 @@ async function loadPyodideAndPackages() { `); console.debug("Loaded the Streamlit package"); - postProgressMessage("Setting up the loggers."); + // postProgressMessage("Setting up the loggers."); console.debug("Setting the loggers"); // Fix the Streamlit's logger instantiating strategy, which violates the standard and is problematic for us. // See https://github.com/streamlit/streamlit/issues/4742 @@ -261,9 +271,9 @@ async function loadPyodideAndPackages() { `); console.debug("Set the loggers"); - postProgressMessage( - "Mocking some Streamlit functions for the browser environment." - ); + // postProgressMessage( + // "Mocking some Streamlit functions for the browser environment." + // ); console.debug("Mocking some Streamlit functions"); // Disable caching. See https://github.com/whitphx/stlite/issues/495 await pyodide.runPythonAsync(` @@ -276,12 +286,15 @@ async function loadPyodideAndPackages() { `); console.debug("Mocked some Streamlit functions"); - postProgressMessage("Booting up the Streamlit server."); + if (!disableLoadingToasts) { + postProgressMessage("🎈 Inflating balloons..."); + } + console.debug("Booting up the Streamlit server"); // The following Python code is based on streamlit.web.cli.main_run(). self.__streamlitFlagOptions__ = { ...streamlitConfig, - "browser.gatherUsageStats": false, + // "browser.gatherUsageStats": true, "runner.fastReruns": false, // Fast reruns do not work well with the async script runner of stlite. See https://github.com/whitphx/stlite/pull/550#issuecomment-1505485865. }; await pyodide.runPythonAsync(` diff --git a/packages/mountable/src/options.ts b/packages/mountable/src/options.ts index 16c337ac0..c4670aa6a 100644 --- a/packages/mountable/src/options.ts +++ b/packages/mountable/src/options.ts @@ -15,6 +15,7 @@ export interface SimplifiedStliteKernelOptions { allowedOriginsResp?: StliteKernelOptions["allowedOriginsResp"]; pyodideUrl?: StliteKernelOptions["pyodideUrl"]; streamlitConfig?: StliteKernelOptions["streamlitConfig"]; + disableLoadingToasts?: StliteKernelOptions["disableLoadingToasts"]; } function canonicalizeFiles( @@ -106,5 +107,6 @@ export function canonicalizeMountOptions( allowedOriginsResp: options.allowedOriginsResp, pyodideUrl: options.pyodideUrl, streamlitConfig: options.streamlitConfig, + disableLoadingToasts: options.disableLoadingToasts, }; } diff --git a/requirements.dev.txt b/requirements.dev.txt index 825238ff2..f28723227 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,3 @@ build poetry -pyodide-build==0.23.4 +pyodide-build==0.24.1 From 8e568aa62821f930148475668eabb35bd1f6f140 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:33:31 +0100 Subject: [PATCH 02/26] Add back connection manager --- packages/kernel/src/index.ts | 1 + .../lib/ConnectionManager.ts | 293 ++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index 028123615..6472649aa 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -1,3 +1,4 @@ export * from "./kernel"; +export * from "./streamlit-replacements/lib/ConnectionManager"; export * from "./react-helpers"; export * from "./types"; diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts new file mode 100644 index 000000000..35d092874 --- /dev/null +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -0,0 +1,293 @@ +/** + * Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReactNode } from "react" + +import { + IHostConfigResponse, + BaseUriParts, + getPossibleBaseUris, + logError, + SessionInfo, + StreamlitEndpoints, + ensureError, + BackMsg, + ForwardMsg, +} from "@streamlit/lib" + +import { ConnectionState } from "./ConnectionState" +import { WebsocketConnection } from "./WebsocketConnection" + +import type { StliteKernel } from "@stlite/kernel" + +interface MessageQueue { + [index: number]: any +} + +/** + * When the websocket connection retries this many times, we show a dialog + * letting the user know we're having problems connecting. This happens + * after about 15 seconds as, before the 6th retry, we've set timeouts for + * a total of approximately 0.5 + 1 + 2 + 4 + 8 = 15.5 seconds (+/- some + * due to jitter). + */ +const RETRY_COUNT_FOR_WARNING = 6 + +interface Props { + /** + * Kernel object to connect to. + */ + kernel: StliteKernel + + /** The app's SessionInfo instance */ + sessionInfo: SessionInfo + + /** The app's StreamlitEndpoints instance */ + endpoints: StreamlitEndpoints + + /** + * Function to be called when we receive a message from the server. + */ + onMessage: (message: ForwardMsg) => void + + /** + * Function to be called when the connection errors out. + */ + onConnectionError: (errNode: ReactNode) => void + + /** + * Called when our ConnectionState is changed. + */ + connectionStateChanged: (connectionState: ConnectionState) => void + + /** + * Function to get the auth token set by the host of this app (if in a + * relevant deployment scenario). + */ + claimHostAuthToken: () => Promise + + /** + * Function to clear the withHostCommunication hoc's auth token. This should + * be called after the promise returned by claimHostAuthToken successfully + * resolves. + */ + resetHostAuthToken: () => void + + /** + * Function to set the host config for this app (if in a relevant deployment + * scenario). + */ + onHostConfigResp: (resp: IHostConfigResponse) => void +} + +/** + * Manages our connection to the Server. + */ +export class ConnectionManager { + private readonly props: Props + + private connection?: WebsocketConnection + + private connectionState: ConnectionState = ConnectionState.INITIAL + + constructor(props: Props) { + this.props = props + + this.props.kernel.onWebSocketMessage(payload => { + if (typeof payload === "string") { + console.error("Unexpected payload type.") + return + } + this.handleMessage(payload) + }) + + this.props.kernel.loaded.then(() => { + console.log("The kernel has been loaded. Start connecting.") + this.props.onHostConfigResp(this.props.kernel.hostConfig) + this.connect() + }) + } + + /** + * Indicates whether we're connected to the server. + */ + public isConnected(): boolean { + return this.connectionState === ConnectionState.CONNECTED + } + + /** + * Return the BaseUriParts for the server we're connected to, + * if we are connected to a server. + */ + public getBaseUriParts(): BaseUriParts | undefined { + // Stlite Modification: + // if (this.connection instanceof WebsocketConnection) { + // return this.connection.getBaseUriParts() + // } + if (this.connectionState === ConnectionState.CONNECTED) { + // The existence of this property became necessary for multi-page apps: https://github.com/streamlit/streamlit/pull/4698/files#diff-e56cb91573ddb6a97ecd071925fe26504bb5a65f921dc64c63e534162950e1ebR967-R975 + // so here a dummy BaseUriParts object is returned. + // The host and port are set as dummy values that are invalid as a URL + // in order to avoid unexpected accesses to external resources, + // while the basePath is representing the actual info. + return { + host: "xxx", + port: 99999, + // When a new session starts, a page name for multi-page apps (a relative path to the app root url) is calculated based on this `basePath` + // then a `rerunScript` BackMsg is sent to the server with `pageName` (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L1064) + // and `window.history.pushState` is called (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L665). + basePath: this.props.kernel.basePath, + } + } + return undefined + } + + public sendMessage(obj: BackMsg): void { + // Stlite Modification: + // if (this.connection instanceof WebsocketConnection && this.isConnected()) { + // this.connection.sendMessage(obj) + // } else { + // // Don't need to make a big deal out of this. Just print to console. + // logError(`Cannot send message when server is disconnected: ${obj}`) + // } + if (this.isConnected()) { + this.props.kernel.sendWebSocketMessage( + BackMsg.encode(BackMsg.create(obj)).finish() + ) + } else { + // Don't need to make a big deal out of this. Just print to console. + console.error(`Cannot send message when server is disconnected: ${obj}`) + } + } + + /** + * Increment the runCount on our message cache, and clear entries + * whose age is greater than the max. + */ + public incrementMessageCacheRunCount(maxMessageAge: number): void { + // StaticConnection does not use a MessageCache. + if (this.connection instanceof WebsocketConnection) { + this.connection.incrementMessageCacheRunCount(maxMessageAge) + } + } + + private async connect(): Promise { + try { + // Stlite Modifications: + // this.connection = await this.connectToRunningServer() + await this.props.kernel.connectWebSocket("/_stcore/stream") + this.setConnectionState(ConnectionState.CONNECTED) + } catch (e) { + const err = ensureError(e) + logError(err.message) + this.setConnectionState( + ConnectionState.DISCONNECTED_FOREVER, + err.message + ) + } + } + + disconnect(): void { + this.connection?.disconnect() + } + + private setConnectionState = ( + connectionState: ConnectionState, + errMsg?: string + ): void => { + if (this.connectionState !== connectionState) { + this.connectionState = connectionState + this.props.connectionStateChanged(connectionState) + } + + if (errMsg || connectionState === ConnectionState.DISCONNECTED_FOREVER) { + this.props.onConnectionError(errMsg || "unknown") + } + } + + private showRetryError = ( + totalRetries: number, + latestError: ReactNode, + // The last argument of this function is unused and exists because the + // WebsocketConnection.OnRetry type allows a third argument to be set to be + // used in tests. + _retryTimeout: number + ): void => { + if (totalRetries === RETRY_COUNT_FOR_WARNING) { + this.props.onConnectionError(latestError) + } + } + + private connectToRunningServer(): WebsocketConnection { + const baseUriPartsList = getPossibleBaseUris() + + return new WebsocketConnection({ + sessionInfo: this.props.sessionInfo, + endpoints: this.props.endpoints, + baseUriPartsList, + onMessage: this.props.onMessage, + onConnectionStateChange: this.setConnectionState, + onRetry: this.showRetryError, + claimHostAuthToken: this.props.claimHostAuthToken, + resetHostAuthToken: this.props.resetHostAuthToken, + onHostConfigResp: this.props.onHostConfigResp, + }) + } + + // Stlite Modifications: + /** + * To guarantee packet transmission order, this is the index of the last + * dispatched incoming message. + */ + private lastDispatchedMessageIndex = -1 + + /** + * And this is the index of the next message we receive. + */ + private nextMessageIndex = 0 + + /** + * This dictionary stores received messages that we haven't sent out yet + * (because we're still decoding previous messages) + */ + private readonly messageQueue: MessageQueue = {} + + private async handleMessage(data: ArrayBuffer): Promise { + // Assign this message an index. + const messageIndex = this.nextMessageIndex + this.nextMessageIndex += 1 + + const encodedMsg = new Uint8Array(data) + const msg = ForwardMsg.decode(encodedMsg) + + // stlite doesn't handle caches. + if (msg.type === "refHash") { + throw new Error(`Unexpected cache reference message.`) + } + + this.messageQueue[messageIndex] = msg + + // Dispatch any pending messages in the queue. This may *not* result + // in our just-decoded message being dispatched: if there are other + // messages that were received earlier than this one but are being + // downloaded, our message won't be sent until they're done. + while (this.lastDispatchedMessageIndex + 1 in this.messageQueue) { + const dispatchMessageIndex = this.lastDispatchedMessageIndex + 1 + this.props.onMessage(this.messageQueue[dispatchMessageIndex]) + delete this.messageQueue[dispatchMessageIndex] + this.lastDispatchedMessageIndex = dispatchMessageIndex + } + } +} From cd2d3e753ed3a4085d4a72582cba4628b13473bf Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:38:19 +0100 Subject: [PATCH 03/26] Move methods --- .../lib/ConnectionManager.ts | 92 ++++++++++--------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 35d092874..b4ac9cf39 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -1,3 +1,6 @@ +// Mimic https://github.com/streamlit/streamlit/blob/1.27.0/frontend/app/src/connection/ConnectionManager.ts +// and WebsocketConnection. + /** * Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022) * @@ -172,51 +175,6 @@ export class ConnectionManager { } } - /** - * Increment the runCount on our message cache, and clear entries - * whose age is greater than the max. - */ - public incrementMessageCacheRunCount(maxMessageAge: number): void { - // StaticConnection does not use a MessageCache. - if (this.connection instanceof WebsocketConnection) { - this.connection.incrementMessageCacheRunCount(maxMessageAge) - } - } - - private async connect(): Promise { - try { - // Stlite Modifications: - // this.connection = await this.connectToRunningServer() - await this.props.kernel.connectWebSocket("/_stcore/stream") - this.setConnectionState(ConnectionState.CONNECTED) - } catch (e) { - const err = ensureError(e) - logError(err.message) - this.setConnectionState( - ConnectionState.DISCONNECTED_FOREVER, - err.message - ) - } - } - - disconnect(): void { - this.connection?.disconnect() - } - - private setConnectionState = ( - connectionState: ConnectionState, - errMsg?: string - ): void => { - if (this.connectionState !== connectionState) { - this.connectionState = connectionState - this.props.connectionStateChanged(connectionState) - } - - if (errMsg || connectionState === ConnectionState.DISCONNECTED_FOREVER) { - this.props.onConnectionError(errMsg || "unknown") - } - } - private showRetryError = ( totalRetries: number, latestError: ReactNode, @@ -264,6 +222,15 @@ export class ConnectionManager { */ private readonly messageQueue: MessageQueue = {} + + /** + * No-op in stlite. + */ + public incrementMessageCacheRunCount(): void { + // no-op. + // Because caching is disabled in stlite. See https://github.com/whitphx/stlite/issues/495 + } + private async handleMessage(data: ArrayBuffer): Promise { // Assign this message an index. const messageIndex = this.nextMessageIndex @@ -290,4 +257,39 @@ export class ConnectionManager { this.lastDispatchedMessageIndex = dispatchMessageIndex } } + + private async connect(): Promise { + try { + // Stlite Modifications: + // this.connection = await this.connectToRunningServer() + await this.props.kernel.connectWebSocket("/_stcore/stream") + this.setConnectionState(ConnectionState.CONNECTED) + } catch (e) { + const err = ensureError(e) + logError(err.message) + this.setConnectionState( + ConnectionState.DISCONNECTED_FOREVER, + err.message + ) + } + } + + disconnect(): void { + this.connection?.disconnect() + } + + private setConnectionState = ( + connectionState: ConnectionState, + errMsg?: string + ): void => { + if (this.connectionState !== connectionState) { + this.connectionState = connectionState + this.props.connectionStateChanged(connectionState) + } + + if (errMsg || connectionState === ConnectionState.DISCONNECTED_FOREVER) { + this.props.onConnectionError(errMsg || "unknown") + } + } + } From a8c2cb6bc1ce36a35e305acdfb0964ea9869487a Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:44:19 +0100 Subject: [PATCH 04/26] Some updates --- .../lib/ConnectionManager.ts | 51 +++++-------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index b4ac9cf39..4b1a8350c 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -19,7 +19,7 @@ import { ReactNode } from "react" import { - IHostConfigResponse, + IAllowedMessageOriginsResponse, BaseUriParts, getPossibleBaseUris, logError, @@ -30,6 +30,7 @@ import { ForwardMsg, } from "@streamlit/lib" +import { DUMMY_BASE_HOSTNAME, DUMMY_BASE_PORT } from "../../consts" import { ConnectionState } from "./ConnectionState" import { WebsocketConnection } from "./WebsocketConnection" @@ -89,10 +90,10 @@ interface Props { resetHostAuthToken: () => void /** - * Function to set the host config for this app (if in a relevant deployment - * scenario). + * Function to set the list of origins that this app should accept + * cross-origin messages from (if in a relevant deployment scenario). */ - onHostConfigResp: (resp: IHostConfigResponse) => void + setAllowedOriginsResp: (resp: IAllowedMessageOriginsResponse) => void } /** @@ -118,7 +119,7 @@ export class ConnectionManager { this.props.kernel.loaded.then(() => { console.log("The kernel has been loaded. Start connecting.") - this.props.onHostConfigResp(this.props.kernel.hostConfig) + this.props.setAllowedOriginsResp(this.props.kernel.allowedOriginsResp) this.connect() }) } @@ -146,8 +147,8 @@ export class ConnectionManager { // in order to avoid unexpected accesses to external resources, // while the basePath is representing the actual info. return { - host: "xxx", - port: 99999, + host: DUMMY_BASE_HOSTNAME, + port: DUMMY_BASE_PORT, // When a new session starts, a page name for multi-page apps (a relative path to the app root url) is calculated based on this `basePath` // then a `rerunScript` BackMsg is sent to the server with `pageName` (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L1064) // and `window.history.pushState` is called (https://github.com/streamlit/streamlit/blob/ace58bfa3582d4f8e7f281b4dbd266ddd8a32b54/frontend/src/App.tsx#L665). @@ -175,34 +176,6 @@ export class ConnectionManager { } } - private showRetryError = ( - totalRetries: number, - latestError: ReactNode, - // The last argument of this function is unused and exists because the - // WebsocketConnection.OnRetry type allows a third argument to be set to be - // used in tests. - _retryTimeout: number - ): void => { - if (totalRetries === RETRY_COUNT_FOR_WARNING) { - this.props.onConnectionError(latestError) - } - } - - private connectToRunningServer(): WebsocketConnection { - const baseUriPartsList = getPossibleBaseUris() - - return new WebsocketConnection({ - sessionInfo: this.props.sessionInfo, - endpoints: this.props.endpoints, - baseUriPartsList, - onMessage: this.props.onMessage, - onConnectionStateChange: this.setConnectionState, - onRetry: this.showRetryError, - claimHostAuthToken: this.props.claimHostAuthToken, - resetHostAuthToken: this.props.resetHostAuthToken, - onHostConfigResp: this.props.onHostConfigResp, - }) - } // Stlite Modifications: /** @@ -222,7 +195,6 @@ export class ConnectionManager { */ private readonly messageQueue: MessageQueue = {} - /** * No-op in stlite. */ @@ -257,12 +229,14 @@ export class ConnectionManager { this.lastDispatchedMessageIndex = dispatchMessageIndex } } - + private async connect(): Promise { + const WEBSOCKET_STREAM_PATH = "_stcore/stream" // The original is defined in streamlit/frontend/src/lib/WebsocketConnection.tsx + try { // Stlite Modifications: // this.connection = await this.connectToRunningServer() - await this.props.kernel.connectWebSocket("/_stcore/stream") + await this.props.kernel.connectWebSocket("/" + WEBSOCKET_STREAM_PATH) this.setConnectionState(ConnectionState.CONNECTED) } catch (e) { const err = ensureError(e) @@ -291,5 +265,4 @@ export class ConnectionManager { this.props.onConnectionError(errMsg || "unknown") } } - } From c4109e792efde700ff5bededf75c7c95070ada2b Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:45:00 +0100 Subject: [PATCH 05/26] Remove header --- .../lib/ConnectionManager.ts | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 4b1a8350c..3e00f2cdc 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -1,21 +1,6 @@ // Mimic https://github.com/streamlit/streamlit/blob/1.27.0/frontend/app/src/connection/ConnectionManager.ts // and WebsocketConnection. -/** - * Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ import { ReactNode } from "react" import { @@ -232,7 +217,7 @@ export class ConnectionManager { private async connect(): Promise { const WEBSOCKET_STREAM_PATH = "_stcore/stream" // The original is defined in streamlit/frontend/src/lib/WebsocketConnection.tsx - + try { // Stlite Modifications: // this.connection = await this.connectToRunningServer() @@ -240,11 +225,8 @@ export class ConnectionManager { this.setConnectionState(ConnectionState.CONNECTED) } catch (e) { const err = ensureError(e) - logError(err.message) - this.setConnectionState( - ConnectionState.DISCONNECTED_FOREVER, - err.message - ) + console.error(err.message) + this.setConnectionState(ConnectionState.DISCONNECTED_FOREVER, err.message) } } From 9788a64355395bb8e229125ab24ded0fc3f14ad1 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:45:39 +0100 Subject: [PATCH 06/26] More cleanup --- .../streamlit-replacements/lib/ConnectionManager.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 3e00f2cdc..34aa9b7c5 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -6,7 +6,6 @@ import { ReactNode } from "react" import { IAllowedMessageOriginsResponse, BaseUriParts, - getPossibleBaseUris, logError, SessionInfo, StreamlitEndpoints, @@ -25,15 +24,6 @@ interface MessageQueue { [index: number]: any } -/** - * When the websocket connection retries this many times, we show a dialog - * letting the user know we're having problems connecting. This happens - * after about 15 seconds as, before the 6th retry, we've set timeouts for - * a total of approximately 0.5 + 1 + 2 + 4 + 8 = 15.5 seconds (+/- some - * due to jitter). - */ -const RETRY_COUNT_FOR_WARNING = 6 - interface Props { /** * Kernel object to connect to. From b1874d01882a9f57ccd2558de10cb9aa5729c048 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:48:24 +0100 Subject: [PATCH 07/26] More changes --- .../streamlit-replacements/lib/ConnectionManager.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 34aa9b7c5..7e94c716a 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -77,14 +77,12 @@ interface Props { export class ConnectionManager { private readonly props: Props - private connection?: WebsocketConnection - private connectionState: ConnectionState = ConnectionState.INITIAL constructor(props: Props) { this.props = props - - this.props.kernel.onWebSocketMessage(payload => { + + this.props.kernel.onWebSocketMessage((payload) => { if (typeof payload === "string") { console.error("Unexpected payload type.") return @@ -220,8 +218,11 @@ export class ConnectionManager { } } + /** + * No-op in stlite. + */ disconnect(): void { - this.connection?.disconnect() + // no-op. } private setConnectionState = ( From 7c765880a1e05e280d210bef1c8094989c3e321a Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:49:43 +0100 Subject: [PATCH 08/26] More cleanup --- .../src/streamlit-replacements/lib/ConnectionManager.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 7e94c716a..bd19b89ab 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -1,12 +1,11 @@ // Mimic https://github.com/streamlit/streamlit/blob/1.27.0/frontend/app/src/connection/ConnectionManager.ts // and WebsocketConnection. -import { ReactNode } from "react" +import type { ReactNode } from "react" import { IAllowedMessageOriginsResponse, BaseUriParts, - logError, SessionInfo, StreamlitEndpoints, ensureError, @@ -16,7 +15,6 @@ import { import { DUMMY_BASE_HOSTNAME, DUMMY_BASE_PORT } from "../../consts" import { ConnectionState } from "./ConnectionState" -import { WebsocketConnection } from "./WebsocketConnection" import type { StliteKernel } from "@stlite/kernel" @@ -81,7 +79,7 @@ export class ConnectionManager { constructor(props: Props) { this.props = props - + this.props.kernel.onWebSocketMessage((payload) => { if (typeof payload === "string") { console.error("Unexpected payload type.") From 2cf5a3a8c876ed7f763fd006d4aba14b69da9151 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:50:39 +0100 Subject: [PATCH 09/26] More modifications --- .../lib/ConnectionManager.ts | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index bd19b89ab..14c2c5f03 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -130,13 +130,6 @@ export class ConnectionManager { } public sendMessage(obj: BackMsg): void { - // Stlite Modification: - // if (this.connection instanceof WebsocketConnection && this.isConnected()) { - // this.connection.sendMessage(obj) - // } else { - // // Don't need to make a big deal out of this. Just print to console. - // logError(`Cannot send message when server is disconnected: ${obj}`) - // } if (this.isConnected()) { this.props.kernel.sendWebSocketMessage( BackMsg.encode(BackMsg.create(obj)).finish() @@ -174,6 +167,13 @@ export class ConnectionManager { // Because caching is disabled in stlite. See https://github.com/whitphx/stlite/issues/495 } + /** + * No-op in stlite. + */ + disconnect(): void { + // no-op. + } + private async handleMessage(data: ArrayBuffer): Promise { // Assign this message an index. const messageIndex = this.nextMessageIndex @@ -216,13 +216,6 @@ export class ConnectionManager { } } - /** - * No-op in stlite. - */ - disconnect(): void { - // no-op. - } - private setConnectionState = ( connectionState: ConnectionState, errMsg?: string From 034cb1ff65634149a9174bb9f03ba798fbda7892 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:53:39 +0100 Subject: [PATCH 10/26] Don't update pyiodide --- packages/desktop/package.json | 2 +- packages/kernel/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 8767ea993..847e55e11 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -53,7 +53,7 @@ "dependencies": { "fs-extra": "^10.1.0", "node-fetch": "2", - "pyodide": "0.24.1", + "pyodide": "0.23.3", "yargs": "^17.5.1" }, "devDependencies": { diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 882c9da82..feb825abc 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -26,7 +26,7 @@ "eslint": "^8.33.0", "eslint-plugin-react": "^7.30.1", "eslint-plugin-react-hooks": "^4.6.0", - "pyodide": "0.24.1", + "pyodide": "0.23.3", "typescript": "^4.9.4", "vitest": "^0.21.1" }, From 426bf35f030e86dbb1ad8517df5c048fb7bc38c3 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:53:50 +0100 Subject: [PATCH 11/26] More cleanup --- .../src/streamlit-replacements/lib/ConnectionManager.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 14c2c5f03..557986fc0 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -107,10 +107,6 @@ export class ConnectionManager { * if we are connected to a server. */ public getBaseUriParts(): BaseUriParts | undefined { - // Stlite Modification: - // if (this.connection instanceof WebsocketConnection) { - // return this.connection.getBaseUriParts() - // } if (this.connectionState === ConnectionState.CONNECTED) { // The existence of this property became necessary for multi-page apps: https://github.com/streamlit/streamlit/pull/4698/files#diff-e56cb91573ddb6a97ecd071925fe26504bb5a65f921dc64c63e534162950e1ebR967-R975 // so here a dummy BaseUriParts object is returned. @@ -140,8 +136,6 @@ export class ConnectionManager { } } - - // Stlite Modifications: /** * To guarantee packet transmission order, this is the index of the last * dispatched incoming message. @@ -205,8 +199,6 @@ export class ConnectionManager { const WEBSOCKET_STREAM_PATH = "_stcore/stream" // The original is defined in streamlit/frontend/src/lib/WebsocketConnection.tsx try { - // Stlite Modifications: - // this.connection = await this.connectToRunningServer() await this.props.kernel.connectWebSocket("/" + WEBSOCKET_STREAM_PATH) this.setConnectionState(ConnectionState.CONNECTED) } catch (e) { From 26d232220fbdd960bc3d6734838f54d9672cb46b Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:54:40 +0100 Subject: [PATCH 12/26] Don't Update pyodide --- packages/kernel/src/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index 5153e74ff..52a745848 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -56,7 +56,7 @@ async function initPyodide( } const DEFAULT_PYODIDE_URL = - "https://cdn.jsdelivr.net/pyodide/v0.24.1/full/pyodide.js"; + "https://cdn.jsdelivr.net/pyodide/v0.23.3//full/pyodide.js"; /** * Load Pyodided and initialize the interpreter. From 1dc48143f30051798eb7b056e10257ca507792aa Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 13:55:15 +0100 Subject: [PATCH 13/26] Don't update pyiodide --- requirements.dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index f28723227..825238ff2 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,3 @@ build poetry -pyodide-build==0.24.1 +pyodide-build==0.23.4 From 6c749a1b39f17685140d3cccfc0d6cb5e219ace0 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 14:11:59 +0100 Subject: [PATCH 14/26] More cleanup --- .../src/toastify-components/callbacks.tsx | 10 +++- packages/desktop/package.json | 2 +- packages/kernel/src/worker.ts | 50 +++++++------------ packages/mountable/src/index.tsx | 5 +- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/packages/common-react/src/toastify-components/callbacks.tsx b/packages/common-react/src/toastify-components/callbacks.tsx index 34ef3a74e..0a46771eb 100644 --- a/packages/common-react/src/toastify-components/callbacks.tsx +++ b/packages/common-react/src/toastify-components/callbacks.tsx @@ -8,10 +8,14 @@ interface ToastKernelCallbacks { onLoad: NonNullable; onError: NonNullable; } -export function makeToastKernelCallbacks(): ToastKernelCallbacks { +export function makeToastKernelCallbacks(showProgressToasts = true, showErrorToasts = true): ToastKernelCallbacks { let prevToastId: ToastId | null = null; const toastIds: ToastId[] = []; const onProgress: StliteKernelOptions["onProgress"] = (message) => { + if (!showProgressToasts) { + return; + } + const id = toast(message, { position: toast.POSITION.BOTTOM_RIGHT, transition: Slide, @@ -33,6 +37,10 @@ export function makeToastKernelCallbacks(): ToastKernelCallbacks { toastIds.forEach((id) => toast.dismiss(id)); }; const onError: StliteKernelOptions["onError"] = (error) => { + if (!showErrorToasts) { + return; + } + toast( , { diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 847e55e11..92dd13b50 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -19,7 +19,7 @@ "eject": "craco eject", "start:electron": "tsc -p electron -w", "build:electron": "tsc -p electron", - "build:pyodide": "curl -L https://github.com/pyodide/pyodide/releases/download/0.24.1/pyodide-core-0.24.1.tar.bz2 | tar xj -C ./build --files-from=./pyodide-files.txt", + "build:pyodide": "curl -L https://github.com/pyodide/pyodide/releases/download/0.23.3/pyodide-core-0.23.3.tar.bz2 | tar xj -C ./build --files-from=./pyodide-files.txt", "build:bin": "./scripts/build_bin.js && sed -i'' -e '1 s/^#!.*$/#!\\/usr\\/bin\\/env node/' ./bin/*.js", "typecheck": "yarn tsc --noEmit -p electron", "start": "concurrently \"cross-env BROWSER=none yarn start:web\" \"wait-on http://localhost:3000 && yarn start:electron\" \"wait-on http://localhost:3000 && tsc -p electron && cross-env NODE_ENV=\"development\" electron .\"", diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index 52a745848..e34b08086 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -56,7 +56,7 @@ async function initPyodide( } const DEFAULT_PYODIDE_URL = - "https://cdn.jsdelivr.net/pyodide/v0.23.3//full/pyodide.js"; + "https://cdn.jsdelivr.net/pyodide/v0.23.3/full/pyodide.js"; /** * Load Pyodided and initialize the interpreter. @@ -80,9 +80,8 @@ async function loadPyodideAndPackages() { disableLoadingToasts, } = await initDataPromiseDelegate.promise; - if (!disableLoadingToasts) { - postProgressMessage("⚙️ Loading Pyodide..."); - } + postProgressMessage("Loading Pyodide."); + console.debug("Loading Pyodide"); pyodide = await initPyodide(pyodideUrl, { stdout: console.log, @@ -91,8 +90,8 @@ async function loadPyodideAndPackages() { console.debug("Loaded Pyodide"); // Mount files - // postProgressMessage("Mounting files."); - console.debug("Mounting files."); + postProgressMessage("Mounting files."); + await Promise.all( Object.keys(files).map(async (path) => { const file = files[path]; @@ -114,8 +113,7 @@ async function loadPyodideAndPackages() { ); // Unpack archives - // postProgressMessage("Unpacking archives."); - console.debug("Unpacking archives."); + postProgressMessage("Unpacking archives."); await Promise.all( archives.map(async (archive) => { let buffer: Parameters[0]; @@ -138,10 +136,7 @@ async function loadPyodideAndPackages() { if (mountedSitePackagesSnapshotFilePath) { // Restore the site-packages director(y|ies) from the mounted snapshot file. - if (!disableLoadingToasts) { - postProgressMessage("Restoring the snapshot..."); - } - // console.debug("Restoring the snapshot."); + postProgressMessage("Restoring the snapshot..."); await pyodide.runPythonAsync(`import tarfile, shutil, site`); @@ -159,20 +154,18 @@ async function loadPyodideAndPackages() { `); console.debug("Restored the snapshot"); - // postProgressMessage("Mocking some packages."); + postProgressMessage("Mocking some packages."); console.debug("Mock pyarrow"); mockPyArrow(pyodide); console.debug("Mocked pyarrow"); } - // NOTE: It's important to install the requirements before loading the streamlit package + // NOTE: It's important to install the requirements before loading the streamlit package // because it allows users to specify the versions of Streamlit's dependencies via requirements.txt // before these versions are automatically resolved by micropip when installing Streamlit. // Also, this must be after restoring the snapshot because the snapshot may contain the site-packages. if (requirements.length > 0) { - if (!disableLoadingToasts) { - postProgressMessage("📦 Processing dependencies..."); - } + postProgressMessage("Installing the requirements."); console.debug("Installing the requirements:", requirements); verifyRequirements(requirements); // Blocks the not allowed wheel URL schemes. await pyodide.loadPackage("micropip"); @@ -182,9 +175,7 @@ async function loadPyodideAndPackages() { } if (wheels) { - if (!disableLoadingToasts) { - postProgressMessage("🎛 Installing Streamlit..."); - } + postProgressMessage("Installing streamlit and its dependencies."); console.debug("Loading stlite-server, and streamlit"); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); @@ -194,7 +185,7 @@ async function loadPyodideAndPackages() { await micropip.install.callKwargs([wheels.streamlit], { keep_going: true }); console.debug("Loaded stlite-server, and streamlit"); - // postProgressMessage("Mocking some packages."); + postProgressMessage("Mocking some packages."); console.debug("Mock pyarrow"); mockPyArrow(pyodide); console.debug("Mocked pyarrow"); @@ -207,7 +198,7 @@ async function loadPyodideAndPackages() { importlib.invalidate_caches() `); - // postProgressMessage("Loading streamlit package."); + postProgressMessage("Loading streamlit package."); console.debug("Loading the Streamlit package"); // Importing the `streamlit` module takes most of the time, // so we first run this step independently for clearer logs and easy exec-time profiling. @@ -218,7 +209,7 @@ async function loadPyodideAndPackages() { `); console.debug("Loaded the Streamlit package"); - // postProgressMessage("Setting up the loggers."); + postProgressMessage("Setting up the loggers."); console.debug("Setting the loggers"); // Fix the Streamlit's logger instantiating strategy, which violates the standard and is problematic for us. // See https://github.com/streamlit/streamlit/issues/4742 @@ -271,9 +262,9 @@ async function loadPyodideAndPackages() { `); console.debug("Set the loggers"); - // postProgressMessage( - // "Mocking some Streamlit functions for the browser environment." - // ); + postProgressMessage( + "Mocking some Streamlit functions for the browser environment." + ); console.debug("Mocking some Streamlit functions"); // Disable caching. See https://github.com/whitphx/stlite/issues/495 await pyodide.runPythonAsync(` @@ -286,15 +277,10 @@ async function loadPyodideAndPackages() { `); console.debug("Mocked some Streamlit functions"); - if (!disableLoadingToasts) { - postProgressMessage("🎈 Inflating balloons..."); - } - - console.debug("Booting up the Streamlit server"); + postProgressMessage("Booting up the Streamlit server."); // The following Python code is based on streamlit.web.cli.main_run(). self.__streamlitFlagOptions__ = { ...streamlitConfig, - // "browser.gatherUsageStats": true, "runner.fastReruns": false, // Fast reruns do not work well with the async script runner of stlite. See https://github.com/whitphx/stlite/pull/550#issuecomment-1505485865. }; await pyodide.runPythonAsync(` diff --git a/packages/mountable/src/index.tsx b/packages/mountable/src/index.tsx index 501bc57ff..9a0ab651f 100644 --- a/packages/mountable/src/index.tsx +++ b/packages/mountable/src/index.tsx @@ -35,10 +35,11 @@ export function mount( options: MountOptions, container: HTMLElement = document.body ) { + const mountOptions = canonicalizeMountOptions(options); const kernel = new StliteKernel({ - ...canonicalizeMountOptions(options), + ...mountOptions, wheelBaseUrl, - ...makeToastKernelCallbacks(), + ...makeToastKernelCallbacks(mountOptions.disableLoadingToast === false), }); ReactDOM.render( From 7bd7814ee23f7e2cf80a80693c6f8f6c72273cc6 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 14:13:50 +0100 Subject: [PATCH 15/26] More cleanup --- packages/kernel/src/worker.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index e34b08086..759ff1112 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -77,11 +77,10 @@ async function loadPyodideAndPackages() { mountedSitePackagesSnapshotFilePath, pyodideUrl = DEFAULT_PYODIDE_URL, streamlitConfig, - disableLoadingToasts, } = await initDataPromiseDelegate.promise; postProgressMessage("Loading Pyodide."); - + console.debug("Loading Pyodide"); pyodide = await initPyodide(pyodideUrl, { stdout: console.log, @@ -136,7 +135,7 @@ async function loadPyodideAndPackages() { if (mountedSitePackagesSnapshotFilePath) { // Restore the site-packages director(y|ies) from the mounted snapshot file. - postProgressMessage("Restoring the snapshot..."); + postProgressMessage("Restoring the snapshot."); await pyodide.runPythonAsync(`import tarfile, shutil, site`); @@ -159,7 +158,7 @@ async function loadPyodideAndPackages() { mockPyArrow(pyodide); console.debug("Mocked pyarrow"); } - + // NOTE: It's important to install the requirements before loading the streamlit package // because it allows users to specify the versions of Streamlit's dependencies via requirements.txt // before these versions are automatically resolved by micropip when installing Streamlit. @@ -278,6 +277,7 @@ async function loadPyodideAndPackages() { console.debug("Mocked some Streamlit functions"); postProgressMessage("Booting up the Streamlit server."); + console.debug("Booting up the Streamlit server"); // The following Python code is based on streamlit.web.cli.main_run(). self.__streamlitFlagOptions__ = { ...streamlitConfig, From 5a71bafb9455e40fdf5b581ffa44aafcc8f604fb Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 14:16:32 +0100 Subject: [PATCH 16/26] Rename of option --- packages/kernel/src/kernel.ts | 6 +++--- packages/kernel/src/types.ts | 2 +- packages/mountable/src/index.tsx | 2 +- packages/mountable/src/options.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index fc94d620f..b06736426 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -108,9 +108,9 @@ export interface StliteKernelOptions { streamlitConfig?: StreamlitConfig; /** - * If true, the loading toasts are disabled. + * If true, no toasts will be shown on loading progress steps. */ - disableLoadingToasts?: boolean; + disableProgressToasts?: boolean; onProgress?: (message: string) => void; @@ -191,7 +191,7 @@ export class StliteKernel { archives: options.archives, requirements: options.requirements, pyodideUrl: options.pyodideUrl, - disableLoadingToasts: options.disableLoadingToasts, + disableProgressToasts: options.disableProgressToasts, wheels, mountedSitePackagesSnapshotFilePath: options.mountedSitePackagesSnapshotFilePath, diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index f2b83d8b7..5959c5598 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -51,7 +51,7 @@ export interface WorkerInitialData { }; mountedSitePackagesSnapshotFilePath?: string; streamlitConfig?: StreamlitConfig; - disableLoadingToasts?: boolean; + disableProgressToasts?: boolean; } /** diff --git a/packages/mountable/src/index.tsx b/packages/mountable/src/index.tsx index 9a0ab651f..961968653 100644 --- a/packages/mountable/src/index.tsx +++ b/packages/mountable/src/index.tsx @@ -39,7 +39,7 @@ export function mount( const kernel = new StliteKernel({ ...mountOptions, wheelBaseUrl, - ...makeToastKernelCallbacks(mountOptions.disableLoadingToast === false), + ...makeToastKernelCallbacks(mountOptions.disableProgressToasts === false), }); ReactDOM.render( diff --git a/packages/mountable/src/options.ts b/packages/mountable/src/options.ts index c4670aa6a..6cea9781c 100644 --- a/packages/mountable/src/options.ts +++ b/packages/mountable/src/options.ts @@ -107,6 +107,6 @@ export function canonicalizeMountOptions( allowedOriginsResp: options.allowedOriginsResp, pyodideUrl: options.pyodideUrl, streamlitConfig: options.streamlitConfig, - disableLoadingToasts: options.disableLoadingToasts, + disableProgressToasts: options.disableProgressToasts, }; } From 167ce79d6c9c04b62038b36959fc578320019220 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 14:19:39 +0100 Subject: [PATCH 17/26] Remove new line --- packages/kernel/src/worker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index 759ff1112..710fade05 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -90,7 +90,6 @@ async function loadPyodideAndPackages() { // Mount files postProgressMessage("Mounting files."); - await Promise.all( Object.keys(files).map(async (path) => { const file = files[path]; From 300985265ede07c14389eb01c7809edadc2dd4ee Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 9 Jan 2024 14:20:56 +0100 Subject: [PATCH 18/26] Minor cleanup --- .../kernel/src/streamlit-replacements/lib/ConnectionManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 557986fc0..878be9560 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -161,7 +161,7 @@ export class ConnectionManager { // Because caching is disabled in stlite. See https://github.com/whitphx/stlite/issues/495 } - /** + /** * No-op in stlite. */ disconnect(): void { From 744fd8e9c48fd5a2fcc1703e4649ca0ca11ed6ab Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Mon, 15 Jan 2024 11:56:52 +0100 Subject: [PATCH 19/26] Fix wrong option name --- packages/mountable/src/options.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mountable/src/options.ts b/packages/mountable/src/options.ts index 6cea9781c..38249e567 100644 --- a/packages/mountable/src/options.ts +++ b/packages/mountable/src/options.ts @@ -15,7 +15,7 @@ export interface SimplifiedStliteKernelOptions { allowedOriginsResp?: StliteKernelOptions["allowedOriginsResp"]; pyodideUrl?: StliteKernelOptions["pyodideUrl"]; streamlitConfig?: StliteKernelOptions["streamlitConfig"]; - disableLoadingToasts?: StliteKernelOptions["disableLoadingToasts"]; + disableProgressToasts?: StliteKernelOptions["disableProgressToasts"]; } function canonicalizeFiles( From 33936c5eb663c30b4de5dff59ea3b25c4f9d6815 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Mon, 15 Jan 2024 12:00:38 +0100 Subject: [PATCH 20/26] Minor renaming --- packages/common-react/src/toastify-components/callbacks.tsx | 6 +++--- packages/mountable/src/index.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/common-react/src/toastify-components/callbacks.tsx b/packages/common-react/src/toastify-components/callbacks.tsx index 0a46771eb..c298a1f60 100644 --- a/packages/common-react/src/toastify-components/callbacks.tsx +++ b/packages/common-react/src/toastify-components/callbacks.tsx @@ -8,11 +8,11 @@ interface ToastKernelCallbacks { onLoad: NonNullable; onError: NonNullable; } -export function makeToastKernelCallbacks(showProgressToasts = true, showErrorToasts = true): ToastKernelCallbacks { +export function makeToastKernelCallbacks(disableProgressToasts = false, disableErrorToasts = false): ToastKernelCallbacks { let prevToastId: ToastId | null = null; const toastIds: ToastId[] = []; const onProgress: StliteKernelOptions["onProgress"] = (message) => { - if (!showProgressToasts) { + if (disableProgressToasts) { return; } @@ -37,7 +37,7 @@ export function makeToastKernelCallbacks(showProgressToasts = true, showErrorToa toastIds.forEach((id) => toast.dismiss(id)); }; const onError: StliteKernelOptions["onError"] = (error) => { - if (!showErrorToasts) { + if (disableErrorToasts) { return; } diff --git a/packages/mountable/src/index.tsx b/packages/mountable/src/index.tsx index 961968653..2aac4485e 100644 --- a/packages/mountable/src/index.tsx +++ b/packages/mountable/src/index.tsx @@ -39,7 +39,7 @@ export function mount( const kernel = new StliteKernel({ ...mountOptions, wheelBaseUrl, - ...makeToastKernelCallbacks(mountOptions.disableProgressToasts === false), + ...makeToastKernelCallbacks(mountOptions.disableProgressToasts === true), }); ReactDOM.render( From c1ff261a26226e429242f8d50c331135968ebc6a Mon Sep 17 00:00:00 2001 From: Lukas Masuch Date: Tue, 16 Jan 2024 12:58:25 +0100 Subject: [PATCH 21/26] Fix typo Co-authored-by: Yuichiro Tachibana (Tsuchiya) --- packages/kernel/py/stlite-server/stlite_server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel/py/stlite-server/stlite_server/server.py b/packages/kernel/py/stlite-server/stlite_server/server.py index 23e07d0c5..5bf14f408 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server.py +++ b/packages/kernel/py/stlite-server/stlite_server/server.py @@ -161,7 +161,7 @@ def receive_http( return method_name = method.lower() if method_name not in ("get", "post", "delete"): - on_response(405, {}, b"Now allowed") + on_response(405, {}, b"Not allowed") return handler_method = getattr(handler, method_name, None) if handler_method is None: From f8d4782689bb750bdd8b8ee0699fa9ffb3495633 Mon Sep 17 00:00:00 2001 From: Lukas Masuch Date: Tue, 16 Jan 2024 13:05:27 +0100 Subject: [PATCH 22/26] Update comment Co-authored-by: Yuichiro Tachibana (Tsuchiya) --- packages/kernel/py/stlite-server/stlite_server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel/py/stlite-server/stlite_server/server.py b/packages/kernel/py/stlite-server/stlite_server/server.py index 5bf14f408..516616d65 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server.py +++ b/packages/kernel/py/stlite-server/stlite_server/server.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger(__name__) -# These route definitions are copied from the original impl at https://github.com/streamlit/streamlit/blob/1.18.1/lib/streamlit/web/server/server.py#L81-L89 # noqa: E501 +# These route definitions are copied from the original impl at https://github.com/streamlit/streamlit/blob/1.27.0/lib/streamlit/web/server/server.py#L83-L92 # noqa: E501 UPLOAD_FILE_ENDPOINT: Final = "/_stcore/upload_file" MEDIA_ENDPOINT: Final = "/media" STREAM_ENDPOINT: Final = "_stcore/stream" From 89d986b7e8ceb620293bc4e893b7c01a9ed7f062 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 16 Jan 2024 13:31:12 +0100 Subject: [PATCH 23/26] Remove paths --- packages/common-react/tsconfig.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/common-react/tsconfig.json b/packages/common-react/tsconfig.json index 0028dd6bf..01d6d4a35 100644 --- a/packages/common-react/tsconfig.json +++ b/packages/common-react/tsconfig.json @@ -28,11 +28,7 @@ // "rootDir": "./", /* Specify the root folder within your source files. */ "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - "paths": { - "theme/src": ["../../streamlit/frontend/lib/src/theme"], - "theme/src/*": ["../../streamlit/frontend/lib/src/theme/*"], - "lib/src/*": ["../../streamlit/frontend/lib/src/*"], - } /* Specify a set of entries that re-map imports to additional lookup locations. */, + // "paths": {} /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */ From b6e8ef55315080a279a81035b622e7c74f9d2806 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 16 Jan 2024 13:34:53 +0100 Subject: [PATCH 24/26] Change to put --- packages/kernel/py/stlite-server/stlite_server/handler.py | 2 +- packages/kernel/py/stlite-server/stlite_server/server.py | 2 +- .../stlite-server/stlite_server/upload_file_request_handler.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/kernel/py/stlite-server/stlite_server/handler.py b/packages/kernel/py/stlite-server/stlite_server/handler.py index 2b35e2a50..5be975ce8 100644 --- a/packages/kernel/py/stlite-server/stlite_server/handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/handler.py @@ -19,7 +19,7 @@ class RequestHandler(abc.ABC): def get(self, request: Request) -> Response | Awaitable[Response]: return Response(status_code=405, headers={}, body="") - def post(self, request: Request) -> Response | Awaitable[Response]: + def put(self, request: Request) -> Response | Awaitable[Response]: return Response(status_code=405, headers={}, body="") def delete(self, request: Request) -> Response | Awaitable[Response]: diff --git a/packages/kernel/py/stlite-server/stlite_server/server.py b/packages/kernel/py/stlite-server/stlite_server/server.py index 516616d65..7991b7fea 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server.py +++ b/packages/kernel/py/stlite-server/stlite_server/server.py @@ -160,7 +160,7 @@ def receive_http( on_response(404, {}, b"No handler found") return method_name = method.lower() - if method_name not in ("get", "post", "delete"): + if method_name not in ("get", "put", "delete"): on_response(405, {}, b"Not allowed") return handler_method = getattr(handler, method_name, None) diff --git a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py index 1203d9e3f..fefa7b600 100644 --- a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py @@ -41,7 +41,7 @@ def _require_arg(args: Dict[str, List[bytes]], name: str) -> str: # Convert bytes to string return arg[0].decode("utf-8") - def post(self, request: Request, **kwargs) -> Response: + def put(self, request: Request, **kwargs) -> Response: # NOTE: The original implementation uses an async function, # but it didn't make use of any async features, # so we made it a regular function here for simplicity sake. From 1b1bd7e0570ffe424cc35673c7a90153411af2f0 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 16 Jan 2024 13:42:10 +0100 Subject: [PATCH 25/26] Update URL --- packages/kernel/py/stlite-server/stlite_server/server_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel/py/stlite-server/stlite_server/server_util.py b/packages/kernel/py/stlite-server/stlite_server/server_util.py index f25d03a23..a921781a8 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server_util.py +++ b/packages/kernel/py/stlite-server/stlite_server/server_util.py @@ -1,4 +1,4 @@ -# Copied from https://github.com/streamlit/streamlit/blob/1.18.1/lib/streamlit/web/server/server_util.py#L73-L77 # noqa: E501 +# Copied from https://github.com/streamlit/streamlit/blob/1.27.1/lib/streamlit/web/server/server_util.py#L73-L77 # noqa: E501 def make_url_path_regex(*path: str, **kwargs) -> str: """Get a regex of the form ^/foo/bar/baz/?$ for a path (foo, bar, baz).""" valid_path = [x.strip("/") for x in path if x] # Filter out falsely components. From 899730a45f0ce9a04ff49ef42c9e776e23264cc0 Mon Sep 17 00:00:00 2001 From: lukasmasuch Date: Tue, 16 Jan 2024 14:09:09 +0100 Subject: [PATCH 26/26] Add gatherUsageStats flag --- packages/kernel/src/worker.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index 710fade05..c25b582ad 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -279,6 +279,8 @@ async function loadPyodideAndPackages() { console.debug("Booting up the Streamlit server"); // The following Python code is based on streamlit.web.cli.main_run(). self.__streamlitFlagOptions__ = { + // gatherUsageStats is disabled as default, but can be enabled explicitly by setting it to true. + "browser.gatherUsageStats": false, ...streamlitConfig, "runner.fastReruns": false, // Fast reruns do not work well with the async script runner of stlite. See https://github.com/whitphx/stlite/pull/550#issuecomment-1505485865. };