From 4ff1970daee604b407aab2f4bac06e49959bcaba Mon Sep 17 00:00:00 2001 From: "Yuichiro Tachibana (Tsuchiya)" Date: Wed, 3 Apr 2024 14:47:49 +0900 Subject: [PATCH 1/3] Rename builtin packages to prebuilt packages --- .../desktop/bin-src/dump_artifacts/index.ts | 58 +++++++++---------- .../dump_artifacts/pyodide_packages.ts | 12 ++-- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/packages/desktop/bin-src/dump_artifacts/index.ts b/packages/desktop/bin-src/dump_artifacts/index.ts index e7d2aa617..a6651688b 100755 --- a/packages/desktop/bin-src/dump_artifacts/index.ts +++ b/packages/desktop/bin-src/dump_artifacts/index.ts @@ -9,7 +9,7 @@ import fetch from "node-fetch"; import { loadPyodide, type PyodideInterface } from "pyodide"; import { parseRequirementsTxt, verifyRequirements } from "@stlite/common"; import { makePyodideUrl } from "./url"; -import { PyodideBuiltinPackagesData } from "./pyodide_packages"; +import { PyodidePrebuiltPackagesData } from "./pyodide_packages"; import { dumpManifest } from "./manifest"; // @ts-ignore @@ -76,16 +76,16 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) { await fsExtra.copy(sourceDir, options.copyTo); } -interface InspectUsedBuiltinPackagesOptions { +interface InspectUsedPrebuiltPackagesOptions { requirements: string[]; } /** - * Get the list of the built-in packages used by the given requirements. + * Get the list of the prebuilt packages used by the given requirements. * These package files (`pyodide/*.whl`) will be vendored in the app executable * and loaded at runtime to avoid problems such as https://github.com/whitphx/stlite/issues/558 */ -async function inspectUsedBuiltinPackages( - options: InspectUsedBuiltinPackagesOptions +async function inspectUsedPrebuiltPackages( + options: InspectUsedPrebuiltPackagesOptions ): Promise { if (options.requirements.length === 0) { return []; @@ -147,7 +147,7 @@ async function installPackages( interface CreateSitePackagesSnapshotOptions { requirements: string[]; - usedBuiltinPackages: string[]; + usedPrebuiltPackages: string[]; saveTo: string; } async function createSitePackagesSnapshot( @@ -160,16 +160,16 @@ async function createSitePackagesSnapshot( await ensureLoadPackage(pyodide, "micropip"); const micropip = pyodide.pyimport("micropip"); - const pyodideBuiltinPackagesData = - await PyodideBuiltinPackagesData.getInstance(); + const pyodidePrebuiltPackagesData = + await PyodidePrebuiltPackagesData.getInstance(); const mockedPackages: string[] = []; - if (options.usedBuiltinPackages.length > 0) { + if (options.usedPrebuiltPackages.length > 0) { console.log( - "Mocking builtin packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..." + "Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..." ); - options.usedBuiltinPackages.forEach((pkg) => { - const packageInfo = pyodideBuiltinPackagesData.getPackageInfoByName(pkg); + options.usedPrebuiltPackages.forEach((pkg) => { + const packageInfo = pyodidePrebuiltPackagesData.getPackageInfoByName(pkg); if (packageInfo == null) { throw new Error(`Package ${pkg} is not found in the lock file.`); } @@ -246,25 +246,25 @@ async function writeRequirements( }); } -interface DownloadPyodideBuiltinPackageWheelsOptions { +interface DownloadPyodidePrebuiltPackageWheelsOptions { packages: string[]; destDir: string; } -async function downloadPyodideBuiltinPackageWheels( - options: DownloadPyodideBuiltinPackageWheelsOptions +async function downloadPyodidePrebuiltPackageWheels( + options: DownloadPyodidePrebuiltPackageWheelsOptions ) { - const pyodideBuiltinPackagesData = - await PyodideBuiltinPackagesData.getInstance(); - const usedBuiltInPackages = options.packages.map((pkgName) => - pyodideBuiltinPackagesData.getPackageInfoByName(pkgName) + const pyodidePrebuiltPackagesData = + await PyodidePrebuiltPackagesData.getInstance(); + const usedPrebuiltPackages = options.packages.map((pkgName) => + pyodidePrebuiltPackagesData.getPackageInfoByName(pkgName) ); - const usedBuiltinPackageUrls = usedBuiltInPackages.map((pkg) => + const usedPrebuiltPackageUrls = usedPrebuiltPackages.map((pkg) => makePyodideUrl(pkg.file_name) ); - console.log("Downloading the used built-in packages..."); + console.log("Downloading the used prebuilt packages..."); await Promise.all( - usedBuiltinPackageUrls.map(async (pkgUrl) => { + usedPrebuiltPackageUrls.map(async (pkgUrl) => { const dstPath = path.resolve( options.destDir, "./pyodide", @@ -336,16 +336,16 @@ yargs(hideBin(process.argv)) } verifyRequirements(requirements); - const usedBuiltinPackages = await inspectUsedBuiltinPackages({ + const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({ requirements: requirements, }); - console.log("The built-in packages loaded for the given requirements:"); - console.log(usedBuiltinPackages); + console.log("The prebuilt packages loaded for the given requirements:"); + console.log(usedPrebuiltPackages); await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild }); await createSitePackagesSnapshot({ requirements: requirements, - usedBuiltinPackages, + usedPrebuiltPackages, saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts. }); // The `requirements.txt` file will be needed to call `micropip.install()` at runtime. @@ -354,14 +354,14 @@ yargs(hideBin(process.argv)) // while the packages downloaded from PyPI will have been included in the site-packages snapshot. await writeRequirements( path.resolve(destDir, "./requirements.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts. - usedBuiltinPackages + usedPrebuiltPackages ); await copyStreamlitAppDirectory({ sourceDir: args.appHomeDirSource, copyTo: path.resolve(destDir, "./streamlit_app"), // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts. }); - await downloadPyodideBuiltinPackageWheels({ - packages: usedBuiltinPackages, + await downloadPyodidePrebuiltPackageWheels({ + packages: usedPrebuiltPackages, destDir, }); await dumpManifest({ diff --git a/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts b/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts index 57b356948..9b080eb31 100644 --- a/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts +++ b/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts @@ -7,13 +7,13 @@ interface PackageInfo { file_name: string; depends: string[]; } -export class PyodideBuiltinPackagesData { - private static _instance: PyodideBuiltinPackagesData; +export class PyodidePrebuiltPackagesData { + private static _instance: PyodidePrebuiltPackagesData; private _data: Record | null = null; private constructor() {} - private static async loadPyodideBuiltinPackageData(): Promise< + private static async loadPyodidePrebuiltPackageData(): Promise< Record > { const url = makePyodideUrl("pyodide-lock.json"); @@ -25,10 +25,10 @@ export class PyodideBuiltinPackagesData { return resJson.packages; } - static async getInstance(): Promise { + static async getInstance(): Promise { if (this._instance == null) { - this._instance = new PyodideBuiltinPackagesData(); - this._instance._data = await this.loadPyodideBuiltinPackageData(); + this._instance = new PyodidePrebuiltPackagesData(); + this._instance._data = await this.loadPyodidePrebuiltPackageData(); } return this._instance; } From e25d8d68838a6b56724285a8f358b8dea3433152 Mon Sep 17 00:00:00 2001 From: "Yuichiro Tachibana (Tsuchiya)" Date: Wed, 3 Apr 2024 15:27:50 +0900 Subject: [PATCH 2/3] Add `prebuiltPackageNames` option to install prebuilt packages by `pyodide.loadPackage()` and fix desktop tp use it --- .../desktop/bin-src/dump_artifacts/index.ts | 24 ++++++++++--------- packages/desktop/electron/main.ts | 22 ++++++++++------- packages/desktop/electron/preload.ts | 3 ++- packages/desktop/src/App.tsx | 11 ++++++--- packages/kernel/src/kernel.ts | 7 ++++++ packages/kernel/src/types.ts | 1 + packages/kernel/src/worker-runtime.ts | 6 +++++ packages/mountable/src/options.test.ts | 20 +++++++++++++++- packages/mountable/src/options.ts | 3 +++ 9 files changed, 73 insertions(+), 24 deletions(-) diff --git a/packages/desktop/bin-src/dump_artifacts/index.ts b/packages/desktop/bin-src/dump_artifacts/index.ts index a6651688b..3e9cf4e97 100755 --- a/packages/desktop/bin-src/dump_artifacts/index.ts +++ b/packages/desktop/bin-src/dump_artifacts/index.ts @@ -236,12 +236,12 @@ async function readRequirements( return parseRequirementsTxt(requirementsTxtData); } -async function writeRequirements( - requirementsTxtPath: string, - requirements: string[] +async function writePrebuiltPackagesTxt( + prebuiltPackagesTxtPath: string, + prebuiltPackages: string[] ): Promise { - const requirementsTxtData = requirements.join("\n"); - await fsPromises.writeFile(requirementsTxtPath, requirementsTxtData, { + const prebuiltPackagesTxtData = prebuiltPackages.join("\n"); + await fsPromises.writeFile(prebuiltPackagesTxtPath, prebuiltPackagesTxtData, { encoding: "utf-8", }); } @@ -348,12 +348,14 @@ yargs(hideBin(process.argv)) usedPrebuiltPackages, saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts. }); - // The `requirements.txt` file will be needed to call `micropip.install()` at runtime. - // The Pyodide-built packages will be vendored in the build artifact as wheel files - // and `micropip.install()` will install them at runtime, - // while the packages downloaded from PyPI will have been included in the site-packages snapshot. - await writeRequirements( - path.resolve(destDir, "./requirements.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts. + // These prebuilt packages will be vendored in the build artifact by `downloadPyodidePrebuiltPackageWheels()` + // and the package names will be saved in the `./prebuilt-packages.txt` file + // so that they will be read and passed to `pyodide.loadPackage()` at runtime to install them from the vendored files. + // While the packages downloaded from PyPI at build time will have been shipped in the site-packages snapshot by `createSitePackagesSnapshot()`, + // the prebuilt packages must be installed at runtime by `pyodide.loadPackage()` or `micropip.install()` + // to avoid problems such as https://github.com/whitphx/stlite/issues/564. + await writePrebuiltPackagesTxt( + path.resolve(destDir, "./prebuilt-packages.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts. usedPrebuiltPackages ); await copyStreamlitAppDirectory({ diff --git a/packages/desktop/electron/main.ts b/packages/desktop/electron/main.ts index 2abe375be..c1ddae739 100644 --- a/packages/desktop/electron/main.ts +++ b/packages/desktop/electron/main.ts @@ -73,19 +73,25 @@ const createWindow = async () => { ); return fsPromises.readFile(archiveFilePath); }); - ipcMain.handle("readRequirements", async (ev): Promise => { + ipcMain.handle("readPrebuiltPackageNames", async (ev): Promise => { if (!isValidIpcSender(ev.senderFrame)) { throw new Error("Invalid IPC sender"); } - const requirementsTxtPath = path.resolve(__dirname, "../requirements.txt"); - const requirementsTxtData = await fsPromises.readFile(requirementsTxtPath, { - encoding: "utf-8", - }); - return requirementsTxtData + const prebuiltPackagesTxtPath = path.resolve( + __dirname, + "../prebuilt-packages.txt" + ); + const prebuiltPackagesTxtData = await fsPromises.readFile( + prebuiltPackagesTxtPath, + { + encoding: "utf-8", + } + ); + return prebuiltPackagesTxtData .split("\n") .map((r) => r.trim()) - .filter((r) => r.length > 0); // Assuming that the input `requirements.txt` file is generated by `dump-artifacts.js` and can be parsed with this simple logic. + .filter((r) => r.length > 0); // Assuming that the input file is generated by `dump-artifacts.js` so can be parsed with this simple logic. }); ipcMain.handle( "readStreamlitAppDirectory", @@ -101,7 +107,7 @@ const createWindow = async () => { mainWindow.on("closed", () => { ipcMain.removeHandler("readSitePackagesSnapshot"); - ipcMain.removeHandler("readRequirements"); + ipcMain.removeHandler("readPrebuiltPackageNames"); ipcMain.removeHandler("readStreamlitAppDirectory"); }); diff --git a/packages/desktop/electron/preload.ts b/packages/desktop/electron/preload.ts index 5e96171da..0023a0192 100644 --- a/packages/desktop/electron/preload.ts +++ b/packages/desktop/electron/preload.ts @@ -13,7 +13,8 @@ export type AppConfig = typeof appConfig; const archivesAPI = { readSitePackagesSnapshot: () => ipcRenderer.invoke("readSitePackagesSnapshot"), - readRequirements: () => ipcRenderer.invoke("readRequirements"), + readPrebuiltPackageNames: () => + ipcRenderer.invoke("readPrebuiltPackageNames"), readStreamlitAppDirectory: () => ipcRenderer.invoke("readStreamlitAppDirectory"), }; diff --git a/packages/desktop/src/App.tsx b/packages/desktop/src/App.tsx index 28c57bb9c..2670fc536 100644 --- a/packages/desktop/src/App.tsx +++ b/packages/desktop/src/App.tsx @@ -25,10 +25,14 @@ function App() { Promise.all([ window.archivesAPI.readSitePackagesSnapshot(), - window.archivesAPI.readRequirements(), + window.archivesAPI.readPrebuiltPackageNames(), window.archivesAPI.readStreamlitAppDirectory(), ]).then( - ([sitePackagesSnapshotFileBin, requirements, streamlitAppFiles]) => { + ([ + sitePackagesSnapshotFileBin, + prebuiltPackageNames, + streamlitAppFiles, + ]) => { if (unmounted) { return; } @@ -52,7 +56,8 @@ function App() { }, }, archives: [], - requirements, + requirements: [], + prebuiltPackageNames, mountedSitePackagesSnapshotFilePath, pyodideUrl, idbfsMountpoints: window.appConfig.idbfsMountpoints, diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index d10f0588a..14e3e27b0 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -47,6 +47,12 @@ export interface StliteKernelOptions { */ requirements: string[]; + /** + * A list of prebuilt package names to be install at the booting-up phase via `pyodide.loadPackage()`. + * These packages basically can be installed via the `requirements` option, but this option is for the packages that are not available in the PyPI. + */ + prebuiltPackageNames: string[]; + /** * Files to mount. */ @@ -195,6 +201,7 @@ export class StliteKernel { files: options.files, archives: options.archives, requirements: options.requirements, + prebuiltPackageNames: options.prebuiltPackageNames, pyodideUrl: options.pyodideUrl, wheels, mountedSitePackagesSnapshotFilePath: diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index 7afeb94cb..79706656b 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -44,6 +44,7 @@ export interface WorkerInitialData { files: Record; archives: Array; requirements: string[]; + prebuiltPackageNames: string[]; pyodideUrl?: string; wheels?: { stliteServer: string; diff --git a/packages/kernel/src/worker-runtime.ts b/packages/kernel/src/worker-runtime.ts index 34b501a8f..6e44631a2 100644 --- a/packages/kernel/src/worker-runtime.ts +++ b/packages/kernel/src/worker-runtime.ts @@ -80,6 +80,7 @@ export function startWorkerEnv( files, archives, requirements, + prebuiltPackageNames: prebuiltPackages, wheels, mountedSitePackagesSnapshotFilePath, pyodideUrl = defaultPyodideUrl, @@ -209,6 +210,11 @@ with tarfile.open("${mountedSitePackagesSnapshotFilePath}", "r") as tar_gz_file: // === // Also, this must be after restoring the snapshot because the snapshot may contain the site-packages. postProgressMessage("Installing packages."); + + console.debug("Installing the prebuilt packages:", prebuiltPackages); + await pyodide.loadPackage(prebuiltPackages); + console.debug("Installed the prebuilt packages"); + await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); if (wheels) { diff --git a/packages/mountable/src/options.test.ts b/packages/mountable/src/options.test.ts index 68992af0f..d26f5049c 100644 --- a/packages/mountable/src/options.test.ts +++ b/packages/mountable/src/options.test.ts @@ -63,11 +63,12 @@ describe("parseMountOptions()", () => { }, }, requirements: [], + prebuiltPackageNames: [], archives: [], }); }); - it("fills `command`, `entrypoint`, and `requirements` fields and converts the `files` into the canonical form", () => { + it("fills `entrypoint`, and `requirements` fields and converts the `files` into the canonical form", () => { const { kernelOptions } = parseMountOptions({ files: { "streamlit_app.py": "foo", @@ -82,6 +83,7 @@ describe("parseMountOptions()", () => { expect(kernelOptions).toEqual({ entrypoint: "streamlit_app.py", requirements: [], + prebuiltPackageNames: [], files: { "streamlit_app.py": { data: "foo", @@ -120,6 +122,7 @@ describe("parseMountOptions()", () => { expect(kernelOptions).toEqual({ entrypoint: "streamlit_app.py", requirements: [], + prebuiltPackageNames: [], files: {}, archives: [ { @@ -151,6 +154,7 @@ describe("parseMountOptions()", () => { expect(kernelOptions).toEqual({ entrypoint: "foo.py", requirements: [], + prebuiltPackageNames: [], files: { "streamlit_app.py": { data: "foo", @@ -166,6 +170,20 @@ describe("parseMountOptions()", () => { }); expect(kernelOptions).toEqual({ requirements: ["matplotlib"], + prebuiltPackageNames: [], + entrypoint: "streamlit_app.py", + files: {}, + archives: [], + }); + }); + + it("preserves the `prebuiltPackageNames` option if specified", () => { + const { kernelOptions } = parseMountOptions({ + prebuiltPackageNames: ["openssl"], + }); + expect(kernelOptions).toEqual({ + requirements: [], + prebuiltPackageNames: ["openssl"], entrypoint: "streamlit_app.py", files: {}, archives: [], diff --git a/packages/mountable/src/options.ts b/packages/mountable/src/options.ts index 6c44ffa62..007bd65c7 100644 --- a/packages/mountable/src/options.ts +++ b/packages/mountable/src/options.ts @@ -8,6 +8,7 @@ import type { MakeToastKernelCallbacksOptions } from "@stlite/common-react"; export interface SimplifiedStliteKernelOptions { entrypoint?: string; requirements?: StliteKernelOptions["requirements"]; + prebuiltPackageNames?: StliteKernelOptions["prebuiltPackageNames"]; files?: Record< string, EmscriptenFile | EmscriptenFileUrl | EmscriptenFile["data"] // EmscriptenFile["data"] is allowed as a shorthand for convenience. @@ -98,6 +99,7 @@ export function parseMountOptions(options: MountOptions): { }, archives: [], requirements: [], + prebuiltPackageNames: [], }, toastCallbackOptions: { disableProgressToasts: false, @@ -115,6 +117,7 @@ export function parseMountOptions(options: MountOptions): { files, archives, requirements: options.requirements || [], + prebuiltPackageNames: options.prebuiltPackageNames || [], hostConfigResponse: options.hostConfig, pyodideUrl: options.pyodideUrl, streamlitConfig: options.streamlitConfig, From ac40cde765078857d7c76cec6983d5a159996faf Mon Sep 17 00:00:00 2001 From: "Yuichiro Tachibana (Tsuchiya)" Date: Wed, 3 Apr 2024 16:26:06 +0900 Subject: [PATCH 3/3] Fix comment --- packages/kernel/src/kernel.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index 14e3e27b0..5ab12d94a 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -49,7 +49,8 @@ export interface StliteKernelOptions { /** * A list of prebuilt package names to be install at the booting-up phase via `pyodide.loadPackage()`. - * These packages basically can be installed via the `requirements` option, but this option is for the packages that are not available in the PyPI. + * These packages basically can be installed via the `requirements` option, + * but some are only installable via this option such as `openssl`. */ prebuiltPackageNames: string[];