Fix/desktop load prebuilt packages separately (#833)

* Rename builtin packages to prebuilt packages

* Add `prebuiltPackageNames` option to install prebuilt packages by `pyodide.loadPackage()` and fix desktop tp use it

* Fix comment

packages/desktop/bin-src/dump_artifacts/index.ts

@@ -9,7 +9,7 @@ import fetch from "node-fetch";
 import { loadPyodide, type PyodideInterface } from "pyodide";
 import { parseRequirementsTxt, verifyRequirements } from "@stlite/common";
 import { makePyodideUrl } from "./url";
-import { PyodideBuiltinPackagesData } from "./pyodide_packages";
+import { PyodidePrebuiltPackagesData } from "./pyodide_packages";
 import { dumpManifest } from "./manifest";
 
 // @ts-ignore
@@ -76,16 +76,16 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
   await fsExtra.copy(sourceDir, options.copyTo);
 }
 
-interface InspectUsedBuiltinPackagesOptions {
+interface InspectUsedPrebuiltPackagesOptions {
   requirements: string[];
 }
 /**
- * Get the list of the built-in packages used by the given requirements.
+ * Get the list of the prebuilt packages used by the given requirements.
  * These package files (`pyodide/*.whl`) will be vendored in the app executable
  * and loaded at runtime to avoid problems such as https://github.com/whitphx/stlite/issues/558
  */
-async function inspectUsedBuiltinPackages(
-  options: InspectUsedBuiltinPackagesOptions
+async function inspectUsedPrebuiltPackages(
+  options: InspectUsedPrebuiltPackagesOptions
 ): Promise<string[]> {
   if (options.requirements.length === 0) {
     return [];
@@ -147,7 +147,7 @@ async function installPackages(
 
 interface CreateSitePackagesSnapshotOptions {
   requirements: string[];
-  usedBuiltinPackages: string[];
+  usedPrebuiltPackages: string[];
   saveTo: string;
 }
 async function createSitePackagesSnapshot(
@@ -160,16 +160,16 @@ async function createSitePackagesSnapshot(
   await ensureLoadPackage(pyodide, "micropip");
   const micropip = pyodide.pyimport("micropip");
 
-  const pyodideBuiltinPackagesData =
-    await PyodideBuiltinPackagesData.getInstance();
+  const pyodidePrebuiltPackagesData =
+    await PyodidePrebuiltPackagesData.getInstance();
 
   const mockedPackages: string[] = [];
-  if (options.usedBuiltinPackages.length > 0) {
+  if (options.usedPrebuiltPackages.length > 0) {
     console.log(
-      "Mocking builtin packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
+      "Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
     );
-    options.usedBuiltinPackages.forEach((pkg) => {
-      const packageInfo = pyodideBuiltinPackagesData.getPackageInfoByName(pkg);
+    options.usedPrebuiltPackages.forEach((pkg) => {
+      const packageInfo = pyodidePrebuiltPackagesData.getPackageInfoByName(pkg);
       if (packageInfo == null) {
         throw new Error(`Package ${pkg} is not found in the lock file.`);
       }
@@ -236,35 +236,35 @@ async function readRequirements(
   return parseRequirementsTxt(requirementsTxtData);
 }
 
-async function writeRequirements(
-  requirementsTxtPath: string,
-  requirements: string[]
+async function writePrebuiltPackagesTxt(
+  prebuiltPackagesTxtPath: string,
+  prebuiltPackages: string[]
 ): Promise<void> {
-  const requirementsTxtData = requirements.join("\n");
-  await fsPromises.writeFile(requirementsTxtPath, requirementsTxtData, {
+  const prebuiltPackagesTxtData = prebuiltPackages.join("\n");
+  await fsPromises.writeFile(prebuiltPackagesTxtPath, prebuiltPackagesTxtData, {
     encoding: "utf-8",
   });
 }
 
-interface DownloadPyodideBuiltinPackageWheelsOptions {
+interface DownloadPyodidePrebuiltPackageWheelsOptions {
   packages: string[];
   destDir: string;
 }
-async function downloadPyodideBuiltinPackageWheels(
-  options: DownloadPyodideBuiltinPackageWheelsOptions
+async function downloadPyodidePrebuiltPackageWheels(
+  options: DownloadPyodidePrebuiltPackageWheelsOptions
 ) {
-  const pyodideBuiltinPackagesData =
-    await PyodideBuiltinPackagesData.getInstance();
-  const usedBuiltInPackages = options.packages.map((pkgName) =>
-    pyodideBuiltinPackagesData.getPackageInfoByName(pkgName)
+  const pyodidePrebuiltPackagesData =
+    await PyodidePrebuiltPackagesData.getInstance();
+  const usedPrebuiltPackages = options.packages.map((pkgName) =>
+    pyodidePrebuiltPackagesData.getPackageInfoByName(pkgName)
   );
-  const usedBuiltinPackageUrls = usedBuiltInPackages.map((pkg) =>
+  const usedPrebuiltPackageUrls = usedPrebuiltPackages.map((pkg) =>
     makePyodideUrl(pkg.file_name)
   );
 
-  console.log("Downloading the used built-in packages...");
+  console.log("Downloading the used prebuilt packages...");
   await Promise.all(
-    usedBuiltinPackageUrls.map(async (pkgUrl) => {
+    usedPrebuiltPackageUrls.map(async (pkgUrl) => {
       const dstPath = path.resolve(
         options.destDir,
         "./pyodide",
@@ -336,32 +336,34 @@ yargs(hideBin(process.argv))
     }
     verifyRequirements(requirements);
 
-    const usedBuiltinPackages = await inspectUsedBuiltinPackages({
+    const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({
       requirements: requirements,
     });
-    console.log("The built-in packages loaded for the given requirements:");
-    console.log(usedBuiltinPackages);
+    console.log("The prebuilt packages loaded for the given requirements:");
+    console.log(usedPrebuiltPackages);
 
     await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
     await createSitePackagesSnapshot({
       requirements: requirements,
-      usedBuiltinPackages,
+      usedPrebuiltPackages,
       saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
     });
-    // The `requirements.txt` file will be needed to call `micropip.install()` at runtime.
-    // The Pyodide-built packages will be vendored in the build artifact as wheel files
-    // and `micropip.install()` will install them at runtime,
-    // while the packages downloaded from PyPI will have been included in the site-packages snapshot.
-    await writeRequirements(
-      path.resolve(destDir, "./requirements.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts.
-      usedBuiltinPackages
+    // These prebuilt packages will be vendored in the build artifact by `downloadPyodidePrebuiltPackageWheels()`
+    // and the package names will be saved in the `./prebuilt-packages.txt` file
+    // so that they will be read and passed to `pyodide.loadPackage()` at runtime to install them from the vendored files.
+    // While the packages downloaded from PyPI at build time will have been shipped in the site-packages snapshot by `createSitePackagesSnapshot()`,
+    // the prebuilt packages must be installed at runtime by `pyodide.loadPackage()` or `micropip.install()`
+    // to avoid problems such as https://github.com/whitphx/stlite/issues/564.
+    await writePrebuiltPackagesTxt(
+      path.resolve(destDir, "./prebuilt-packages.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts.
+      usedPrebuiltPackages
     );
     await copyStreamlitAppDirectory({
       sourceDir: args.appHomeDirSource,
       copyTo: path.resolve(destDir, "./streamlit_app"), // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
     });
-    await downloadPyodideBuiltinPackageWheels({
-      packages: usedBuiltinPackages,
+    await downloadPyodidePrebuiltPackageWheels({
+      packages: usedPrebuiltPackages,
       destDir,
     });
     await dumpManifest({

packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts

@@ -7,13 +7,13 @@ interface PackageInfo {
   file_name: string;
   depends: string[];
 }
-export class PyodideBuiltinPackagesData {
-  private static _instance: PyodideBuiltinPackagesData;
+export class PyodidePrebuiltPackagesData {
+  private static _instance: PyodidePrebuiltPackagesData;
   private _data: Record<string, PackageInfo> | null = null;
 
   private constructor() {}
 
-  private static async loadPyodideBuiltinPackageData(): Promise<
+  private static async loadPyodidePrebuiltPackageData(): Promise<
     Record<string, PackageInfo>
   > {
     const url = makePyodideUrl("pyodide-lock.json");
@@ -25,10 +25,10 @@ export class PyodideBuiltinPackagesData {
     return resJson.packages;
   }
 
-  static async getInstance(): Promise<PyodideBuiltinPackagesData> {
+  static async getInstance(): Promise<PyodidePrebuiltPackagesData> {
     if (this._instance == null) {
-      this._instance = new PyodideBuiltinPackagesData();
-      this._instance._data = await this.loadPyodideBuiltinPackageData();
+      this._instance = new PyodidePrebuiltPackagesData();
+      this._instance._data = await this.loadPyodidePrebuiltPackageData();
     }
     return this._instance;
   }

packages/desktop/electron/main.ts

@@ -73,19 +73,25 @@ const createWindow = async () => {
     );
     return fsPromises.readFile(archiveFilePath);
   });
-  ipcMain.handle("readRequirements", async (ev): Promise<string[]> => {
+  ipcMain.handle("readPrebuiltPackageNames", async (ev): Promise<string[]> => {
     if (!isValidIpcSender(ev.senderFrame)) {
       throw new Error("Invalid IPC sender");
     }
 
-    const requirementsTxtPath = path.resolve(__dirname, "../requirements.txt");
-    const requirementsTxtData = await fsPromises.readFile(requirementsTxtPath, {
-      encoding: "utf-8",
-    });
-    return requirementsTxtData
+    const prebuiltPackagesTxtPath = path.resolve(
+      __dirname,
+      "../prebuilt-packages.txt"
+    );
+    const prebuiltPackagesTxtData = await fsPromises.readFile(
+      prebuiltPackagesTxtPath,
+      {
+        encoding: "utf-8",
+      }
+    );
+    return prebuiltPackagesTxtData
       .split("\n")
       .map((r) => r.trim())
-      .filter((r) => r.length > 0); // Assuming that the input `requirements.txt` file is generated by `dump-artifacts.js` and can be parsed with this simple logic.
+      .filter((r) => r.length > 0); // Assuming that the input file is generated by `dump-artifacts.js` so can be parsed with this simple logic.
   });
   ipcMain.handle(
     "readStreamlitAppDirectory",
@@ -101,7 +107,7 @@ const createWindow = async () => {
 
   mainWindow.on("closed", () => {
     ipcMain.removeHandler("readSitePackagesSnapshot");
-    ipcMain.removeHandler("readRequirements");
+    ipcMain.removeHandler("readPrebuiltPackageNames");
     ipcMain.removeHandler("readStreamlitAppDirectory");
   });
 

packages/desktop/electron/preload.ts

@@ -13,7 +13,8 @@ export type AppConfig = typeof appConfig;
 const archivesAPI = {
   readSitePackagesSnapshot: () =>
     ipcRenderer.invoke("readSitePackagesSnapshot"),
-  readRequirements: () => ipcRenderer.invoke("readRequirements"),
+  readPrebuiltPackageNames: () =>
+    ipcRenderer.invoke("readPrebuiltPackageNames"),
   readStreamlitAppDirectory: () =>
     ipcRenderer.invoke("readStreamlitAppDirectory"),
 };

packages/desktop/src/App.tsx

@@ -25,10 +25,14 @@ function App() {
 
     Promise.all([
       window.archivesAPI.readSitePackagesSnapshot(),
-      window.archivesAPI.readRequirements(),
+      window.archivesAPI.readPrebuiltPackageNames(),
       window.archivesAPI.readStreamlitAppDirectory(),
     ]).then(
-      ([sitePackagesSnapshotFileBin, requirements, streamlitAppFiles]) => {
+      ([
+        sitePackagesSnapshotFileBin,
+        prebuiltPackageNames,
+        streamlitAppFiles,
+      ]) => {
         if (unmounted) {
           return;
         }
@@ -52,7 +56,8 @@ function App() {
             },
           },
           archives: [],
-          requirements,
+          requirements: [],
+          prebuiltPackageNames,
           mountedSitePackagesSnapshotFilePath,
           pyodideUrl,
           idbfsMountpoints: window.appConfig.idbfsMountpoints,

packages/kernel/src/kernel.ts

@@ -47,6 +47,13 @@ export interface StliteKernelOptions {
    */
   requirements: string[];
 
+  /**
+   * A list of prebuilt package names to be install at the booting-up phase via `pyodide.loadPackage()`.
+   * These packages basically can be installed via the `requirements` option,
+   * but some are only installable via this option such as `openssl`.
+   */
+  prebuiltPackageNames: string[];
+
   /**
    * Files to mount.
    */
@@ -195,6 +202,7 @@ export class StliteKernel {
       files: options.files,
       archives: options.archives,
       requirements: options.requirements,
+      prebuiltPackageNames: options.prebuiltPackageNames,
       pyodideUrl: options.pyodideUrl,
       wheels,
       mountedSitePackagesSnapshotFilePath:

packages/kernel/src/types.ts

@@ -44,6 +44,7 @@ export interface WorkerInitialData {
   files: Record<string, EmscriptenFile | EmscriptenFileUrl>;
   archives: Array<PyodideArchive | PyodideArchiveUrl>;
   requirements: string[];
+  prebuiltPackageNames: string[];
   pyodideUrl?: string;
   wheels?: {
     stliteServer: string;

packages/kernel/src/worker-runtime.ts

@@ -80,6 +80,7 @@ export function startWorkerEnv(
       files,
       archives,
       requirements,
+      prebuiltPackageNames: prebuiltPackages,
       wheels,
       mountedSitePackagesSnapshotFilePath,
       pyodideUrl = defaultPyodideUrl,
@@ -209,6 +210,11 @@ with tarfile.open("${mountedSitePackagesSnapshotFilePath}", "r") as tar_gz_file:
     // ===
     // Also, this must be after restoring the snapshot because the snapshot may contain the site-packages.
     postProgressMessage("Installing packages.");
+
+    console.debug("Installing the prebuilt packages:", prebuiltPackages);
+    await pyodide.loadPackage(prebuiltPackages);
+    console.debug("Installed the prebuilt packages");
+
     await pyodide.loadPackage("micropip");
     const micropip = pyodide.pyimport("micropip");
     if (wheels) {

packages/mountable/src/options.test.ts

@@ -63,11 +63,12 @@ describe("parseMountOptions()", () => {
         },
       },
       requirements: [],
+      prebuiltPackageNames: [],
       archives: [],
     });
   });
 
-  it("fills `command`, `entrypoint`, and `requirements` fields and converts the `files` into the canonical form", () => {
+  it("fills `entrypoint`, and `requirements` fields and converts the `files` into the canonical form", () => {
     const { kernelOptions } = parseMountOptions({
       files: {
         "streamlit_app.py": "foo",
@@ -82,6 +83,7 @@ describe("parseMountOptions()", () => {
     expect(kernelOptions).toEqual({
       entrypoint: "streamlit_app.py",
       requirements: [],
+      prebuiltPackageNames: [],
       files: {
         "streamlit_app.py": {
           data: "foo",
@@ -120,6 +122,7 @@ describe("parseMountOptions()", () => {
     expect(kernelOptions).toEqual({
       entrypoint: "streamlit_app.py",
       requirements: [],
+      prebuiltPackageNames: [],
       files: {},
       archives: [
         {
@@ -151,6 +154,7 @@ describe("parseMountOptions()", () => {
     expect(kernelOptions).toEqual({
       entrypoint: "foo.py",
       requirements: [],
+      prebuiltPackageNames: [],
       files: {
         "streamlit_app.py": {
           data: "foo",
@@ -166,6 +170,20 @@ describe("parseMountOptions()", () => {
     });
     expect(kernelOptions).toEqual({
       requirements: ["matplotlib"],
+      prebuiltPackageNames: [],
+      entrypoint: "streamlit_app.py",
+      files: {},
+      archives: [],
+    });
+  });
+
+  it("preserves the `prebuiltPackageNames` option if specified", () => {
+    const { kernelOptions } = parseMountOptions({
+      prebuiltPackageNames: ["openssl"],
+    });
+    expect(kernelOptions).toEqual({
+      requirements: [],
+      prebuiltPackageNames: ["openssl"],
       entrypoint: "streamlit_app.py",
       files: {},
       archives: [],