Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Install the user-specified requirements before installing the Streamlit package so users can specify Streamlit's dependencies' versions #664

Merged
merged 4 commits into from
Dec 26, 2023
Merged

Install the user-specified requirements before installing the Streamlit package so users can specify Streamlit's dependencies' versions #664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c49532e
Install the user-specified requirements before installing the Streaml…
whitphx Dec 20, 2023
30d1ed3
Fix
whitphx Dec 21, 2023
9cfc0ff
Update the comment
whitphx Dec 21, 2023
ee529b8
Fix dump_artifacts.ts to install user-specified requirements before S…
whitphx Dec 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 37 additions & 16 deletions packages/desktop/bin/dump_artifacts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,11 @@ async function inspectUsedBuiltinPackages(
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");

micropip.add_mock_package("streamlit", "1.24.0");

await micropip.install(options.requirements);
await installStreamlitWheels(pyodide, {
useLocalKernelWheels: options.useLocalKernelWheels,
});

return Object.entries(pyodide.loadedPackages)
.filter(([, channel]) => channel === "default channel")
.map(([name]) => name);
Expand Down Expand Up @@ -120,21 +122,13 @@ async function installLocalWheel(pyodide: PyodideInterface, localPath: string) {
await micropip.install.callKwargs(requirement, { keep_going: true });
}

interface CreateSitePackagesSnapshotOptions {
interface InstallStreamlitWheelsOptions {
useLocalKernelWheels: boolean;
requirements: string[];
usedBuiltinPackages: string[];
saveTo: string;
}
async function createSitePackagesSnapshot(
options: CreateSitePackagesSnapshotOptions
async function installStreamlitWheels(
pyodide: PyodideInterface,
options: InstallStreamlitWheelsOptions
) {
console.info("Create the site-packages snapshot file...");

const pyodide = await loadPyodide();

await pyodide.loadPackage(["micropip"]);

if (options.useLocalKernelWheels) {
const stliteKernelDir = path.dirname(require.resolve("@stlite/kernel")); // -> /path/to/kernel/dist
const stliteKernelPyDir = path.resolve(stliteKernelDir, "../py"); // -> /path/to/kernel/py
Expand Down Expand Up @@ -172,11 +166,28 @@ async function createSitePackagesSnapshot(
console.log("Install", wheelUrls);
await micropip.install.callKwargs(wheelUrls, { keep_going: true });
}
}

interface CreateSitePackagesSnapshotOptions {
useLocalKernelWheels: boolean;
requirements: string[];
usedBuiltinPackages: string[];
saveTo: string;
}
async function createSitePackagesSnapshot(
options: CreateSitePackagesSnapshotOptions
) {
console.info("Create the site-packages snapshot file...");

const pyodide = await loadPyodide();

await pyodide.loadPackage(["micropip"]);

const micropip = pyodide.pyimport("micropip");

const pyodideBuiltinPackageMap = await loadPyodideBuiltinPackageData();

const mockedPackages: string[] = [];
if (options.usedBuiltinPackages.length > 0) {
console.log(
"Mocking builtin packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
Expand All @@ -189,6 +200,7 @@ async function createSitePackagesSnapshot(

console.log(`Mock ${packageInfo.name} ${packageInfo.version}`);
micropip.add_mock_package(packageInfo.name, packageInfo.version);
mockedPackages.push(packageInfo.name);
});
}

Expand All @@ -197,10 +209,17 @@ async function createSitePackagesSnapshot(
);

await micropip.install.callKwargs(options.requirements, { keep_going: true });
await installStreamlitWheels(pyodide, {
useLocalKernelWheels: options.useLocalKernelWheels,
});

console.log("Remove the mocked packages", mockedPackages);
mockedPackages.forEach((pkg) => micropip.remove_mock_package(pkg));

console.log("Archive the site-packages director(y|ies)");
const archiveFilePath = "/tmp/site-packages-snapshot.tar.gz";
await pyodide.runPythonAsync(`
import os
import tarfile
import site

Expand All @@ -209,6 +228,8 @@ async function createSitePackagesSnapshot(
tar_file_name = '${archiveFilePath}'
with tarfile.open(tar_file_name, mode='w:gz') as gzf:
for site_packages in site_packages_dirs:
print("Add site-package:", site_packages)
print(os.listdir(site_packages))
gzf.add(site_packages)
`);

Expand Down Expand Up @@ -400,12 +421,12 @@ yargs(hideBin(process.argv))
saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
});
// The `requirements.txt` file will be needed to call `micropip.install()` at runtime.
// The built-in packages will be vendored in the build artifact as wheel files
// The Pyodide-built packages will be vendored in the build artifact as wheel files
// and `micropip.install()` will install them at runtime,
// while the packages downloaded from PyPI will have been included in the site-packages snapshot.
await writeRequirements(
path.resolve(destDir, "./requirements.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts.
requirements
usedBuiltinPackages
);
await copyStreamlitAppDirectory({
sourceDir: args.appHomeDirSource,
Expand Down
34 changes: 21 additions & 13 deletions packages/kernel/src/worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

let pyodide: Pyodide.PyodideInterface;

let httpServer: any;

Check warning on line 15 in packages/kernel/src/worker.ts

View workflow job for this annotation

GitHub Actions / test-kernel 

Unexpected any. Specify a different type

interface StliteWorkerContext extends DedicatedWorkerGlobalScope {
postMessage(message: OutMessage, transfer: Transferable[]): void;
Expand Down Expand Up @@ -50,7 +50,7 @@
loadPyodide = pyodideModule.loadPyodide;
} else {
importScripts(pyodideUrl);
loadPyodide = (self as any).loadPyodide;

Check warning on line 53 in packages/kernel/src/worker.ts

View workflow job for this annotation

GitHub Actions / test-kernel 

Unexpected any. Specify a different type
}
return loadPyodide({ ...loadPyodideOptions, indexURL: indexUrl });
}
Expand Down Expand Up @@ -128,6 +128,10 @@
})
);

if (!mountedSitePackagesSnapshotFilePath && !wheels) {
throw new Error(`Neither snapshot nor wheel files are provided.`);
}

if (mountedSitePackagesSnapshotFilePath) {
// Restore the site-packages director(y|ies) from the mounted snapshot file.
postProgressMessage("Restoring the snapshot.");
Expand All @@ -152,7 +156,23 @@
console.debug("Mock pyarrow");
mockPyArrow(pyodide);
console.debug("Mocked pyarrow");
} else if (wheels) {
}

// NOTE: It's important to install the requirements before loading the streamlit package
// because it allows users to specify the versions of Streamlit's dependencies via requirements.txt
// before these versions are automatically resolved by micropip when installing Streamlit.
// Also, this must be after restoring the snapshot because the snapshot may contain the site-packages.
if (requirements.length > 0) {
postProgressMessage("Installing the requirements.");
console.debug("Installing the requirements:", requirements);
verifyRequirements(requirements); // Blocks the not allowed wheel URL schemes.
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");
await micropip.install.callKwargs(requirements, { keep_going: true });
console.debug("Installed the requirements:", requirements);
}

if (wheels) {
postProgressMessage("Installing streamlit and its dependencies.");
console.debug("Loading stlite-server, and streamlit");
await pyodide.loadPackage("micropip");
Expand All @@ -168,18 +188,6 @@
console.debug("Mock pyarrow");
mockPyArrow(pyodide);
console.debug("Mocked pyarrow");
} else {
throw new Error(`Neither snapshot nor wheel files are provided.`);
}

if (requirements.length > 0) {
postProgressMessage("Installing the requirements.");
console.debug("Installing the requirements:", requirements);
verifyRequirements(requirements); // Blocks the not allowed wheel URL schemes.
await pyodide.loadPackage("micropip");
const micropip = pyodide.pyimport("micropip");
await micropip.install.callKwargs(requirements, { keep_going: true });
console.debug("Installed the requirements:", requirements);
}

// The following code is necessary to avoid errors like `NameError: name '_imp' is not defined`
Expand Down Expand Up @@ -342,7 +350,7 @@

httpServer.start_websocket(
path,
(messageProxy: any, binary: boolean) => {

Check warning on line 353 in packages/kernel/src/worker.ts

View workflow job for this annotation

GitHub Actions / test-kernel 

Unexpected any. Specify a different type
// XXX: Now there is no session mechanism

if (binary) {
Expand Down Expand Up @@ -388,7 +396,7 @@

const { request } = msg.data;

const onResponse = (statusCode: number, _headers: any, _body: any) => {

Check warning on line 399 in packages/kernel/src/worker.ts

View workflow job for this annotation

GitHub Actions / test-kernel 

Unexpected any. Specify a different type
const headers = _headers.toJs();
const body = _body.toJs();
console.debug({ statusCode, headers, body });
Expand Down
Loading