From 1cb7b0b19f4c3704f2f7e8e0742a3008401a219e Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 9 Feb 2024 21:58:41 -0800 Subject: [PATCH] Python: add support for subdirectory structure (#1635) --- src/pyodide/internal/metadatafs.js | 43 ++++++++++++++----- src/workerd/server/tests/python/BUILD.bazel | 34 ++++++++++----- .../server/tests/python/subdirectory/a.py | 10 +++++ .../python/subdirectory/subdir/__init__.py | 0 .../tests/python/subdirectory/subdir/a.py | 1 + .../python/subdirectory/subdirectory.wd-test | 18 ++++++++ 6 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 src/workerd/server/tests/python/subdirectory/a.py create mode 100644 src/workerd/server/tests/python/subdirectory/subdir/__init__.py create mode 100644 src/workerd/server/tests/python/subdirectory/subdir/a.py create mode 100644 src/workerd/server/tests/python/subdirectory/subdirectory.wd-test diff --git a/src/pyodide/internal/metadatafs.js b/src/pyodide/internal/metadatafs.js index cd0e5b2433b..812883d6648 100644 --- a/src/pyodide/internal/metadatafs.js +++ b/src/pyodide/internal/metadatafs.js @@ -1,32 +1,55 @@ import { default as MetadataReader } from "pyodide-internal:runtime-generated/metadata"; import { createReadonlyFS } from "pyodide-internal:readOnlyFS"; +function createTree(paths) { + const tree = new Map(); + paths.forEach((elt, idx) => { + let subTree = tree; + const parts = elt.split("/"); + const name = parts.pop(); + for (const part of parts) { + let next = subTree.get(part); + if (!next) { + next = new Map(); + subTree.set(part, next); + } + subTree = next; + } + subTree.set(name, idx); + }); + return tree; +} + export function createMetadataFS(Module) { const TIMESTAMP = Date.now(); const names = MetadataReader.getNames(); const sizes = MetadataReader.getSizes(); - const nameToIndex = new Map(names.map((val, idx) => [val, idx])); - + const rootInfo = createTree(names); const FSOps = { - getNodeMode(parent, name, index) { + getNodeMode(parent, name, info) { return { permissions: 0o555, // read and execute but not write - isDir: index === undefined, + isDir: typeof info !== "number", }; }, - setNodeAttributes(node, index, isDir) { + setNodeAttributes(node, info, isDir) { node.modtime = TIMESTAMP; node.usedBytes = 0; - if (!isDir) { - node.index = index; - node.usedBytes = sizes[index]; + if (info === undefined) { + info = rootInfo; + } + if (isDir) { + node.tree = info; + } else { + node.index = info; + node.usedBytes = sizes[info]; } }, readdir(node) { - return names; + return Array.from(node.tree.keys()); }, lookup(parent, name) { - return nameToIndex.get(name); + return parent.tree.get(name); }, read(stream, position, buffer) { return MetadataReader.read(stream.node.index, position, buffer); diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index d8fc6d9732f..7c178564672 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -22,21 +22,35 @@ wd_test( ), ) +# langchain test: disabled for now because it's flaky +# TODO: reenable this? +# +# wd_test( +# src = "langchain/langchain.wd-test", +# args = ["--experimental"], +# data = glob( +# [ +# "langchain/*", +# ], +# exclude = ["**/*.wd-test"], +# ), +# # TODO: fails in micropip on windows with: +# # +# # ValueError: Can't fetch metadata for 'openai'. Please make sure you have entered a correct +# # package name and correctly specified index_urls (if you changed them). +# target_compatible_with = select({ +# "@platforms//os:windows": ["@platforms//:incompatible"], +# "//conditions:default": [], +# }), +# ) + wd_test( - src = "langchain/langchain.wd-test", + src = "subdirectory/subdirectory.wd-test", args = ["--experimental"], data = glob( [ - "langchain/*", + "subdirectory/**", ], exclude = ["**/*.wd-test"], ), - # TODO: fails in micropip on windows with: - # - # ValueError: Can't fetch metadata for 'openai'. Please make sure you have entered a correct - # package name and correctly specified index_urls (if you changed them). - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), ) diff --git a/src/workerd/server/tests/python/subdirectory/a.py b/src/workerd/server/tests/python/subdirectory/a.py new file mode 100644 index 00000000000..553291704c9 --- /dev/null +++ b/src/workerd/server/tests/python/subdirectory/a.py @@ -0,0 +1,10 @@ +from js import Response +from subdir.a import x + + +def fetch(request): + return Response.new("hello world") + + +def test(): + print("Hi there, this is a test", x) diff --git a/src/workerd/server/tests/python/subdirectory/subdir/__init__.py b/src/workerd/server/tests/python/subdirectory/subdir/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/workerd/server/tests/python/subdirectory/subdir/a.py b/src/workerd/server/tests/python/subdirectory/subdir/a.py new file mode 100644 index 00000000000..0553d3a2eb3 --- /dev/null +++ b/src/workerd/server/tests/python/subdirectory/subdir/a.py @@ -0,0 +1 @@ +x = 7 diff --git a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test new file mode 100644 index 00000000000..3f1b00393ef --- /dev/null +++ b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test @@ -0,0 +1,18 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "main", + worker = ( + modules = [ + (name = "a.py", pythonModule = embed "./a.py"), + (name = "subdir/__init__.py", pythonModule = embed "./subdir/__init__.py"), + (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"), + ], + compatibilityDate = "2023-12-18", + compatibilityFlags = ["experimental"], + ) + ), + ], +); +