Add Python development compatibility flag for tests

cloudflare · Aug 21, 2024 · 905cc6f · 905cc6f
1 parent 72d13e7
commit 905cc6f
Show file tree

Hide file tree

Showing 13 changed files with 46 additions and 39 deletions.
diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++
@@ -425,8 +425,7 @@ void Socket::handleReadableEof(jsg::Lock& js, jsg::Promise<void> onEof) {
   KJ_ASSERT(!getAllowHalfOpen(options));
   // Listen for EOF on the ReadableStream.
   onEof
-      .then(
-          js,
+      .then(js,
           JSG_VISITABLE_LAMBDA(
               (ref = JSG_THIS), (ref), (jsg::Lock& js) { return ref->maybeCloseWriteSide(js); }))
       .markAsHandled(js);

diff --git a/src/workerd/io/compatibility-date.c++ b/src/workerd/io/compatibility-date.c++
@@ -355,4 +355,12 @@ kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
   return result;
 }
 
+kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease) {
+  if (pyodideRelease.getPyodide() == "dev") {
+    return kj::str("dev");
+  }
+  return kj::str(pyodideRelease.getPyodide(), "_", pyodideRelease.getPyodideRevision(), "_",
+      pyodideRelease.getBackport());
+}
+
 }  // namespace workerd
diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp
@@ -424,7 +424,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   pythonWorkers @43 :Bool
       $compatEnableFlag("python_workers")
       $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
-          packages = "2024-03-01", backport = 0);
+          packages = "2024-03-01", backport = 0)
+      $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
   # Enables Python Workers. Access to this flag is not restricted, instead bundles containing
   # Python modules are restricted in EWC.
   #
@@ -572,4 +573,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   # would not be cleared until the next time the queue was processed. This behavior leads
   # to a situtation where the stream can hang if the consumer stops consuming. When set,
   # this flag changes the behavior to clear the queue immediately upon abort.
+
+  pythonWorkersDevPyodide @58 :Bool
+    $compatEnableFlag("python_workers_development")
+    $pythonSnapshotRelease(pyodide = "dev", pyodideRevision = "dev",
+          packages = "2024-03-01", backport = 0)
+    $experimental;
+  # Enables Python Workers and uses the bundle from the Pyodide source directory directly. For testing only.
 }
diff --git a/src/workerd/io/compatibility-date.h b/src/workerd/io/compatibility-date.h
@@ -53,6 +53,7 @@ kj::String currentDateStr();
 
 kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
     CompatibilityFlags::Reader featureFlags);
+kj::String getPythonBundleName(PythonSnapshotRelease::Reader pyodideRelease);
 
 // These values come from src/workerd/io/compatibility-date.capnp
 static constexpr uint64_t COMPAT_ENABLE_FLAG_ANNOTATION_ID = 0xb6dabbc87cd1b03eull;

diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel
@@ -4,14 +4,14 @@ load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 
-# pyodide-dev.capnp.bin represents a custom pyodide version "dev" that is generated
+# pyodide_dev.capnp.bin represents a custom pyodide version "dev" that is generated
 # at build time using the latest contents of the src/pyodide directory.
 # This is used to run tests to ensure that they are always run against the latest build of
 # the Pyodide bundle.
 copy_file(
-    name = "pyodide-dev.capnp.bin@rule",
+    name = "pyodide_dev.capnp.bin@rule",
     src = "//src/pyodide:pyodide.capnp.bin",
-    out = "pyodide-bundle-cache/pyodide-dev.capnp.bin"
+    out = "pyodide-bundle-cache/pyodide_dev.capnp.bin"
 )
 
 py_wd_test(

diff --git a/src/workerd/server/tests/python/env-param/env.wd-test b/src/workerd/server/tests/python/env-param/env.wd-test
@@ -14,7 +14,7 @@ const unitTests :Workerd.Config = (
           ),
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

diff --git a/src/workerd/server/tests/python/hello/hello.wd-test b/src/workerd/server/tests/python/hello/hello.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl
@@ -23,7 +23,7 @@ const unitTests :Workerd.Config = (
           (name = "{}", pythonRequirement = ""),
         ],
         compatibilityDate = "2024-05-02",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl
@@ -7,8 +7,8 @@ def py_wd_test(
   args = [],
   **kwargs
 ):
-    data += ["pyodide-dev.capnp.bin@rule"]
-    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."]
+    data += ["pyodide_dev.capnp.bin@rule"]
+    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide_dev.capnp.bin@rule)/.."]
 
     wd_test(
         src = src,

diff --git a/src/workerd/server/tests/python/random/random.wd-test b/src/workerd/server/tests/python/random/random.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

diff --git a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test
@@ -10,7 +10,7 @@ const unitTests :Workerd.Config = (
           (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"),
         ],
         compatibilityDate = "2023-12-18",
-        compatibilityFlags = ["python_workers"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],

diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++
@@ -411,7 +411,7 @@ kj::Maybe<jsg::ModuleRegistry::ModuleInfo> WorkerdApi::tryCompileModule(jsg::Loc
 
 namespace {
 kj::Path getPyodideBundleFileName(kj::StringPtr version) {
-  return kj::Path(kj::str("pyodide-", version, ".capnp.bin"));
+  return kj::Path(kj::str("pyodide_", version, ".capnp.bin"));
 }
 
 kj::Maybe<kj::Own<const kj::ReadableFile>> getPyodideBundleFile(
@@ -476,7 +476,7 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
       kj::HttpHeaders headers(table);
 
       kj::String url =
-          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide-",
+          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
               version, ".capnp.bin");
 
       auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);
@@ -511,19 +511,11 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam,
           "The python_workers compatibility flag is required to use Python.");
       // Inject Pyodide bundle
       if (util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) {
-        KJ_IF_SOME(bundle, fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) {
-          modules->addBuiltinBundle(bundle, kj::none);
-        } else {
-          auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
-          auto pyodide = pythonRelease.getPyodide();
-          auto pyodideRevision = pythonRelease.getPyodideRevision();
-          auto backport = pythonRelease.getBackport();
-
-          auto version = kj::str(pyodide, "_", pyodideRevision, "_", backport);
-          auto bundle = KJ_ASSERT_NONNULL(
-              fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
-          modules->addBuiltinBundle(bundle, kj::none);
-        }
+        auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
+        auto version = getPythonBundleName(pythonRelease);
+        auto bundle = KJ_ASSERT_NONNULL(
+            fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
+        modules->addBuiltinBundle(bundle, kj::none);
       }
       // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
       {

diff --git a/src/workerd/tests/bench-tools.h b/src/workerd/tests/bench-tools.h
@@ -13,15 +13,14 @@
 // enough to not need ns precision.
 #define WD_BENCHMARK(X) BENCHMARK(X)->Unit(benchmark::kMicrosecond)
 
-/* Macro inspired by KJ_TEST() to enable benchmarking without requiring the benchmark::State */                                            \
-/* argument and make it easy to convert tests to benchmarks. */ /* Make sure the linker fails if tests are not in anonymous namespaces. */ \
-#define WD_BENCH(description)                                                                                                              \
-  extern int KJ_CONCAT(YouMustWrapTestsInAnonymousNamespace, __COUNTER__) KJ_UNUSED;                                                       \
-  void KJ_UNIQUE_NAME(Bench)();                                                                                                            \
-  void KJ_UNIQUE_NAME(BenchImpl)(benchmark::State & state) {                                                                               \
-    for (auto _: state) {                                                                                                                  \
-      KJ_UNIQUE_NAME(Bench)();                                                                                                             \
-    }                                                                                                                                      \
-  }                                                                                                                                        \
-  WD_BENCHMARK(KJ_UNIQUE_NAME(BenchImpl))->Name(description);                                                                              \
+/* Macro inspired by KJ_TEST() to enable benchmarking without requiring the benchmark::State */ /* argument and make it easy to convert tests to benchmarks. */ /* Make sure the linker fails if tests are not in anonymous namespaces. */ \
+#define WD_BENCH(description)                                                                                                                                                                                                              \
+  extern int KJ_CONCAT(YouMustWrapTestsInAnonymousNamespace, __COUNTER__) KJ_UNUSED;                                                                                                                                                       \
+  void KJ_UNIQUE_NAME(Bench)();                                                                                                                                                                                                            \
+  void KJ_UNIQUE_NAME(BenchImpl)(benchmark::State & state) {                                                                                                                                                                               \
+    for (auto _: state) {                                                                                                                                                                                                                  \
+      KJ_UNIQUE_NAME(Bench)();                                                                                                                                                                                                             \
+    }                                                                                                                                                                                                                                      \
+  }                                                                                                                                                                                                                                        \
+  WD_BENCHMARK(KJ_UNIQUE_NAME(BenchImpl))->Name(description);                                                                                                                                                                              \
   void KJ_UNIQUE_NAME(Bench)()