From c04de8ea47412afe7f123c097bfad2bf0de41f49 Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Tue, 19 Mar 2024 21:41:47 +0000 Subject: [PATCH] begin migration to kj/filesystem --- src/workerd/api/pyodide/pyodide.c++ | 77 ++++++++--------------------- src/workerd/api/pyodide/pyodide.h | 6 +-- src/workerd/server/workerd-api.c++ | 18 +++++++ src/workerd/server/workerd-api.h | 1 + 4 files changed, 42 insertions(+), 60 deletions(-) diff --git a/src/workerd/api/pyodide/pyodide.c++ b/src/workerd/api/pyodide/pyodide.c++ index 4373e76c71e2..6f8c27811cf1 100644 --- a/src/workerd/api/pyodide/pyodide.c++ +++ b/src/workerd/api/pyodide/pyodide.c++ @@ -1,10 +1,6 @@ #include "pyodide.h" -#include -#include -#include #include #include -#include namespace workerd::api::pyodide { @@ -114,67 +110,34 @@ jsg::Ref makePyodideMetadataReader(Worker::Reader conf) { false /* isTracing */); } -namespace { -// Checks that the given string contains no slashes, forward or backward -bool hasNoSlashes(const kj::String &str) { - for (auto c : str) { - if (c == '/' || c == '\\') { - return false; - } - } - return true; -} - -kj::String getDiskCachePath(kj::String &key) { - StringBuffer<64> fpath(64); - fpath.append("/home/garrett/.cache/workerd"); //TODO: do the thingy - fpath.append(key); - return fpath.toString(); -} -} //namespace - jsg::Optional> DiskCache::get(jsg::Lock& js, kj::String key) { - if(!hasNoSlashes(key)) { - KJ_LOG(ERROR, "DiskCache key contains slashes: ", key); - return kj::none; - } - - std::ifstream file; - kj::String fpath = getDiskCachePath(key); - - file.open(fpath.cStr(), std::ios::binary | std::ios::ate); - if (!file.is_open()) { - KJ_LOG(INFO, "DiskCache get: file not found: ", fpath); + KJ_IF_SOME(root, cacheRoot) { + kj::Path path(key); + auto file = root->tryOpenFile(path); + + KJ_IF_SOME(f, file) { + return f->readAllBytes(); + } else { + return kj::none; + } + } else { return kj::none; } - std::size_t size = file.tellg(); - file.seekg(0, std::ios::beg); - - auto data = kj::heapArray(size); - file.read(reinterpret_cast(data.begin()), size); - - return data; } void DiskCache::put(jsg::Lock& js, kj::String key, kj::Array data) { - if(!hasNoSlashes(key)) { - KJ_LOG(ERROR, "DiskCache key contains slashes: ", key); - return; - } - std::ofstream file; - kj::String fpath = getDiskCachePath(key); - file.open(fpath.cStr(), std::ios::binary | std::ios::trunc); - - if(!file.is_open()) { - KJ_LOG(ERROR, "DiskCache put: failed to open file: ", fpath); + KJ_IF_SOME(root, cacheRoot) { + kj::Path path(key); + auto file = root->tryOpenFile(path, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + KJ_IF_SOME(f, file) { + f->writeAll(data); + } else { + KJ_LOG(ERROR, "DiskCache: Failed to open file", key); + } + } else { return; } - - file.write(reinterpret_cast(data.begin()), data.size()); - - if(!file.good()) { - KJ_LOG(ERROR, "DiskCache put: failed to write to file: ", fpath); - } } } // namespace workerd::api::pyodide diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h index 26054accc45d..d05bb36f1aa8 100644 --- a/src/workerd/api/pyodide/pyodide.h +++ b/src/workerd/api/pyodide/pyodide.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -169,8 +170,9 @@ class DisabledInternalJaeger : public jsg::Object { // This cache is used by Pyodide to store wheels fetched over the internet across workerd restarts in local dev only class DiskCache: public jsg::Object { + kj::Maybe> cacheRoot; public: - DiskCache() = default; + DiskCache(kj::Maybe> cacheRoot): cacheRoot(kj::mv(cacheRoot)) {}; jsg::Optional> get(jsg::Lock& js, kj::String key); void put(jsg::Lock& js, kj::String key, kj::Array data); @@ -198,8 +200,6 @@ template void registerPyodideModules(Registry& registry, auto f registry.addBuiltinBundle(PYODIDE_BUNDLE, kj::none); registry.template addBuiltinModule( "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL); - registry.template addBuiltinModule( - "pyodide-internal:disk_cache", workerd::jsg::ModuleRegistry::Type::INTERNAL); //TODO: ideally this should only happen in local dev } } diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index e376e1c751c9..3a75b63ef8ea 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -453,6 +453,24 @@ void WorkerdApi::compileModules( }, jsg::ModuleRegistry::Type::INTERNAL); } + + // Inject disk cache module + KJ_IF_SOME(pcr, pyodideCacheRoot) { + using ModuleInfo = jsg::ModuleRegistry::ModuleInfo; + using ObjectModuleInfo = jsg::ModuleRegistry::ObjectModuleInfo; + using ResolveMethod = jsg::ModuleRegistry::ResolveMethod; + auto specifier = "pyodide-internal:disk_cache"; + auto diskCache = jsg::alloc(pcr->clone()); + modules->addBuiltinModule( + specifier, + [specifier = kj::str(specifier), diskCache = kj::mv(diskCache)]( + jsg::Lock& js, ResolveMethod, kj::Maybe&) mutable { + auto& wrapper = JsgWorkerdIsolate_TypeWrapper::from(js.v8Isolate); + auto wrap = wrapper.wrap(js.v8Context(), kj::none, kj::mv(diskCache)); + return kj::Maybe(ModuleInfo(js, specifier, kj::none, ObjectModuleInfo(js, wrap))); + }, + jsg::ModuleRegistry::Type::INTERNAL); + } } for (auto module: confModules) { diff --git a/src/workerd/server/workerd-api.h b/src/workerd/server/workerd-api.h index 70b1bd3fec29..342d2038bf0c 100644 --- a/src/workerd/server/workerd-api.h +++ b/src/workerd/server/workerd-api.h @@ -223,6 +223,7 @@ class WorkerdApi final: public Worker::Api { private: struct Impl; kj::Own impl; + kj::Maybe> pyodideCacheRoot; kj::Array compileScriptGlobals( jsg::Lock& lock,