From 525eb0277f1527fe5c0d0da130926654b3c5a023 Mon Sep 17 00:00:00 2001 From: "malonso@cloudflare.com" Date: Wed, 16 Dec 2020 16:47:42 -0600 Subject: [PATCH] support uploading module-based scripts --- src/preview/mod.rs | 7 +- src/settings/toml/builder.rs | 2 + src/upload/form/mod.rs | 122 +++++++++++++++++++++++++----- src/upload/form/modules_worker.rs | 75 ++++++++++++++++++ src/upload/form/project_assets.rs | 98 +++++++++++++++++++++++- src/upload/form/text_blob.rs | 3 + src/upload/form/wasm_module.rs | 7 +- src/upload/package.rs | 22 +++++- 8 files changed, 308 insertions(+), 28 deletions(-) create mode 100644 src/upload/form/modules_worker.rs diff --git a/src/preview/mod.rs b/src/preview/mod.rs index 6e93c1f73..b3111f6ea 100644 --- a/src/preview/mod.rs +++ b/src/preview/mod.rs @@ -17,13 +17,13 @@ use log::info; use url::Url; use ws::{Sender, WebSocket}; -use crate::build::build_target; use crate::http; use crate::settings::global_user::GlobalUser; use crate::settings::toml::Target; use crate::terminal::message::{Message, StdOut}; use crate::terminal::open_browser; use crate::watch::watch_and_build; +use crate::{build::build_target, settings::toml::ScriptFormat}; pub fn preview( mut target: Target, @@ -31,6 +31,11 @@ pub fn preview( options: PreviewOpt, verbose: bool, ) -> Result<(), failure::Error> { + if let Some(build) = &target.build { + if matches!(build.upload_format, ScriptFormat::Modules) { + failure::bail!("wrangler preview does not support previewing modules scripts. Please use wrangler dev instead."); + } + } build_target(&target)?; let sites_preview: bool = target.site.is_some(); diff --git a/src/settings/toml/builder.rs b/src/settings/toml/builder.rs index 95c7b4bef..2094e0b10 100644 --- a/src/settings/toml/builder.rs +++ b/src/settings/toml/builder.rs @@ -18,6 +18,8 @@ pub struct Builder { #[serde(default = "upload_dir")] pub upload_dir: PathBuf, pub upload_format: ScriptFormat, + pub upload_include: Option>, + pub upload_exclude: Option>, #[serde(default = "watch_dir")] pub watch_dir: PathBuf, } diff --git a/src/upload/form/mod.rs b/src/upload/form/mod.rs index a417f8bfd..b2b7ec077 100644 --- a/src/upload/form/mod.rs +++ b/src/upload/form/mod.rs @@ -1,3 +1,4 @@ +mod modules_worker; mod plain_text; mod project_assets; mod service_worker; @@ -9,19 +10,24 @@ use std::fs; use std::path::Path; use std::path::PathBuf; +use ignore::overrides::{Override, OverrideBuilder}; +use ignore::WalkBuilder; + use crate::settings::binding; -use crate::settings::toml::{Target, TargetType}; +use crate::settings::toml::{Builder, ScriptFormat, Target, TargetType}; use crate::sites::AssetManifest; use crate::wranglerjs; use plain_text::PlainText; -use project_assets::ServiceWorkerAssets; +use project_assets::{ModulesAssets, ServiceWorkerAssets}; use text_blob::TextBlob; use wasm_module::WasmModule; // TODO: https://github.com/cloudflare/wrangler/issues/1083 use super::{krate, Package}; +use self::project_assets::Module; + pub fn build( target: &Target, asset_manifest: Option, @@ -70,23 +76,76 @@ pub fn build( service_worker::build_form(&assets, session_config) } - TargetType::JavaScript => { - log::info!("JavaScript project detected. Publishing..."); - let package_dir = target.package_dir()?; - let package = Package::new(&package_dir)?; - - let script_path = package.main(&package_dir)?; - - let assets = ServiceWorkerAssets::new( - script_path, - wasm_modules, - kv_namespaces.to_vec(), - text_blobs, - plain_texts, - )?; - - service_worker::build_form(&assets, session_config) - } + TargetType::JavaScript => match &target.build { + Some(config) => match &config.upload_format { + ScriptFormat::ServiceWorker => { + log::info!("Plain JavaScript project detected. Publishing..."); + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + let script_path = package.main(&package_dir)?; + + let assets = ServiceWorkerAssets::new( + script_path, + wasm_modules, + kv_namespaces.to_vec(), + text_blobs, + plain_texts, + )?; + + service_worker::build_form(&assets, session_config) + } + ScriptFormat::Modules => { + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + let main_module = package.module(&package_dir)?; + let main_module_name = filename_from_path(&main_module) + .ok_or_else(|| failure::err_msg("filename required for main module"))?; + + let ignore = build_ignore(config, &package_dir)?; + let modules_iter = WalkBuilder::new(config.upload_dir.clone()) + .standard_filters(false) + .hidden(true) + .overrides(ignore) + .build(); + + let mut modules: Vec = vec![]; + + for entry in modules_iter { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + log::info!("Adding module {}", path.display()); + modules.push(Module::new(path.to_owned())?); + } + } + + let assets = ModulesAssets::new( + main_module_name, + modules, + kv_namespaces.to_vec(), + plain_texts, + )?; + + modules_worker::build_form(&assets, session_config) + } + }, + None => { + log::info!("Plain JavaScript project detected. Publishing..."); + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + let script_path = package.main(&package_dir)?; + + let assets = ServiceWorkerAssets::new( + script_path, + wasm_modules, + kv_namespaces.to_vec(), + text_blobs, + plain_texts, + )?; + + service_worker::build_form(&assets, session_config) + } + }, TargetType::Webpack => { log::info!("webpack project detected. Publishing..."); // TODO: https://github.com/cloudflare/wrangler/issues/850 @@ -128,10 +187,15 @@ fn get_asset_manifest_blob(asset_manifest: AssetManifest) -> Result Option { +fn filestem_from_path(path: &PathBuf) -> Option { path.file_stem()?.to_str().map(|s| s.to_string()) } +fn filename_from_path(path: &PathBuf) -> Option { + path.file_name() + .map(|filename| filename.to_string_lossy().into_owned()) +} + fn build_generated_dir() -> Result<(), failure::Error> { let dir = "./worker/generated"; if !Path::new(dir).is_dir() { @@ -151,3 +215,21 @@ fn concat_js(name: &str) -> Result<(), failure::Error> { fs::write("./worker/generated/script.js", js.as_bytes())?; Ok(()) } + +fn build_ignore(config: &Builder, directory: &Path) -> Result { + let mut overrides = OverrideBuilder::new(directory); + // If `include` present, use it and don't touch the `exclude` field + if let Some(included) = &config.upload_include { + for i in included { + overrides.add(&i)?; + log::info!("Including {}", i); + } + } else if let Some(excluded) = &config.upload_exclude { + for e in excluded { + overrides.add(&format!("!{}", e))?; + log::info!("Ignoring {}", e); + } + } + + Ok(overrides.build()?) +} diff --git a/src/upload/form/modules_worker.rs b/src/upload/form/modules_worker.rs new file mode 100644 index 000000000..cc4989619 --- /dev/null +++ b/src/upload/form/modules_worker.rs @@ -0,0 +1,75 @@ +use std::fs::File; + +use reqwest::blocking::multipart::{Form, Part}; +use serde::Serialize; + +use crate::settings::binding::Binding; + +use super::ModulesAssets; + +#[derive(Serialize, Debug)] +struct Metadata { + pub main_module: String, + pub bindings: Vec, +} + +pub fn build_form( + assets: &ModulesAssets, + session_config: Option, +) -> Result { + let mut form = Form::new(); + + // The preview service in particular streams the request form, and requires that the + // "metadata" part be set first, so this order is important. + form = add_metadata(form, assets)?; + form = add_files(form, assets)?; + if let Some(session_config) = session_config { + form = add_session_config(form, session_config)? + } + + log::info!("building form"); + log::info!("{:#?}", &form); + + Ok(form) +} + +fn add_files(mut form: Form, assets: &ModulesAssets) -> Result { + for module in &assets.modules { + let file_name = module + .filename() + .ok_or_else(|| failure::err_msg("a filename is required for each module"))?; + let part = Part::reader(File::open(module.path.clone())?) + .mime_str(module.module_type.content_type())? + .file_name(file_name.clone()); + form = form.part(file_name.clone(), part); + } + Ok(form) +} + +fn add_metadata(mut form: Form, assets: &ModulesAssets) -> Result { + let metadata_json = serde_json::json!(&Metadata { + main_module: assets.main_module.clone(), + bindings: assets.bindings(), + }); + + let metadata = Part::text(metadata_json.to_string()) + .file_name("metadata.json") + .mime_str("application/json")?; + + form = form.part("metadata", metadata); + + Ok(form) +} + +fn add_session_config( + mut form: Form, + session_config: serde_json::Value, +) -> Result { + let wrangler_session_config = Part::text(session_config.to_string()) + .file_name("") + .mime_str("application/json")?; + + form = form.part("wrangler-session-config", wrangler_session_config); + + Ok(form) +} diff --git a/src/upload/form/project_assets.rs b/src/upload/form/project_assets.rs index 9ffdab502..a833dc69a 100644 --- a/src/upload/form/project_assets.rs +++ b/src/upload/form/project_assets.rs @@ -3,10 +3,10 @@ use std::path::PathBuf; use failure::format_err; use super::binding::Binding; -use super::filename_from_path; use super::plain_text::PlainText; use super::text_blob::TextBlob; use super::wasm_module::WasmModule; +use super::{filename_from_path, filestem_from_path}; use crate::settings::toml::KvNamespace; @@ -28,7 +28,7 @@ impl ServiceWorkerAssets { text_blobs: Vec, plain_texts: Vec, ) -> Result<Self, failure::Error> { - let script_name = filename_from_path(&script_path).ok_or_else(|| { + let script_name = filestem_from_path(&script_path).ok_or_else(|| { format_err!("filename should not be empty: {}", script_path.display()) })?; @@ -73,3 +73,97 @@ impl ServiceWorkerAssets { self.script_path.clone() } } + +pub struct Module { + pub path: PathBuf, + pub module_type: ModuleType, +} + +impl Module { + pub fn new(path: PathBuf) -> Result<Module, failure::Error> { + let extension = path + .extension() + .ok_or_else(|| { + failure::err_msg(format!( + "File {} lacks an extension. An extension is required to determine module type", + path.display() + )) + })? + .to_string_lossy(); + + let module_type = match extension.as_ref() { + "mjs" => ModuleType::ES6, + "js" => ModuleType::CommonJS, + "wasm" => ModuleType::Wasm, + "txt" => ModuleType::Text, + _ => ModuleType::Data, + }; + + Ok(Module { path, module_type }) + } + + pub fn filename(&self) -> Option<String> { + filename_from_path(&self.path) + } +} + +pub enum ModuleType { + ES6, + CommonJS, + Wasm, + Text, + Data, +} + +impl ModuleType { + pub fn content_type(&self) -> &str { + match &self { + Self::ES6 => "application/javascript+module", + Self::CommonJS => "application/javascript", + Self::Wasm => "application/wasm", + Self::Text => "text/plain", + Self::Data => "application/octet-stream", + } + } +} + +pub struct ModulesAssets { + pub main_module: String, + pub modules: Vec<Module>, + pub kv_namespaces: Vec<KvNamespace>, + pub plain_texts: Vec<PlainText>, +} + +impl ModulesAssets { + pub fn new( + main_module: String, + modules: Vec<Module>, + kv_namespaces: Vec<KvNamespace>, + plain_texts: Vec<PlainText>, + ) -> Result<Self, failure::Error> { + Ok(Self { + main_module, + modules, + kv_namespaces, + plain_texts, + }) + } + + pub fn bindings(&self) -> Vec<Binding> { + let mut bindings = Vec::new(); + + // Bindings that refer to a `part` of the uploaded files + // in the service-worker format, are now modules. + + for kv in &self.kv_namespaces { + let binding = kv.binding(); + bindings.push(binding); + } + for plain_text in &self.plain_texts { + let binding = plain_text.binding(); + bindings.push(binding); + } + + bindings + } +} diff --git a/src/upload/form/text_blob.rs b/src/upload/form/text_blob.rs index e6c7e4694..921c8a820 100644 --- a/src/upload/form/text_blob.rs +++ b/src/upload/form/text_blob.rs @@ -1,6 +1,9 @@ use super::binding::Binding; use serde::{Deserialize, Serialize}; +// Note: This is only used for service-worker scripts. +// modules scripts use the universal Module class instead of this. + #[derive(Debug, Deserialize, Serialize)] pub struct TextBlob { pub data: String, diff --git a/src/upload/form/wasm_module.rs b/src/upload/form/wasm_module.rs index bb1406ddf..e5b93b95a 100644 --- a/src/upload/form/wasm_module.rs +++ b/src/upload/form/wasm_module.rs @@ -3,7 +3,10 @@ use std::path::PathBuf; use failure::format_err; use super::binding::Binding; -use super::filename_from_path; +use super::filestem_from_path; + +// Note: This is only used for service-worker scripts. +// modules scripts use the universal Module class instead of this. #[derive(Debug)] pub struct WasmModule { @@ -14,7 +17,7 @@ pub struct WasmModule { impl WasmModule { pub fn new(path: PathBuf, binding: String) -> Result<Self, failure::Error> { - let filename = filename_from_path(&path) + let filename = filestem_from_path(&path) .ok_or_else(|| format_err!("filename should not be empty: {}", path.display()))?; Ok(Self { diff --git a/src/upload/package.rs b/src/upload/package.rs index 33b293a45..f48ca79cb 100644 --- a/src/upload/package.rs +++ b/src/upload/package.rs @@ -7,6 +7,8 @@ use serde::{self, Deserialize}; pub struct Package { #[serde(default)] main: PathBuf, + #[serde(default)] + module: PathBuf, } impl Package { pub fn main(&self, package_dir: &PathBuf) -> Result<PathBuf, failure::Error> { @@ -23,16 +25,30 @@ impl Package { Ok(self.main.clone()) } } + pub fn module(&self, package_dir: &PathBuf) -> Result<PathBuf, failure::Error> { + if self.module == PathBuf::from("") { + failure::bail!( + "The `module` key in your `package.json` file is required when using the module script format; please specify the entry point of your Worker.", + ) + } else if !package_dir.join(&self.module).exists() { + failure::bail!( + "The entrypoint of your Worker ({}) could not be found.", + self.module.display() + ) + } else { + Ok(self.module.clone()) + } + } } impl Package { - pub fn new(pkg_path: &PathBuf) -> Result<Package, failure::Error> { - let manifest_path = pkg_path.join("package.json"); + pub fn new(package_dir: &PathBuf) -> Result<Package, failure::Error> { + let manifest_path = package_dir.join("package.json"); if !manifest_path.is_file() { failure::bail!( "Your JavaScript project is missing a `package.json` file; is `{}` the \ wrong directory?", - pkg_path.display() + package_dir.display() ) }