support uploading module-based scripts

src/preview/mod.rs

@@ -17,20 +17,25 @@ use log::info;
 use url::Url;
 use ws::{Sender, WebSocket};
 
-use crate::build::build_target;
 use crate::http;
 use crate::settings::global_user::GlobalUser;
 use crate::settings::toml::Target;
 use crate::terminal::message::{Message, StdOut};
 use crate::terminal::open_browser;
 use crate::watch::watch_and_build;
+use crate::{build::build_target, settings::toml::ScriptFormat};
 
 pub fn preview(
     mut target: Target,
     user: Option<GlobalUser>,
     options: PreviewOpt,
     verbose: bool,
 ) -> Result<(), failure::Error> {
+    if let Some(build) = &target.build {
+        if matches!(build.upload_format, ScriptFormat::Modules) {
+            failure::bail!("wrangler preview does not support previewing modules scripts. Please use wrangler dev instead.");
+        }
+    }
     build_target(&target)?;
 
     let sites_preview: bool = target.site.is_some();

src/settings/toml/builder.rs

@@ -18,6 +18,8 @@ pub struct Builder {
     #[serde(default = "upload_dir")]
     pub upload_dir: PathBuf,
     pub upload_format: ScriptFormat,
+    pub upload_include: Option<Vec<String>>,
+    pub upload_exclude: Option<Vec<String>>,
     #[serde(default = "watch_dir")]
     pub watch_dir: PathBuf,
 }

src/upload/form/mod.rs

@@ -1,3 +1,4 @@
+mod modules_worker;
 mod plain_text;
 mod project_assets;
 mod service_worker;
@@ -9,19 +10,24 @@ use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
 
+use ignore::overrides::{Override, OverrideBuilder};
+use ignore::WalkBuilder;
+
 use crate::settings::binding;
-use crate::settings::toml::{Target, TargetType};
+use crate::settings::toml::{Builder, ScriptFormat, Target, TargetType};
 use crate::sites::AssetManifest;
 use crate::wranglerjs;
 
 use plain_text::PlainText;
-use project_assets::ServiceWorkerAssets;
+use project_assets::{ModulesAssets, ServiceWorkerAssets};
 use text_blob::TextBlob;
 use wasm_module::WasmModule;
 
 // TODO: https://github.com/cloudflare/wrangler/issues/1083
 use super::{krate, Package};
 
+use self::project_assets::Module;
+
 pub fn build(
     target: &Target,
     asset_manifest: Option<AssetManifest>,
@@ -70,23 +76,76 @@ pub fn build(
 
             service_worker::build_form(&assets, session_config)
         }
-        TargetType::JavaScript => {
-            log::info!("JavaScript project detected. Publishing...");
-            let package_dir = target.package_dir()?;
-            let package = Package::new(&package_dir)?;
-
-            let script_path = package.main(&package_dir)?;
-
-            let assets = ServiceWorkerAssets::new(
-                script_path,
-                wasm_modules,
-                kv_namespaces.to_vec(),
-                text_blobs,
-                plain_texts,
-            )?;
-
-            service_worker::build_form(&assets, session_config)
-        }
+        TargetType::JavaScript => match &target.build {
+            Some(config) => match &config.upload_format {
+                ScriptFormat::ServiceWorker => {
+                    log::info!("Plain JavaScript project detected. Publishing...");
+                    let package_dir = target.package_dir()?;
+                    let package = Package::new(&package_dir)?;
+                    let script_path = package.main(&package_dir)?;
+
+                    let assets = ServiceWorkerAssets::new(
+                        script_path,
+                        wasm_modules,
+                        kv_namespaces.to_vec(),
+                        text_blobs,
+                        plain_texts,
+                    )?;
+
+                    service_worker::build_form(&assets, session_config)
+                }
+                ScriptFormat::Modules => {
+                    let package_dir = target.package_dir()?;
+                    let package = Package::new(&package_dir)?;
+                    let main_module = package.module(&package_dir)?;
+                    let main_module_name = filename_from_path(&main_module)
+                        .ok_or_else(|| failure::err_msg("filename required for main module"))?;
+
+                    let ignore = build_ignore(config, &package_dir)?;
+                    let modules_iter = WalkBuilder::new(config.upload_dir.clone())
+                        .standard_filters(false)
+                        .hidden(true)
+                        .overrides(ignore)
+                        .build();
+
+                    let mut modules: Vec<Module> = vec![];
+
+                    for entry in modules_iter {
+                        let entry = entry?;
+                        let path = entry.path();
+                        if path.is_file() {
+                            log::info!("Adding module {}", path.display());
+                            modules.push(Module::new(path.to_owned())?);
+                        }
+                    }
+
+                    let assets = ModulesAssets::new(
+                        main_module_name,
+                        modules,
+                        kv_namespaces.to_vec(),
+                        plain_texts,
+                    )?;
+
+                    modules_worker::build_form(&assets, session_config)
+                }
+            },
+            None => {
+                log::info!("Plain JavaScript project detected. Publishing...");
+                let package_dir = target.package_dir()?;
+                let package = Package::new(&package_dir)?;
+                let script_path = package.main(&package_dir)?;
+
+                let assets = ServiceWorkerAssets::new(
+                    script_path,
+                    wasm_modules,
+                    kv_namespaces.to_vec(),
+                    text_blobs,
+                    plain_texts,
+                )?;
+
+                service_worker::build_form(&assets, session_config)
+            }
+        },
         TargetType::Webpack => {
             log::info!("webpack project detected. Publishing...");
             // TODO: https://github.com/cloudflare/wrangler/issues/850
@@ -128,10 +187,15 @@ fn get_asset_manifest_blob(asset_manifest: AssetManifest) -> Result<String, fail
     Ok(asset_manifest)
 }
 
-fn filename_from_path(path: &PathBuf) -> Option<String> {
+fn filestem_from_path(path: &PathBuf) -> Option<String> {
     path.file_stem()?.to_str().map(|s| s.to_string())
 }
 
+fn filename_from_path(path: &PathBuf) -> Option<String> {
+    path.file_name()
+        .map(|filename| filename.to_string_lossy().into_owned())
+}
+
 fn build_generated_dir() -> Result<(), failure::Error> {
     let dir = "./worker/generated";
     if !Path::new(dir).is_dir() {
@@ -151,3 +215,21 @@ fn concat_js(name: &str) -> Result<(), failure::Error> {
     fs::write("./worker/generated/script.js", js.as_bytes())?;
     Ok(())
 }
+
+fn build_ignore(config: &Builder, directory: &Path) -> Result<Override, failure::Error> {
+    let mut overrides = OverrideBuilder::new(directory);
+    // If `include` present, use it and don't touch the `exclude` field
+    if let Some(included) = &config.upload_include {
+        for i in included {
+            overrides.add(&i)?;
+            log::info!("Including {}", i);
+        }
+    } else if let Some(excluded) = &config.upload_exclude {
+        for e in excluded {
+            overrides.add(&format!("!{}", e))?;
+            log::info!("Ignoring {}", e);
+        }
+    }
+
+    Ok(overrides.build()?)
+}

src/upload/form/modules_worker.rs

@@ -0,0 +1,75 @@
+use std::fs::File;
+
+use reqwest::blocking::multipart::{Form, Part};
+use serde::Serialize;
+
+use crate::settings::binding::Binding;
+
+use super::ModulesAssets;
+
+#[derive(Serialize, Debug)]
+struct Metadata {
+    pub main_module: String,
+    pub bindings: Vec<Binding>,
+}
+
+pub fn build_form(
+    assets: &ModulesAssets,
+    session_config: Option<serde_json::Value>,
+) -> Result<Form, failure::Error> {
+    let mut form = Form::new();
+
+    // The preview service in particular streams the request form, and requires that the
+    // "metadata" part be set first, so this order is important.
+    form = add_metadata(form, assets)?;
+    form = add_files(form, assets)?;
+    if let Some(session_config) = session_config {
+        form = add_session_config(form, session_config)?
+    }
+
+    log::info!("building form");
+    log::info!("{:#?}", &form);
+
+    Ok(form)
+}
+
+fn add_files(mut form: Form, assets: &ModulesAssets) -> Result<Form, failure::Error> {
+    for module in &assets.modules {
+        let file_name = module
+            .filename()
+            .ok_or_else(|| failure::err_msg("a filename is required for each module"))?;
+        let part = Part::reader(File::open(module.path.clone())?)
+            .mime_str(module.module_type.content_type())?
+            .file_name(file_name.clone());
+        form = form.part(file_name.clone(), part);
+    }
+    Ok(form)
+}
+
+fn add_metadata(mut form: Form, assets: &ModulesAssets) -> Result<Form, failure::Error> {
+    let metadata_json = serde_json::json!(&Metadata {
+        main_module: assets.main_module.clone(),
+        bindings: assets.bindings(),
+    });
+
+    let metadata = Part::text(metadata_json.to_string())
+        .file_name("metadata.json")
+        .mime_str("application/json")?;
+
+    form = form.part("metadata", metadata);
+
+    Ok(form)
+}
+
+fn add_session_config(
+    mut form: Form,
+    session_config: serde_json::Value,
+) -> Result<Form, failure::Error> {
+    let wrangler_session_config = Part::text(session_config.to_string())
+        .file_name("")
+        .mime_str("application/json")?;
+
+    form = form.part("wrangler-session-config", wrangler_session_config);
+
+    Ok(form)
+}

src/upload/form/project_assets.rs

@@ -3,10 +3,10 @@ use std::path::PathBuf;
 use failure::format_err;
 
 use super::binding::Binding;
-use super::filename_from_path;
 use super::plain_text::PlainText;
 use super::text_blob::TextBlob;
 use super::wasm_module::WasmModule;
+use super::{filename_from_path, filestem_from_path};
 
 use crate::settings::toml::KvNamespace;
 
@@ -28,7 +28,7 @@ impl ServiceWorkerAssets {
         text_blobs: Vec<TextBlob>,
         plain_texts: Vec<PlainText>,
     ) -> Result<Self, failure::Error> {
-        let script_name = filename_from_path(&script_path).ok_or_else(|| {
+        let script_name = filestem_from_path(&script_path).ok_or_else(|| {
             format_err!("filename should not be empty: {}", script_path.display())
         })?;
 
@@ -73,3 +73,97 @@ impl ServiceWorkerAssets {
         self.script_path.clone()
     }
 }
+
+pub struct Module {
+    pub path: PathBuf,
+    pub module_type: ModuleType,
+}
+
+impl Module {
+    pub fn new(path: PathBuf) -> Result<Module, failure::Error> {
+        let extension = path
+            .extension()
+            .ok_or_else(|| {
+                failure::err_msg(format!(
+                    "File {} lacks an extension. An extension is required to determine module type",
+                    path.display()
+                ))
+            })?
+            .to_string_lossy();
+
+        let module_type = match extension.as_ref() {
+            "mjs" => ModuleType::ES6,
+            "js" => ModuleType::CommonJS,
+            "wasm" => ModuleType::Wasm,
+            "txt" => ModuleType::Text,
+            _ => ModuleType::Data,
+        };
+
+        Ok(Module { path, module_type })
+    }
+
+    pub fn filename(&self) -> Option<String> {
+        filename_from_path(&self.path)
+    }
+}
+
+pub enum ModuleType {
+    ES6,
+    CommonJS,
+    Wasm,
+    Text,
+    Data,
+}
+
+impl ModuleType {
+    pub fn content_type(&self) -> &str {
+        match &self {
+            Self::ES6 => "application/javascript+module",
+            Self::CommonJS => "application/javascript",
+            Self::Wasm => "application/wasm",
+            Self::Text => "text/plain",
+            Self::Data => "application/octet-stream",
+        }
+    }
+}
+
+pub struct ModulesAssets {
+    pub main_module: String,
+    pub modules: Vec<Module>,
+    pub kv_namespaces: Vec<KvNamespace>,
+    pub plain_texts: Vec<PlainText>,
+}
+
+impl ModulesAssets {
+    pub fn new(
+        main_module: String,
+        modules: Vec<Module>,
+        kv_namespaces: Vec<KvNamespace>,
+        plain_texts: Vec<PlainText>,
+    ) -> Result<Self, failure::Error> {
+        Ok(Self {
+            main_module,
+            modules,
+            kv_namespaces,
+            plain_texts,
+        })
+    }
+
+    pub fn bindings(&self) -> Vec<Binding> {
+        let mut bindings = Vec::new();
+
+        // Bindings that refer to a `part` of the uploaded files
+        // in the service-worker format, are now modules.
+
+        for kv in &self.kv_namespaces {
+            let binding = kv.binding();
+            bindings.push(binding);
+        }
+        for plain_text in &self.plain_texts {
+            let binding = plain_text.binding();
+            bindings.push(binding);
+        }
+
+        bindings
+    }
+}