Asset Manifest (#629)

Add asset manifest logic (creation and binding upon worker upload)

Cargo.toml

@@ -44,6 +44,8 @@ walkdir = "2.2.9"
 percent-encoding = "1.0.1"
 http = "0.1.1"
 regex = "1"
+sha2 = "0.8.0"
+data-encoding = "2.1.2"
 
 [dev-dependencies]
 assert_cmd = "0.11.1"

src/commands/kv/bucket/mod.rs

@@ -4,10 +4,14 @@ mod delete;
 mod sync;
 mod upload;
 
+use data_encoding::HEXLOWER;
+use sha2::{Digest, Sha256};
+
 pub use delete::delete;
 pub use sync::sync;
 pub use upload::upload;
 
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::path::Path;
 
@@ -17,34 +21,37 @@ use walkdir::WalkDir;
 
 use crate::terminal::message;
 
-fn directory_keys_values(
+pub fn directory_keys_values(
     directory: &Path,
     verbose: bool,
-) -> Result<Vec<KeyValuePair>, failure::Error> {
+) -> Result<(Vec<KeyValuePair>, HashMap<String, String>), failure::Error> {
     let mut upload_vec: Vec<KeyValuePair> = Vec::new();
+    let mut key_manifest: HashMap<String, String> = HashMap::new();
+
     for entry in WalkDir::new(directory) {
         let entry = entry.unwrap();
         let path = entry.path();
         if path.is_file() {
-            let key = generate_key(path, directory)?;
-
             let value = std::fs::read(path)?;
 
             // Need to base64 encode value
             let b64_value = base64::encode(&value);
+
+            let (path, key) = generate_key(path, directory, Some(b64_value.clone()))?;
             if verbose {
                 message::working(&format!("Parsing {}...", key.clone()));
             }
             upload_vec.push(KeyValuePair {
-                key,
+                key: key.clone(),
                 value: b64_value,
                 expiration: None,
                 expiration_ttl: None,
                 base64: Some(true),
             });
+            key_manifest.insert(path, key);
         }
     }
-    Ok(upload_vec)
+    Ok((upload_vec, key_manifest))
 }
 
 fn directory_keys_only(directory: &Path) -> Result<Vec<String>, failure::Error> {
@@ -53,16 +60,28 @@ fn directory_keys_only(directory: &Path) -> Result<Vec<String>, failure::Error>
         let entry = entry.unwrap();
         let path = entry.path();
         if path.is_file() {
-            let key = generate_key(path, directory)?;
+            let (key, _) = generate_key(path, directory, None)?;
 
             upload_vec.push(key);
         }
     }
     Ok(upload_vec)
 }
 
-// Courtesy of Steve Kalabnik's PoC :) Used for bulk operations (write, delete)
-fn generate_key(path: &Path, directory: &Path) -> Result<String, failure::Error> {
+fn get_digest(value: String) -> Result<String, failure::Error> {
+    let mut hasher = Sha256::new();
+    hasher.input(value);
+    let digest = hasher.result();
+    let hex_digest = HEXLOWER.encode(digest.as_ref());
+    Ok(hex_digest)
+}
+
+// Courtesy of Steve Klabnik's PoC :) Used for bulk operations (write, delete)
+fn generate_key(
+    path: &Path,
+    directory: &Path,
+    value: Option<String>,
+) -> Result<(String, String), failure::Error> {
     let path = path.strip_prefix(directory).unwrap();
 
     // next, we have to re-build the paths: if we're on Windows, we have paths with
@@ -84,5 +103,12 @@ fn generate_key(path: &Path, directory: &Path) -> Result<String, failure::Error>
         .to_str()
         .unwrap_or_else(|| panic!("found a non-UTF-8 path, {:?}", path_with_forward_slash));
 
-    Ok(path.to_string())
+    let path_with_hash = if let Some(value) = value {
+        let digest = get_digest(value)?;
+        format!("{}-{}", path, digest)
+    } else {
+        path.to_string()
+    };
+
+    Ok((path.to_string(), path_with_hash))
 }

src/commands/kv/bucket/upload.rs

@@ -1,13 +1,13 @@
 use std::fs::metadata;
 use std::path::Path;
 
-use cloudflare::endpoints::workerskv::write_bulk::KeyValuePair;
-
 use crate::commands::kv::bucket::directory_keys_values;
 use crate::commands::kv::bulk::put::put_bulk;
 use crate::settings::global_user::GlobalUser;
 use crate::settings::target::Target;
 use crate::terminal::message;
+use cloudflare::endpoints::workerskv::write_bulk::KeyValuePair;
+use failure::format_err;
 
 const KEY_MAX_SIZE: usize = 512;
 const VALUE_MAX_SIZE: usize = 2 * 1024 * 1024;
@@ -40,12 +40,15 @@ pub fn upload_files(
     verbose: bool,
 ) -> Result<(), failure::Error> {
     let mut pairs: Vec<KeyValuePair> = match &metadata(path) {
-        Ok(file_type) if file_type.is_dir() => directory_keys_values(path, verbose),
+        Ok(file_type) if file_type.is_dir() => {
+            let (p, _) = directory_keys_values(path, verbose)?;
+            Ok(p)
+        }
         Ok(_file_type) => {
             // any other file types (files, symlinks)
-            failure::bail!("wrangler kv:bucket upload takes a directory")
+            Err(format_err!("wrangler kv:bucket upload takes a directory"))
         }
-        Err(e) => failure::bail!("{}", e),
+        Err(e) => Err(format_err!("{}", e)),
     }?;
 
     validate_file_uploads(pairs.clone())?;

src/commands/publish/upload_form/mod.rs

@@ -1,18 +1,20 @@
 mod project_assets;
+mod text_blob;
 mod wasm_module;
 
 use reqwest::multipart::{Form, Part};
 use std::fs;
 use std::path::Path;
 
 use crate::commands::build::wranglerjs;
+use crate::commands::kv::bucket::directory_keys_values;
 use crate::settings::binding;
-
 use crate::settings::metadata::Metadata;
 use crate::settings::target::kv_namespace;
 use crate::settings::target::{Target, TargetType};
 
 use project_assets::ProjectAssets;
+use text_blob::TextBlob;
 use wasm_module::WasmModule;
 
 use super::{krate, Package};
@@ -34,7 +36,8 @@ pub fn build_script_upload_form(target: &Target) -> Result<Form, failure::Error>
 
             let script_path = "./worker/generated/script.js".to_string();
 
-            let assets = ProjectAssets::new(script_path, vec![wasm_module], kv_namespaces)?;
+            let assets =
+                ProjectAssets::new(script_path, vec![wasm_module], kv_namespaces, Vec::new())?;
 
             build_form(&assets)
         }
@@ -45,7 +48,7 @@ pub fn build_script_upload_form(target: &Target) -> Result<Form, failure::Error>
 
             let script_path = package.main(&build_dir)?;
 
-            let assets = ProjectAssets::new(script_path, Vec::new(), kv_namespaces)?;
+            let assets = ProjectAssets::new(script_path, Vec::new(), kv_namespaces, Vec::new())?;
 
             build_form(&assets)
         }
@@ -63,16 +66,33 @@ pub fn build_script_upload_form(target: &Target) -> Result<Form, failure::Error>
                 let path = bundle.wasm_path();
                 let binding = bundle.get_wasm_binding();
                 let wasm_module = WasmModule::new(path, binding)?;
-                wasm_modules.push(wasm_module)
+                wasm_modules.push(wasm_module);
             }
 
-            let assets = ProjectAssets::new(script_path, wasm_modules, kv_namespaces)?;
+            let mut text_blobs = Vec::new();
+
+            if let Some(site) = &target.site {
+                log::info!("adding __STATIC_CONTENT_MANIFEST");
+                let binding = "__STATIC_CONTENT_MANIFEST".to_string();
+                let asset_manifest = get_asset_manifest(&site.bucket)?;
+                let text_blob = TextBlob::new(asset_manifest, binding)?;
+                text_blobs.push(text_blob);
+            }
+
+            let assets = ProjectAssets::new(script_path, wasm_modules, kv_namespaces, text_blobs)?;
 
             build_form(&assets)
         }
     }
 }
 
+fn get_asset_manifest(directory: &str) -> Result<String, failure::Error> {
+    let directory = Path::new(&directory);
+    let (_, manifest) = directory_keys_values(directory, false)?;
+    let manifest = serde_json::to_string(&manifest)?;
+    Ok(manifest)
+}
+
 fn build_form(assets: &ProjectAssets) -> Result<Form, failure::Error> {
     let mut form = Form::new();
 
@@ -81,7 +101,7 @@ fn build_form(assets: &ProjectAssets) -> Result<Form, failure::Error> {
     form = add_metadata(form, assets)?;
     form = add_files(form, assets)?;
 
-    log::info!("{:?}", &form);
+    log::info!("{:#?}", &form);
 
     Ok(form)
 }
@@ -93,6 +113,14 @@ fn add_files(mut form: Form, assets: &ProjectAssets) -> Result<Form, failure::Er
         form = form.file(wasm_module.filename(), wasm_module.path())?;
     }
 
+    for text_blob in &assets.text_blobs {
+        let part = Part::text(text_blob.data.clone())
+            .file_name(text_blob.binding.clone())
+            .mime_str("text/plain")?;
+
+        form = form.part(text_blob.binding.clone(), part);
+    }
+
     Ok(form)
 }
 

src/commands/publish/upload_form/project_assets.rs

@@ -3,6 +3,7 @@ use failure::format_err;
 use super::binding::Binding;
 use super::filename_from_path;
 use super::kv_namespace::KvNamespace;
+use super::text_blob::TextBlob;
 use super::wasm_module::WasmModule;
 
 #[derive(Debug)]
@@ -11,13 +12,15 @@ pub struct ProjectAssets {
     script_path: String,
     pub wasm_modules: Vec<WasmModule>,
     pub kv_namespaces: Vec<KvNamespace>,
+    pub text_blobs: Vec<TextBlob>,
 }
 
 impl ProjectAssets {
     pub fn new(
         script_path: String,
         wasm_modules: Vec<WasmModule>,
         kv_namespaces: Vec<KvNamespace>,
+        text_blobs: Vec<TextBlob>,
     ) -> Result<Self, failure::Error> {
         let script_name = filename_from_path(&script_path)
             .ok_or_else(|| format_err!("filename should not be empty: {}", script_path))?;
@@ -27,6 +30,7 @@ impl ProjectAssets {
             script_path,
             wasm_modules,
             kv_namespaces,
+            text_blobs,
         })
     }
 
@@ -41,6 +45,10 @@ impl ProjectAssets {
             let binding = kv.binding();
             bindings.push(binding);
         }
+        for blob in &self.text_blobs {
+            let binding = blob.binding();
+            bindings.push(binding);
+        }
 
         bindings
     }

src/commands/publish/upload_form/text_blob.rs

@@ -0,0 +1,18 @@
+use super::binding::Binding;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct TextBlob {
+    pub data: String,
+    pub binding: String,
+}
+
+impl TextBlob {
+    pub fn new(data: String, binding: String) -> Result<Self, failure::Error> {
+        Ok(Self { data, binding })
+    }
+
+    pub fn binding(&self) -> Binding {
+        Binding::new_text_blob(self.binding.clone(), self.binding.clone())
+    }
+}

src/settings/binding.rs

@@ -7,6 +7,8 @@ pub enum Binding {
     WasmModule { name: String, part: String },
     #[serde(rename = "kv_namespace")]
     KvNamespace { name: String, namespace_id: String },
+    #[serde(rename = "text_blob")]
+    TextBlob { name: String, part: String },
 }
 
 impl Binding {
@@ -20,4 +22,8 @@ impl Binding {
     pub fn new_kv_namespace(name: String, namespace_id: String) -> Binding {
         Binding::KvNamespace { name, namespace_id }
     }
+
+    pub fn new_text_blob(name: String, part: String) -> Binding {
+        Binding::TextBlob { name, part }
+    }
 }