diff --git a/Cargo.lock b/Cargo.lock index 538abfaaa..0bee93a56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,6 +1789,17 @@ dependencies = [ "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sha2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "siphasher" version = "0.2.3" @@ -2387,6 +2398,7 @@ dependencies = [ "cloudflare 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "config 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "console 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)", + "data-encoding 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "exitfailure 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2407,6 +2419,7 @@ dependencies = [ "reqwest 0.9.20 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "text_io 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2673,6 +2686,7 @@ dependencies = [ "checksum serde_with_macros 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6304d92ad5493e340b95c353b8328c312d020f0eb5cb6df8506f160f5b7300d" "checksum serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "38b08a9a90e5260fe01c6480ec7c811606df6d3a660415808c3c3fa8ed95b582" "checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68" +"checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cc9c640a4adbfbcc11ffb95efe5aa7af7309e002adab54b185507dbf2377b99" diff --git a/Cargo.toml b/Cargo.toml index da25101b7..73b5dcedd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ walkdir = "2.2.9" percent-encoding = "1.0.1" http = "0.1.1" regex = "1" +sha2 = "0.8.0" +data-encoding = "2.1.2" [dev-dependencies] assert_cmd = "0.11.1" diff --git a/src/commands/kv/bucket/mod.rs b/src/commands/kv/bucket/mod.rs index be0ecaf5c..b2fa0033c 100644 --- a/src/commands/kv/bucket/mod.rs +++ b/src/commands/kv/bucket/mod.rs @@ -4,10 +4,14 @@ mod delete; mod sync; mod upload; +use data_encoding::HEXLOWER; +use sha2::{Digest, Sha256}; + pub use delete::delete; pub use sync::sync; pub use upload::upload; +use std::collections::HashMap; use std::ffi::OsString; use std::path::Path; @@ -17,34 +21,37 @@ use walkdir::WalkDir; use crate::terminal::message; -fn directory_keys_values( +pub fn directory_keys_values( directory: &Path, verbose: bool, -) -> Result, failure::Error> { +) -> Result<(Vec, HashMap), failure::Error> { let mut upload_vec: Vec = Vec::new(); + let mut key_manifest: HashMap = HashMap::new(); + for entry in WalkDir::new(directory) { let entry = entry.unwrap(); let path = entry.path(); if path.is_file() { - let key = generate_key(path, directory)?; - let value = std::fs::read(path)?; // Need to base64 encode value let b64_value = base64::encode(&value); + + let (path, key) = generate_key(path, directory, Some(b64_value.clone()))?; if verbose { message::working(&format!("Parsing {}...", key.clone())); } upload_vec.push(KeyValuePair { - key, + key: key.clone(), value: b64_value, expiration: None, expiration_ttl: None, base64: Some(true), }); + key_manifest.insert(path, key); } } - Ok(upload_vec) + Ok((upload_vec, key_manifest)) } fn directory_keys_only(directory: &Path) -> Result, failure::Error> { @@ -53,7 +60,7 @@ fn directory_keys_only(directory: &Path) -> Result, failure::Error> let entry = entry.unwrap(); let path = entry.path(); if path.is_file() { - let key = generate_key(path, directory)?; + let (key, _) = generate_key(path, directory, None)?; upload_vec.push(key); } @@ -61,8 +68,20 @@ fn directory_keys_only(directory: &Path) -> Result, failure::Error> Ok(upload_vec) } -// Courtesy of Steve Kalabnik's PoC :) Used for bulk operations (write, delete) -fn generate_key(path: &Path, directory: &Path) -> Result { +fn get_digest(value: String) -> Result { + let mut hasher = Sha256::new(); + hasher.input(value); + let digest = hasher.result(); + let hex_digest = HEXLOWER.encode(digest.as_ref()); + Ok(hex_digest) +} + +// Courtesy of Steve Klabnik's PoC :) Used for bulk operations (write, delete) +fn generate_key( + path: &Path, + directory: &Path, + value: Option, +) -> Result<(String, String), failure::Error> { let path = path.strip_prefix(directory).unwrap(); // next, we have to re-build the paths: if we're on Windows, we have paths with @@ -84,5 +103,12 @@ fn generate_key(path: &Path, directory: &Path) -> Result .to_str() .unwrap_or_else(|| panic!("found a non-UTF-8 path, {:?}", path_with_forward_slash)); - Ok(path.to_string()) + let path_with_hash = if let Some(value) = value { + let digest = get_digest(value)?; + format!("{}-{}", path, digest) + } else { + path.to_string() + }; + + Ok((path.to_string(), path_with_hash)) } diff --git a/src/commands/kv/bucket/upload.rs b/src/commands/kv/bucket/upload.rs index 29f0ac7ef..fd077d68b 100644 --- a/src/commands/kv/bucket/upload.rs +++ b/src/commands/kv/bucket/upload.rs @@ -1,13 +1,13 @@ use std::fs::metadata; use std::path::Path; -use cloudflare::endpoints::workerskv::write_bulk::KeyValuePair; - use crate::commands::kv::bucket::directory_keys_values; use crate::commands::kv::bulk::put::put_bulk; use crate::settings::global_user::GlobalUser; use crate::settings::target::Target; use crate::terminal::message; +use cloudflare::endpoints::workerskv::write_bulk::KeyValuePair; +use failure::format_err; const KEY_MAX_SIZE: usize = 512; const VALUE_MAX_SIZE: usize = 2 * 1024 * 1024; @@ -40,12 +40,15 @@ pub fn upload_files( verbose: bool, ) -> Result<(), failure::Error> { let mut pairs: Vec = match &metadata(path) { - Ok(file_type) if file_type.is_dir() => directory_keys_values(path, verbose), + Ok(file_type) if file_type.is_dir() => { + let (p, _) = directory_keys_values(path, verbose)?; + Ok(p) + } Ok(_file_type) => { // any other file types (files, symlinks) - failure::bail!("wrangler kv:bucket upload takes a directory") + Err(format_err!("wrangler kv:bucket upload takes a directory")) } - Err(e) => failure::bail!("{}", e), + Err(e) => Err(format_err!("{}", e)), }?; validate_file_uploads(pairs.clone())?; diff --git a/src/commands/publish/upload_form/mod.rs b/src/commands/publish/upload_form/mod.rs index c25c8caf6..ba8617b0e 100644 --- a/src/commands/publish/upload_form/mod.rs +++ b/src/commands/publish/upload_form/mod.rs @@ -1,4 +1,5 @@ mod project_assets; +mod text_blob; mod wasm_module; use reqwest::multipart::{Form, Part}; @@ -6,13 +7,14 @@ use std::fs; use std::path::Path; use crate::commands::build::wranglerjs; +use crate::commands::kv::bucket::directory_keys_values; use crate::settings::binding; - use crate::settings::metadata::Metadata; use crate::settings::target::kv_namespace; use crate::settings::target::{Target, TargetType}; use project_assets::ProjectAssets; +use text_blob::TextBlob; use wasm_module::WasmModule; use super::{krate, Package}; @@ -34,7 +36,8 @@ pub fn build_script_upload_form(target: &Target) -> Result let script_path = "./worker/generated/script.js".to_string(); - let assets = ProjectAssets::new(script_path, vec![wasm_module], kv_namespaces)?; + let assets = + ProjectAssets::new(script_path, vec![wasm_module], kv_namespaces, Vec::new())?; build_form(&assets) } @@ -45,7 +48,7 @@ pub fn build_script_upload_form(target: &Target) -> Result let script_path = package.main(&build_dir)?; - let assets = ProjectAssets::new(script_path, Vec::new(), kv_namespaces)?; + let assets = ProjectAssets::new(script_path, Vec::new(), kv_namespaces, Vec::new())?; build_form(&assets) } @@ -63,16 +66,33 @@ pub fn build_script_upload_form(target: &Target) -> Result let path = bundle.wasm_path(); let binding = bundle.get_wasm_binding(); let wasm_module = WasmModule::new(path, binding)?; - wasm_modules.push(wasm_module) + wasm_modules.push(wasm_module); } - let assets = ProjectAssets::new(script_path, wasm_modules, kv_namespaces)?; + let mut text_blobs = Vec::new(); + + if let Some(site) = &target.site { + log::info!("adding __STATIC_CONTENT_MANIFEST"); + let binding = "__STATIC_CONTENT_MANIFEST".to_string(); + let asset_manifest = get_asset_manifest(&site.bucket)?; + let text_blob = TextBlob::new(asset_manifest, binding)?; + text_blobs.push(text_blob); + } + + let assets = ProjectAssets::new(script_path, wasm_modules, kv_namespaces, text_blobs)?; build_form(&assets) } } } +fn get_asset_manifest(directory: &str) -> Result { + let directory = Path::new(&directory); + let (_, manifest) = directory_keys_values(directory, false)?; + let manifest = serde_json::to_string(&manifest)?; + Ok(manifest) +} + fn build_form(assets: &ProjectAssets) -> Result { let mut form = Form::new(); @@ -81,7 +101,7 @@ fn build_form(assets: &ProjectAssets) -> Result { form = add_metadata(form, assets)?; form = add_files(form, assets)?; - log::info!("{:?}", &form); + log::info!("{:#?}", &form); Ok(form) } @@ -93,6 +113,14 @@ fn add_files(mut form: Form, assets: &ProjectAssets) -> Result, pub kv_namespaces: Vec, + pub text_blobs: Vec, } impl ProjectAssets { @@ -18,6 +20,7 @@ impl ProjectAssets { script_path: String, wasm_modules: Vec, kv_namespaces: Vec, + text_blobs: Vec, ) -> Result { let script_name = filename_from_path(&script_path) .ok_or_else(|| format_err!("filename should not be empty: {}", script_path))?; @@ -27,6 +30,7 @@ impl ProjectAssets { script_path, wasm_modules, kv_namespaces, + text_blobs, }) } @@ -41,6 +45,10 @@ impl ProjectAssets { let binding = kv.binding(); bindings.push(binding); } + for blob in &self.text_blobs { + let binding = blob.binding(); + bindings.push(binding); + } bindings } diff --git a/src/commands/publish/upload_form/text_blob.rs b/src/commands/publish/upload_form/text_blob.rs new file mode 100644 index 000000000..e6c7e4694 --- /dev/null +++ b/src/commands/publish/upload_form/text_blob.rs @@ -0,0 +1,18 @@ +use super::binding::Binding; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct TextBlob { + pub data: String, + pub binding: String, +} + +impl TextBlob { + pub fn new(data: String, binding: String) -> Result { + Ok(Self { data, binding }) + } + + pub fn binding(&self) -> Binding { + Binding::new_text_blob(self.binding.clone(), self.binding.clone()) + } +} diff --git a/src/settings/binding.rs b/src/settings/binding.rs index 45dba32bd..1b6ad6e45 100644 --- a/src/settings/binding.rs +++ b/src/settings/binding.rs @@ -7,6 +7,8 @@ pub enum Binding { WasmModule { name: String, part: String }, #[serde(rename = "kv_namespace")] KvNamespace { name: String, namespace_id: String }, + #[serde(rename = "text_blob")] + TextBlob { name: String, part: String }, } impl Binding { @@ -20,4 +22,8 @@ impl Binding { pub fn new_kv_namespace(name: String, namespace_id: String) -> Binding { Binding::KvNamespace { name, namespace_id } } + + pub fn new_text_blob(name: String, part: String) -> Binding { + Binding::TextBlob { name, part } + } }