From ab5b845efc9f87c5de8939cc2e5cbb3aec03e7c4 Mon Sep 17 00:00:00 2001 From: Avery Harnish Date: Wed, 22 Apr 2020 11:33:52 -0500 Subject: [PATCH] use twox_hash for asset manifest --- Cargo.lock | 24 ++++++++++------------- Cargo.toml | 3 +-- src/commands/kv/bucket/mod.rs | 33 ++++++++++++++++++-------------- src/commands/kv/bucket/upload.rs | 1 - 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d311874cf..927c3add2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2084,18 +2084,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "sha2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" -dependencies = [ - "block-buffer", - "digest", - "fake-simd", - "opaque-debug", -] - [[package]] name = "siphasher" version = "0.2.3" @@ -2519,6 +2507,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +dependencies = [ + "rand 0.7.3", +] + [[package]] name = "typenum" version = "1.11.2" @@ -2790,7 +2787,6 @@ dependencies = [ "cloudflare", "config", "console 0.9.2", - "data-encoding", "dirs 1.0.5", "env_logger", "exitfailure", @@ -2818,13 +2814,13 @@ dependencies = [ "serde 1.0.105", "serde_json", "serde_with", - "sha2", "tempfile", "term_size", "text_io", "tokio", "tokio-tungstenite", "toml", + "twox-hash", "url", "uuid", "which", diff --git a/Cargo.toml b/Cargo.toml index 16c97ee51..e84b31490 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ clap = "2.32.0" cloudflare = "0.6.4" config = "0.10.1" console = "0.9.1" -data-encoding = "2.1.2" dirs = "1.0.5" env_logger = "0.6.1" exitfailure = "0.5.1" @@ -48,7 +47,6 @@ reqwest = { version = "0.10.1", features = ["blocking", "json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.39" serde_with = "1.3.1" -sha2 = "0.8.0" tempfile = "3.1.0" term_size = "0.3" text_io = "0.1.7" @@ -59,6 +57,7 @@ url = "2.1.0" uuid = { version = "0.8", features = ["v4"] } which = "2.0.1" ws = "0.9.0" +twox-hash = "1.5.0" [dev-dependencies] assert_cmd = "0.11.1" diff --git a/src/commands/kv/bucket/mod.rs b/src/commands/kv/bucket/mod.rs index 77a350a71..7eabb04b0 100644 --- a/src/commands/kv/bucket/mod.rs +++ b/src/commands/kv/bucket/mod.rs @@ -10,14 +10,14 @@ pub use upload::upload_files; use std::ffi::OsString; use std::fs; +use std::hash::Hasher; use std::path::Path; -use data_encoding::HEXLOWER; use failure::format_err; use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; use indicatif::{ProgressBar, ProgressStyle}; -use sha2::{Digest, Sha256}; +use twox_hash::XxHash64; use cloudflare::endpoints::workerskv::write_bulk::KeyValuePair; @@ -197,10 +197,15 @@ pub fn generate_path_and_key( let relative_path = path.strip_prefix(directory).unwrap(); let url_safe_path = generate_url_safe_path(relative_path)?; - let path_with_hash = if let Some(value) = value { - let digest = get_digest(value)?; - + let digest = get_digest(value); + // it is ok to truncate the digest here because + // we also include the file name in the asset manifest key + // + // the most important thing here is to detect changes + // of a single file to invalidate the cache and + // it's impossible to serve two different files with the same name + let digest = digest[0..10].to_string(); generate_path_with_hash(relative_path, digest)? } else { url_safe_path.to_owned() @@ -209,12 +214,13 @@ pub fn generate_path_and_key( Ok((url_safe_path, path_with_hash)) } -fn get_digest(value: String) -> Result { - let mut hasher = Sha256::new(); - hasher.input(value); - let digest = hasher.result(); - let hex_digest = HEXLOWER.encode(digest.as_ref())[0..10].to_string(); - Ok(hex_digest) +fn get_digest(value: String) -> String { + let value = value.as_bytes(); + let mut hasher = XxHash64::default(); + hasher.write(value); + let digest = hasher.finish(); + // encode u64 as hexadecimal to save space and information + format!("{:x}", digest) } // Assumes that `path` is a file (called from a match branch for path.is_file()) @@ -473,7 +479,7 @@ mod tests { #[test] fn it_inserts_hash_before_extension() { let value = "

Hello World!

"; - let hashed_value = get_digest(String::from(value)).unwrap(); + let hashed_value = get_digest(String::from(value)); let path = PathBuf::from("path").join("to").join("asset.html"); let actual_path_with_hash = @@ -487,7 +493,7 @@ mod tests { #[test] fn it_inserts_hash_without_extension() { let value = "

Hello World!

"; - let hashed_value = get_digest(String::from(value)).unwrap(); + let hashed_value = get_digest(String::from(value)); let path = PathBuf::from("path").join("to").join("asset"); let actual_path_with_hash = @@ -525,7 +531,6 @@ mod tests { let directory = Path::new("./build"); let value = Some("

Hello World!

".to_string()); let (path, key) = generate_path_and_key(path, directory, value).unwrap(); - let expected_path = "path/to/asset.ext".to_string(); let expected_key_regex = Regex::new(r"^path/to/asset\.[0-9a-f]{10}\.ext").unwrap(); diff --git a/src/commands/kv/bucket/upload.rs b/src/commands/kv/bucket/upload.rs index 415a8468d..fbbbc1b59 100644 --- a/src/commands/kv/bucket/upload.rs +++ b/src/commands/kv/bucket/upload.rs @@ -34,7 +34,6 @@ pub fn upload_files( } else { None }; - while !(pairs.is_empty() && key_value_batch.is_empty()) { if pairs.is_empty() { // Last batch to upload