Remove github restriction

Cargo.toml

@@ -11,6 +11,7 @@ license = "MIT"
 anyhow = "1.0.71"
 dunce = "1.0.2"
 env_logger = "0.10"
+fnv = "1.0"
 globset = { version = "0.4.13", features = ["serde1"] }
 ignore = "0.4"
 indexmap = { version = "1.9.2", features = ["arbitrary", "rayon", "serde-1"] }
@@ -22,18 +23,27 @@ nom = "7.1"
 once_cell = "1.12"
 proc-macro2 = { version = "1.0.64", features = ["span-locations"] }
 rayon = "1.2"
-regex = "1.9.2"
 rustsec = { version = "0.26", features = ["fix"] }
 semver = { version = "1.0.17", features = ["serde"] }
 serde = { version = "1.0.185", features = ["derive", "rc"] }
-serde_json = { version = "1.0.100", features = ["float_roundtrip", "unbounded_depth"] }
+serde_json = { version = "1.0.100", features = [
+    "float_roundtrip",
+    "unbounded_depth",
+] }
 serde_starlark = "0.1.13"
 structopt = "0.3.23"
 strum = { version = "0.24", features = ["derive"] }
-syn = { version = "2.0.23", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn = { version = "2.0.23", features = [
+    "extra-traits",
+    "fold",
+    "full",
+    "visit",
+    "visit-mut",
+] }
 termcolor = "1.0"
 toml = "0.7.3"
 unicode-ident = "1.0.10"
+url = "2.4"
 walkdir = "2.3"
 
 [dev-dependencies]

src/buckify.rs

@@ -23,8 +23,6 @@ use std::sync::Mutex;
 use anyhow::bail;
 use anyhow::Context;
 use anyhow::Result;
-use once_cell::sync::OnceCell;
-use regex::Regex;
 
 use crate::buck;
 use crate::buck::Alias;
@@ -237,7 +235,7 @@ fn generate_nonvendored_sources_archive<'scope>(
         } => generate_git_fetch(repo, commit_hash).map(Some),
         Source::Unrecognized(_) => {
             bail!(
-                "`vendor = false` mode is supported only with exclusively crates.io and GitHub dependencies. \"{}\" {} is coming from some source other than crates.io or GitHub",
+                "`vendor = false` mode is supported only with exclusively crates.io and https git dependencies. \"{}\" {} is coming from some other source",
                 pkg.name,
                 pkg.version,
             );
@@ -289,23 +287,53 @@ fn generate_git_fetch(repo: &str, commit_hash: &str) -> Result<Rule> {
     }))
 }
 
-/// Extract the "serde-rs/serde" part of "https://github.com/serde-rs/serde.git".
-pub fn short_name_for_git_repo(repo: &str) -> Result<&str> {
-    static GITHUB_URL_REGEX: OnceCell<Regex> = OnceCell::new();
-    let github_url_regex = GITHUB_URL_REGEX
-        .get_or_try_init(|| Regex::new(r"^https://github.com/([[:alnum:].-]+/[[:alnum:].-]+)$"))?;
+/// Create a uniquely hashed directory name for the arbitrary source url
+pub fn short_name_for_git_repo(repo: &str) -> Result<String> {
+    let mut canonical = url::Url::parse(&repo.to_lowercase()).context("invalid url")?;
 
-    let repo_without_extension = repo.strip_suffix(".git").unwrap_or(repo);
-    if let Some(captures) = github_url_regex.captures(repo_without_extension) {
-        Ok(captures.get(1).unwrap().as_str())
-    } else {
-        // TODO: come up with some mangling scheme for arbitrary repo URLs. Buck
-        // does not permit ':' in target names.
-        bail!(
-            "unsupported git URL: {:?}, currently vendor=false mode only supports \"https://github.com/$OWNER/$REPO\" repositories",
-            repo,
-        );
+    anyhow::ensure!(
+        canonical.scheme() == "https",
+        "only https git urls are supported"
+    );
+
+    canonical.set_query(None);
+    canonical.set_fragment(None);
+
+    // It would be nice to just say "you're using a .git extension, please remove it",
+    // but unfortunately some git providers (notably gitlab) require the .git extension
+    // in the url, but other providers, notably github, treat urls with or without
+    // the extension exactly the same. If we don't take the .git extension into
+    // account at all we could run into a situation where 2 or more crates are
+    // sourced from the same git repo but with and without the .git extension,
+    // causing them to be hashed and placed differently
+    if canonical.path().ends_with(".git") {
+        // This is less efficient but far simpler than using the really unfriendly
+        // path_segments_mut API
+        let stripped = canonical.path().trim_end_matches(".git").to_owned();
+        canonical.set_path(&stripped);
+    }
+
+    let mut dir_name = canonical
+        .path_segments()
+        .and_then(|mut it| it.next_back())
+        .unwrap_or("_empty")
+        .to_owned();
+
+    let hash = {
+        use std::hash::{Hash, Hasher};
+        let mut hasher = fnv::FnvHasher::default();
+        canonical.hash(&mut hasher);
+        hasher.finish()
+    };
+
+    dir_name.push('-');
+
+    for byte in hash.to_le_bytes() {
+        use std::fmt::Write;
+        write!(&mut dir_name, "{byte:0x}").unwrap();
     }
+
+    Ok(dir_name)
 }
 
 /// Find the git repository containing the given manifest directory.
@@ -914,3 +942,29 @@ pub(crate) fn buckify(config: &Config, args: &Args, paths: &Paths, stdout: bool)
 
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+    use super::short_name_for_git_repo;
+
+    #[test]
+    fn hashes_with_same_repo_variations() {
+        let same = [
+            short_name_for_git_repo("https://github.com/facebookincubator/reindeer.git").unwrap(),
+            short_name_for_git_repo("https://github.com/facebookincubator/reindeer").unwrap(),
+            short_name_for_git_repo("https://github.com/FacebookIncubator/reindeer.git").unwrap(),
+        ];
+
+        assert!(!same
+            .iter()
+            .any(|dir_name| dir_name != "reindeer-3e497668718a129"));
+    }
+
+    #[test]
+    fn hashes_non_github() {
+        assert_eq!(
+            short_name_for_git_repo("https://gitlab.com/gilrs-project/gilrs.git").unwrap(),
+            "gilrs-1b413f0b5e8e0bb"
+        );
+    }
+}