diff --git a/maintainers/scripts/convert-to-import-cargo-lock.sh b/maintainers/scripts/convert-to-import-cargo-lock.sh new file mode 100755 index 0000000000000..b38825d4d3e0c --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=. -i bash -p "import ./maintainers/scripts/convert-to-import-cargo-lock" nix-prefetch-git + +convert-to-import-cargo-lock "$@" diff --git a/maintainers/scripts/convert-to-import-cargo-lock/.gitignore b/maintainers/scripts/convert-to-import-cargo-lock/.gitignore new file mode 100644 index 0000000000000..ea8c4bf7f35f6 --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/.gitignore @@ -0,0 +1 @@ +/target diff --git a/maintainers/scripts/convert-to-import-cargo-lock/Cargo.lock b/maintainers/scripts/convert-to-import-cargo-lock/Cargo.lock new file mode 100644 index 0000000000000..b69fbc59ae846 --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/Cargo.lock @@ -0,0 +1,106 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" + +[[package]] +name = "basic-toml" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e819b667739967cd44d308b8c7b71305d8bb0729ac44a248aa08f33d01950b4" +dependencies = [ + "serde", +] + +[[package]] +name = "convert-to-import-cargo-lock" +version = "0.1.0" +dependencies = [ + "anyhow", + "basic-toml", + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" diff --git a/maintainers/scripts/convert-to-import-cargo-lock/Cargo.toml b/maintainers/scripts/convert-to-import-cargo-lock/Cargo.toml new file mode 100644 index 0000000000000..41f5729f01a2c --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "convert-to-import-cargo-lock" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = { version = "1.0.69" } +basic-toml = "0.1.1" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" diff --git a/maintainers/scripts/convert-to-import-cargo-lock/default.nix b/maintainers/scripts/convert-to-import-cargo-lock/default.nix new file mode 100644 index 0000000000000..f4c1f553d64f9 --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/default.nix @@ -0,0 +1,16 @@ +with import ../../../. { }; + +rustPlatform.buildRustPackage { + name = "convert-to-import-cargo-lock"; + + src = lib.cleanSourceWith { + src = ./.; + filter = name: type: + let + name' = builtins.baseNameOf name; + in + name' != "default.nix" && name' != "target"; + }; + + cargoLock.lockFile = ./Cargo.lock; +} diff --git a/maintainers/scripts/convert-to-import-cargo-lock/shell.nix b/maintainers/scripts/convert-to-import-cargo-lock/shell.nix new file mode 100644 index 0000000000000..8e913fdcd8be5 --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/shell.nix @@ -0,0 +1,5 @@ +with import ../../../. { }; + +mkShell { + packages = [ rustc cargo clippy rustfmt ] ++ lib.optional stdenv.isDarwin libiconv; +} diff --git a/maintainers/scripts/convert-to-import-cargo-lock/src/main.rs b/maintainers/scripts/convert-to-import-cargo-lock/src/main.rs new file mode 100644 index 0000000000000..3b9bef83e6838 --- /dev/null +++ b/maintainers/scripts/convert-to-import-cargo-lock/src/main.rs @@ -0,0 +1,246 @@ +#![warn(clippy::pedantic)] +#![allow(clippy::too_many_lines)] + +use anyhow::anyhow; +use serde::Deserialize; +use std::{collections::HashMap, env, fs, path::PathBuf, process::Command}; + +#[derive(Deserialize)] +struct CargoLock<'a> { + #[serde(rename = "package", borrow)] + packages: Vec>, + metadata: Option>, +} + +#[derive(Deserialize)] +struct Package<'a> { + name: &'a str, + version: &'a str, + source: Option<&'a str>, + checksum: Option<&'a str>, +} + +#[derive(Deserialize)] +struct PrefetchOutput { + sha256: String, +} + +fn main() -> anyhow::Result<()> { + let mut hashes = HashMap::new(); + + let attr_count = env::args().len() - 1; + + for (i, attr) in env::args().skip(1).enumerate() { + println!("converting {attr} ({}/{attr_count})", i + 1); + + convert(&attr, &mut hashes)?; + } + + Ok(()) +} + +fn convert(attr: &str, hashes: &mut HashMap) -> anyhow::Result<()> { + let package_path = nix_eval(format!("{attr}.meta.position"))? + .and_then(|p| p.split_once(':').map(|(f, _)| PathBuf::from(f))); + + if package_path.is_none() { + eprintln!("can't automatically convert {attr}: doesn't exist"); + return Ok(()); + } + + let package_path = package_path.unwrap(); + + if package_path.with_file_name("Cargo.lock").exists() { + eprintln!("skipping {attr}: already has a vendored Cargo.lock"); + return Ok(()); + } + + let mut src = PathBuf::from( + String::from_utf8( + Command::new("nix-build") + .arg("-A") + .arg(format!("{attr}.src")) + .output()? + .stdout, + )? + .trim(), + ); + + if !src.exists() { + eprintln!("can't automatically convert {attr}: src doesn't exist (bad attr?)"); + return Ok(()); + } else if !src.metadata()?.is_dir() { + eprintln!("can't automatically convert {attr}: src isn't a directory"); + return Ok(()); + } + + if let Some(mut source_root) = nix_eval(format!("{attr}.sourceRoot"))?.map(PathBuf::from) { + source_root = source_root.components().skip(1).collect(); + src.push(source_root); + } + + let cargo_lock_path = src.join("Cargo.lock"); + + if !cargo_lock_path.exists() { + eprintln!("can't automatically convert {attr}: src doesn't contain Cargo.lock"); + return Ok(()); + } + + let cargo_lock_content = fs::read_to_string(cargo_lock_path)?; + + let cargo_lock: CargoLock = basic_toml::from_str(&cargo_lock_content)?; + + let mut git_dependencies = Vec::new(); + + for package in cargo_lock.packages.iter().filter(|p| { + p.source.is_some() + && p.checksum + .or_else(|| { + cargo_lock + .metadata + .as_ref()? + .get( + format!("checksum {} {} ({})", p.name, p.version, p.source.unwrap()) + .as_str(), + ) + .copied() + }) + .is_none() + }) { + let (typ, original_url) = package + .source + .unwrap() + .split_once('+') + .expect("dependency should have well-formed source url"); + + if let Some(hash) = hashes.get(original_url) { + git_dependencies.push(( + format!("{}-{}", package.name, package.version), + hash.clone(), + )); + + continue; + } + + assert_eq!( + typ, "git", + "packages without checksums should be git dependencies" + ); + + let (mut url, rev) = original_url + .split_once('#') + .expect("git dependency should have commit"); + + // TODO: improve + if let Some((u, _)) = url.split_once('?') { + url = u; + } + + let prefetch_output: PrefetchOutput = serde_json::from_slice( + &Command::new("nix-prefetch-git") + .args(["--url", url, "--rev", rev, "--quiet"]) + .output()? + .stdout, + )?; + + let output_hash = String::from_utf8( + Command::new("nix") + .args([ + "--extra-experimental-features", + "nix-command", + "hash", + "to-sri", + "--type", + "sha256", + &prefetch_output.sha256, + ]) + .output()? + .stdout, + )?; + + let hash = output_hash.trim().to_string(); + + git_dependencies.push(( + format!("{}-{}", package.name, package.version), + output_hash.trim().to_string().clone(), + )); + + hashes.insert(original_url.to_string(), hash); + } + + fs::write( + package_path.with_file_name("Cargo.lock"), + cargo_lock_content, + )?; + + let mut package_lines: Vec<_> = fs::read_to_string(&package_path)? + .lines() + .map(String::from) + .collect(); + + let (cargo_deps_line_index, cargo_deps_line) = package_lines + .iter_mut() + .enumerate() + .find(|(_, l)| { + l.trim_start().starts_with("cargoHash") || l.trim_start().starts_with("cargoSha256") + }) + .expect("package should contain cargoHash/cargoSha256"); + + let spaces = " ".repeat(cargo_deps_line.len() - cargo_deps_line.trim_start().len()); + + if git_dependencies.is_empty() { + *cargo_deps_line = format!("{spaces}cargoLock.lockFile = ./Cargo.lock;"); + } else { + *cargo_deps_line = format!("{spaces}cargoLock = {{"); + + let mut index_iter = cargo_deps_line_index + 1..; + + package_lines.insert( + index_iter.next().unwrap(), + format!("{spaces} lockFile = ./Cargo.lock;"), + ); + + package_lines.insert( + index_iter.next().unwrap(), + format!("{spaces} outputHashes = {{"), + ); + + for ((dep, hash), index) in git_dependencies.drain(..).zip(&mut index_iter) { + package_lines.insert(index, format!("{spaces} {dep:?} = {hash:?};")); + } + + package_lines.insert(index_iter.next().unwrap(), format!("{spaces} }};")); + package_lines.insert(index_iter.next().unwrap(), format!("{spaces}}};")); + } + + if package_lines.last().map(String::as_str) != Some("") { + package_lines.push(String::new()); + } + + fs::write(package_path, package_lines.join("\n"))?; + + Ok(()) +} + +fn nix_eval(attr: impl AsRef) -> anyhow::Result> { + let output = String::from_utf8( + Command::new("nix-instantiate") + .args(["--eval", "-A", attr.as_ref()]) + .output()? + .stdout, + )?; + + let trimmed = output.trim(); + + if trimmed.is_empty() || trimmed == "null" { + Ok(None) + } else { + Ok(Some( + trimmed + .strip_prefix('"') + .and_then(|p| p.strip_suffix('"')) + .ok_or_else(|| anyhow!("couldn't parse nix-instantiate output: {output:?}"))? + .to_string(), + )) + } +}