diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..ca9746e9a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +zcash_client_backend/src/proto/compact_formats.rs linguist-generated=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91ba4f2bd1..2e53292958 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: command: test args: --all-features --verbose --release --all -- --ignored + - name: Verify working directory is clean + run: git diff --exit-code + build: name: Build target ${{ matrix.target }} runs-on: ubuntu-latest @@ -200,14 +203,6 @@ jobs: toolchain: 1.56.1 override: true - # cargo fmt does not build the code, and running it in a fresh clone of - # the codebase will fail because the protobuf code has not been generated. - - name: cargo build - uses: actions-rs/cargo@v1 - with: - command: build - args: --all - # Ensure all code has been formatted with rustfmt - run: rustup component add rustfmt - name: Check formatting diff --git a/zcash_client_backend/.gitignore b/zcash_client_backend/.gitignore deleted file mode 100644 index 7025829d91..0000000000 --- a/zcash_client_backend/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Protobufs -src/proto/ diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 21f90ef80c..b21a08975a 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -41,6 +41,7 @@ and this library adheres to Rust's notion of - `WalletWrite::remove_unmined_tx` (behind the `unstable` feature flag). - `WalletWrite::get_next_available_address` - `WalletWrite::put_received_transparent_utxo` + - `impl From for error::Error` - `zcash_client_backend::decrypt`: - `TransferType` - `zcash_client_backend::proto`: @@ -68,6 +69,8 @@ and this library adheres to Rust's notion of - Bumped dependencies to `ff 0.12`, `group 0.12`, `bls12_381 0.7`, `jubjub 0.9`, `zcash_primitives 0.8`, `orchard 0.3`. - `zcash_client_backend::proto`: + - The Protocol Buffers bindings are now generated for `prost 0.11` instead of + `protobuf 2`. - `compact_formats::CompactSpend` has been renamed to `CompactSaplingSpend`, and its `epk` field (and associated `set_epk` method) has been renamed to `ephemeralKey` (and `set_ephemeralKey`). @@ -104,6 +107,8 @@ and this library adheres to Rust's notion of `store_decrypted_tx`. - `data_api::ReceivedTransaction` has been renamed to `DecryptedTransaction`, and its `outputs` field has been renamed to `sapling_outputs`. +- `data_api::error::Error::Protobuf` now wraps `prost::DecodeError` instead of + `protobuf::ProtobufError`. - `data_api::error::Error` has the following additional cases: - `Error::MemoForbidden` to report the condition where a memo was specified to be sent to a transparent recipient. @@ -150,6 +155,12 @@ and this library adheres to Rust's notion of `WalletRead::get_unified_full_viewing_keys` instead). - `WalletRead::get_address` (use `WalletRead::get_current_address` or `WalletWrite::get_next_available_address` instead.) + - `impl From for error::Error` +- `zcash_client_backend::proto::compact_formats`: + - `Compact*::new` methods (use `Default::default` or struct instantiation + instead). + - Getters (use dedicated typed methods or direct field access instead). + - Setters (use direct field access instead). - The hardcoded `data_api::wallet::ANCHOR_OFFSET` constant. - `zcash_client_backend::wallet::AccountId` (moved to `zcash_primitives::zip32::AccountId`). diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 78cd3685cd..f3c414260b 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -13,6 +13,9 @@ license = "MIT OR Apache-2.0" edition = "2021" rust-version = "1.56.1" +# Exclude proto files so crates.io consumers don't need protoc. +exclude = ["*.proto"] + [package.metadata.cargo-udeps.ignore] development = ["zcash_proofs"] @@ -40,7 +43,7 @@ memuse = "0.2" tracing = "0.1" # - Protobuf interfaces -protobuf = "~2.27.1" # MSRV 1.52.1 +prost = "0.11" # - Secret management secrecy = "0.8" @@ -68,7 +71,8 @@ crossbeam-channel = "0.5" rayon = "1.5" [build-dependencies] -protobuf-codegen-pure = "~2.27.1" # MSRV 1.52.1 +prost-build = "0.11" +which = "4" [dev-dependencies] gumdrop = "0.8" diff --git a/zcash_client_backend/build.rs b/zcash_client_backend/build.rs index 65b9417eba..c12d9dbac5 100644 --- a/zcash_client_backend/build.rs +++ b/zcash_client_backend/build.rs @@ -1,8 +1,43 @@ -fn main() { - protobuf_codegen_pure::Codegen::new() - .out_dir("src/proto") - .inputs(["proto/compact_formats.proto"]) - .includes(["proto"]) - .run() - .expect("Protobuf codegen failed"); +use std::env; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +const COMPACT_FORMATS_PROTO: &str = "proto/compact_formats.proto"; + +fn main() -> io::Result<()> { + // We don't include the proto files in releases so that downstreams do not need to + // regenerate the bindings even if protoc is present. + if Path::new(COMPACT_FORMATS_PROTO).exists() { + println!("cargo:rerun-if-changed={}", COMPACT_FORMATS_PROTO); + + // We check for the existence of protoc in the same way as prost_build, so that people + // building from source do not need to have protoc installed. If they make changes to + // the proto files, the discrepancy will be caught by CI. + if env::var_os("PROTOC") + .map(PathBuf::from) + .or_else(|| which::which("protoc").ok()) + .is_some() + { + build()?; + } + } + + Ok(()) +} + +fn build() -> io::Result<()> { + let out: PathBuf = env::var_os("OUT_DIR") + .expect("Cannot find OUT_DIR environment variable") + .into(); + + prost_build::compile_protos(&[COMPACT_FORMATS_PROTO], &["proto/"])?; + + // Copy the generated files into the source tree so changes can be committed. + fs::copy( + out.join("cash.z.wallet.sdk.rpc.rs"), + "src/proto/compact_formats.rs", + )?; + + Ok(()) } diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 55913fb3ee..e6ac0bd5f6 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -69,7 +69,7 @@ pub enum Error { Builder(builder::Error), /// An error occurred decoding a protobuf message. - Protobuf(protobuf::ProtobufError), + Protobuf(prost::DecodeError), /// The wallet attempted a sapling-only operation at a block /// height when Sapling was not yet active. @@ -180,8 +180,8 @@ impl From for Error { } } -impl From for Error { - fn from(e: protobuf::ProtobufError) -> Self { +impl From for Error { + fn from(e: prost::DecodeError) -> Self { Error::Protobuf(e) } } diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index a1eb43dbc9..aa129a099d 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -11,6 +11,7 @@ use zcash_primitives::{ use zcash_note_encryption::{EphemeralKeyBytes, COMPACT_NOTE_SIZE}; +#[rustfmt::skip] pub mod compact_formats; impl compact_formats::CompactBlock { @@ -44,7 +45,7 @@ impl compact_formats::CompactBlock { if let Some(header) = self.header() { header.prev_block } else { - BlockHash::from_slice(&self.prevHash) + BlockHash::from_slice(&self.prev_hash) } } @@ -99,7 +100,7 @@ impl compact_formats::CompactSaplingOutput { /// /// [`CompactOutput.epk`]: #structfield.epk pub fn ephemeral_key(&self) -> Result { - self.ephemeralKey[..] + self.ephemeral_key[..] .try_into() .map(EphemeralKeyBytes) .map_err(|_| ()) @@ -110,11 +111,11 @@ impl From> for compact_formats::CompactSaplingOutput { fn from(out: sapling::OutputDescription) -> compact_formats::CompactSaplingOutput { - let mut result = compact_formats::CompactSaplingOutput::new(); - result.set_cmu(out.cmu.to_repr().to_vec()); - result.set_ephemeralKey(out.ephemeral_key.as_ref().to_vec()); - result.set_ciphertext(out.enc_ciphertext[..COMPACT_NOTE_SIZE].to_vec()); - result + compact_formats::CompactSaplingOutput { + cmu: out.cmu.to_repr().to_vec(), + ephemeral_key: out.ephemeral_key.as_ref().to_vec(), + ciphertext: out.enc_ciphertext[..COMPACT_NOTE_SIZE].to_vec(), + } } } diff --git a/zcash_client_backend/src/proto/.keep b/zcash_client_backend/src/proto/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs new file mode 100644 index 0000000000..451e115ceb --- /dev/null +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -0,0 +1,102 @@ +// Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. +// bytes fields of hashes are in canonical little-endian format. + +/// CompactBlock is a packaging of ONLY the data from a block that's needed to: +/// 1. Detect a payment to your shielded Sapling address +/// 2. Detect a spend of your shielded Sapling notes +/// 3. Update your witnesses to generate new Sapling spend proofs. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactBlock { + /// the version of this wire format, for storage + #[prost(uint32, tag="1")] + pub proto_version: u32, + /// the height of this block + #[prost(uint64, tag="2")] + pub height: u64, + /// the ID (hash) of this block, same as in block explorers + #[prost(bytes="vec", tag="3")] + pub hash: ::prost::alloc::vec::Vec, + /// the ID (hash) of this block's predecessor + #[prost(bytes="vec", tag="4")] + pub prev_hash: ::prost::alloc::vec::Vec, + /// Unix epoch time when the block was mined + #[prost(uint32, tag="5")] + pub time: u32, + /// (hash, prevHash, and time) OR (full header) + #[prost(bytes="vec", tag="6")] + pub header: ::prost::alloc::vec::Vec, + /// zero or more compact transactions from this block + #[prost(message, repeated, tag="7")] + pub vtx: ::prost::alloc::vec::Vec, +} +/// CompactTx contains the minimum information for a wallet to know if this transaction +/// is relevant to it (either pays to it or spends from it) via shielded elements +/// only. This message will not encode a transparent-to-transparent transaction. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactTx { + /// Index and hash will allow the receiver to call out to chain + /// explorers or other data structures to retrieve more information + /// about this transaction. + /// + /// the index within the full block + #[prost(uint64, tag="1")] + pub index: u64, + /// the ID (hash) of this transaction, same as in block explorers + #[prost(bytes="vec", tag="2")] + pub hash: ::prost::alloc::vec::Vec, + /// The transaction fee: present if server can provide. In the case of a + /// stateless server and a transaction with transparent inputs, this will be + /// unset because the calculation requires reference to prior transactions. + /// If there are no transparent inputs, the fee will be calculable as: + /// valueBalanceSapling + valueBalanceOrchard + sum(vPubNew) - sum(vPubOld) - sum(tOut) + #[prost(uint32, tag="3")] + pub fee: u32, + #[prost(message, repeated, tag="4")] + pub spends: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="5")] + pub outputs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="6")] + pub actions: ::prost::alloc::vec::Vec, +} +/// CompactSaplingSpend is a Sapling Spend Description as described in 7.3 of the Zcash +/// protocol specification. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactSaplingSpend { + /// nullifier (see the Zcash protocol specification) + #[prost(bytes="vec", tag="1")] + pub nf: ::prost::alloc::vec::Vec, +} +/// output encodes the `cmu` field, `ephemeralKey` field, and a 52-byte prefix of the +/// `encCiphertext` field of a Sapling Output Description. These fields are described in +/// section 7.4 of the Zcash protocol spec: +/// +/// Total size is 116 bytes. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactSaplingOutput { + /// note commitment u-coordinate + #[prost(bytes="vec", tag="1")] + pub cmu: ::prost::alloc::vec::Vec, + /// ephemeral public key + #[prost(bytes="vec", tag="2")] + pub ephemeral_key: ::prost::alloc::vec::Vec, + /// first 52 bytes of ciphertext + #[prost(bytes="vec", tag="3")] + pub ciphertext: ::prost::alloc::vec::Vec, +} +/// +/// (but not all fields are needed) +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompactOrchardAction { + /// \[32\] The nullifier of the input note + #[prost(bytes="vec", tag="1")] + pub nullifier: ::prost::alloc::vec::Vec, + /// \[32\] The x-coordinate of the note commitment for the output note + #[prost(bytes="vec", tag="2")] + pub cmx: ::prost::alloc::vec::Vec, + /// \[32\] An encoding of an ephemeral Pallas public key + #[prost(bytes="vec", tag="3")] + pub ephemeral_key: ::prost::alloc::vec::Vec, + /// \[52\] The first 52 bytes of the encCiphertext field + #[prost(bytes="vec", tag="4")] + pub ciphertext: ::prost::alloc::vec::Vec, +} diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 9d64b3e441..09cf89fbac 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -459,16 +459,16 @@ mod tests { let fake_epk = SPENDING_KEY_GENERATOR * fake_esk; fake_epk.to_bytes().to_vec() }; - let mut cspend = CompactSaplingSpend::new(); - cspend.set_nf(fake_nf); - let mut cout = CompactSaplingOutput::new(); - cout.set_cmu(fake_cmu); - cout.set_ephemeralKey(fake_epk); - cout.set_ciphertext(vec![0; 52]); - let mut ctx = CompactTx::new(); + let cspend = CompactSaplingSpend { nf: fake_nf }; + let cout = CompactSaplingOutput { + cmu: fake_cmu, + ephemeral_key: fake_epk, + ciphertext: vec![0; 52], + }; + let mut ctx = CompactTx::default(); let mut txid = vec![0; 32]; rng.fill_bytes(&mut txid); - ctx.set_hash(txid); + ctx.hash = txid; ctx.spends.push(cspend); ctx.outputs.push(cout); ctx @@ -503,17 +503,19 @@ mod tests { &mut rng, ); let cmu = note.cmu().to_repr().as_ref().to_owned(); - let epk = encryptor.epk().to_bytes().to_vec(); + let ephemeral_key = encryptor.epk().to_bytes().to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); // Create a fake CompactBlock containing the note - let mut cb = CompactBlock::new(); - cb.set_hash({ - let mut hash = vec![0; 32]; - rng.fill_bytes(&mut hash); - hash - }); - cb.set_height(height.into()); + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + height: height.into(), + ..Default::default() + }; // Add a random Sapling tx before ours { @@ -522,16 +524,16 @@ mod tests { cb.vtx.push(tx); } - let mut cspend = CompactSaplingSpend::new(); - cspend.set_nf(nf.0.to_vec()); - let mut cout = CompactSaplingOutput::new(); - cout.set_cmu(cmu); - cout.set_ephemeralKey(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - let mut ctx = CompactTx::new(); + let cspend = CompactSaplingSpend { nf: nf.0.to_vec() }; + let cout = CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + }; + let mut ctx = CompactTx::default(); let mut txid = vec![0; 32]; rng.fill_bytes(&mut txid); - ctx.set_hash(txid); + ctx.hash = txid; ctx.spends.push(cspend); ctx.outputs.push(cout); ctx.index = cb.vtx.len() as u64; diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 742d2c4392..e17e1c2502 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -24,7 +24,7 @@ bs58 = { version = "0.4", features = ["check"] } hdwallet = { version = "0.3.1", optional = true } # - Protobuf interfaces -protobuf = "~2.27.1" # MSRV 1.52.1 +prost = "0.11" # - Secret management secrecy = "0.8" diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index ce8a4a9a49..c3f3ad0930 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -1,6 +1,6 @@ //! Functions for enforcing chain validity and handling chain reorgs. -use protobuf::Message; +use prost::Message; use rusqlite::params; use zcash_primitives::consensus::BlockHeight; @@ -14,7 +14,7 @@ use { crate::{BlockHash, FsBlockDb}, rusqlite::Connection, std::fs::File, - std::io::BufReader, + std::io::Read, std::path::{Path, PathBuf}, }; @@ -60,7 +60,7 @@ where for row_result in rows { let cbr = row_result?; - let block: CompactBlock = Message::parse_from_bytes(&cbr.data).map_err(Error::from)?; + let block = CompactBlock::decode(&cbr.data[..]).map_err(Error::from)?; if block.height() != cbr.height { return Err(SqliteClientError::CorruptedData(format!( @@ -195,11 +195,11 @@ where for row_result in rows { let cbr = row_result?; - let block_file = File::open(cbr.block_file_path(&cache.blocks_dir))?; - let mut buf_reader = BufReader::new(block_file); + let mut block_file = File::open(cbr.block_file_path(&cache.blocks_dir))?; + let mut block_data = vec![]; + block_file.read_to_end(&mut block_data)?; - let block: CompactBlock = - Message::parse_from_reader(&mut buf_reader).map_err(Error::from)?; + let block = CompactBlock::decode(&block_data[..]).map_err(Error::from)?; if block.height() != cbr.height { return Err(SqliteClientError::CorruptedData(format!( diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 4f36f10790..ec06bddfdc 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -915,7 +915,7 @@ impl BlockSource for FsBlockDb { #[allow(deprecated)] mod tests { use group::{ff::PrimeField, GroupEncoding}; - use protobuf::Message; + use prost::Message; use rand_core::{OsRng, RngCore}; use rusqlite::params; use std::collections::HashMap; @@ -1045,24 +1045,30 @@ mod tests { &mut rng, ); let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); + let ephemeral_key = encryptor.epk().to_bytes().to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); // Create a fake CompactBlock containing the note - let mut cout = CompactSaplingOutput::new(); - cout.set_cmu(cmu); - cout.set_ephemeralKey(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - let mut ctx = CompactTx::new(); + let cout = CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + }; + let mut ctx = CompactTx::default(); let mut txid = vec![0; 32]; rng.fill_bytes(&mut txid); - ctx.set_hash(txid); + ctx.hash = txid; ctx.outputs.push(cout); - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + height: height.into(), + ..Default::default() + }; + cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1081,12 +1087,11 @@ mod tests { let rseed = generate_random_rseed(&network(), height, &mut rng); // Create a fake CompactBlock containing the note - let mut cspend = CompactSaplingSpend::new(); - cspend.set_nf(nf.to_vec()); - let mut ctx = CompactTx::new(); + let cspend = CompactSaplingSpend { nf: nf.to_vec() }; + let mut ctx = CompactTx::default(); let mut txid = vec![0; 32]; rng.fill_bytes(&mut txid); - ctx.set_hash(txid); + ctx.hash = txid; ctx.spends.push(cspend); // Create a fake Note for the payment @@ -1105,14 +1110,14 @@ mod tests { &mut rng, ); let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); + let ephemeral_key = encryptor.epk().to_bytes().to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); - let mut cout = CompactSaplingOutput::new(); - cout.set_cmu(cmu); - cout.set_ephemeralKey(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout + CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + } }); // Create a fake Note for the change @@ -1133,28 +1138,33 @@ mod tests { &mut rng, ); let cmu = note.cmu().to_repr().as_ref().to_vec(); - let epk = encryptor.epk().to_bytes().to_vec(); + let ephemeral_key = encryptor.epk().to_bytes().to_vec(); let enc_ciphertext = encryptor.encrypt_note_plaintext(); - let mut cout = CompactSaplingOutput::new(); - cout.set_cmu(cmu); - cout.set_ephemeralKey(epk); - cout.set_ciphertext(enc_ciphertext.as_ref()[..52].to_vec()); - cout + CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + } }); - let mut cb = CompactBlock::new(); - cb.set_height(u64::from(height)); - cb.hash.resize(32, 0); - rng.fill_bytes(&mut cb.hash); - cb.prevHash.extend_from_slice(&prev_hash.0); + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + height: height.into(), + ..Default::default() + }; + cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); cb } /// Insert a fake CompactBlock into the cache DB. pub(crate) fn insert_into_cache(db_cache: &BlockDb, cb: &CompactBlock) { - let cb_bytes = cb.write_to_bytes().unwrap(); + let cb_bytes = cb.encode_to_vec(); db_cache .0 .prepare("INSERT INTO compactblocks (height, data) VALUES (?, ?)")