feat: measure occupied capacity

Cargo.toml

@@ -55,6 +55,7 @@ members = [
     "util/dir",
     "util/instrument",
     "util/build-info",
+    "util/occupied-capacity",
     "network",
     "protocol",
     "sync",

core/Cargo.toml

@@ -13,6 +13,7 @@ numext-fixed-hash = { version = "0.1", features = ["support_rand", "support_heap
 numext-fixed-uint = { version = "0.1", features = ["support_rand", "support_heapsize", "support_serde"] }
 hash = {path = "../util/hash"}
 crypto = {path = "../util/crypto"}
+occupied-capacity = {path = "../util/occupied-capacity"}
 bit-vec = "0.5.0"
 crossbeam-channel = "0.3"
 ckb-util = { path = "../util" }

core/src/script.rs

@@ -1,5 +1,6 @@
 use hash::sha3_256;
 use numext_fixed_hash::H256;
+use occupied_capacity::OccupiedCapacity;
 use serde_derive::{Deserialize, Serialize};
 use std::io::Write;
 use std::mem;
@@ -85,14 +86,6 @@ impl Script {
         (version, args, reference, binary, signed_args)
     }
 
-    pub fn bytes_len(&self) -> usize {
-        mem::size_of::<u8>()
-            + self.args.iter().map(|a| a.len()).sum::<usize>()
-            + self.reference.as_ref().map_or(0, |_| H256::size_of())
-            + self.binary.as_ref().map_or(0, |script| script.len())
-            + self.signed_args.iter().map(|a| a.len()).sum::<usize>()
-    }
-
     pub fn type_hash(&self) -> H256 {
         match self.version {
             0 => {
@@ -123,26 +116,12 @@ impl Script {
     }
 }
 
-// impl From<&'static str> for Script {
-// 	fn from(s: &'static str) -> Self {
-// 		Script::new(s.into())
-// 	}
-// }
-
-// impl From<Bytes> for Script {
-// 	fn from(s: Bytes) -> Self {
-// 		Script::new(s)
-// 	}
-// }
-
-// impl From<Vec<u8>> for Script {
-// 	fn from(v: Vec<u8>) -> Self {
-// 		Script::new(v.into())
-// 	}
-// }
-
-// impl From<Script> for Bytes {
-// 	fn from(script: Script) -> Self {
-// 		script.data
-// 	}
-// }
+impl OccupiedCapacity for Script {
+    fn occupied_capacity(&self) -> usize {
+        mem::size_of::<u8>()
+            + self.args.occupied_capacity()
+            + self.reference.occupied_capacity()
+            + self.binary.occupied_capacity()
+            + self.signed_args.occupied_capacity()
+    }
+}

core/src/transaction.rs

@@ -7,11 +7,12 @@ use bincode::{deserialize, serialize};
 use faster_hex::hex_string;
 use hash::sha3_256;
 use numext_fixed_hash::H256;
+use occupied_capacity::OccupiedCapacity;
 use serde_derive::{Deserialize, Serialize};
+use std::fmt;
 use std::ops::{Deref, DerefMut};
-use std::{fmt, mem};
 
-#[derive(Clone, Serialize, Deserialize, Eq, PartialEq, Hash, Debug)]
+#[derive(Clone, Serialize, Deserialize, Eq, PartialEq, Hash, Debug, OccupiedCapacity)]
 pub struct OutPoint {
     // Hash of Transaction
     pub hash: H256,
@@ -47,7 +48,7 @@ impl OutPoint {
     }
 }
 
-#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq, Hash, Debug)]
+#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq, Hash, Debug, OccupiedCapacity)]
 pub struct CellInput {
     pub previous_output: OutPoint,
     // Depends on whether the operation is Transform or Destroy, this is the proof to transform
@@ -85,7 +86,7 @@ impl CellInput {
     }
 }
 
-#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq, Hash, Debug)]
+#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq, Hash, Debug, OccupiedCapacity)]
 pub struct CellOutput {
     pub capacity: Capacity,
     pub data: Vec<u8>,
@@ -119,7 +120,7 @@ impl CellOutput {
     }
 }
 
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug, Default)]
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug, Default, OccupiedCapacity)]
 pub struct Transaction {
     version: Version,
     deps: Vec<OutPoint>,
@@ -133,15 +134,6 @@ pub struct IndexTransaction {
     pub transaction: Transaction,
 }
 
-impl CellOutput {
-    pub fn bytes_len(&self) -> usize {
-        mem::size_of::<Capacity>()
-            + self.data.len()
-            + H256::size_of()
-            + self.type_.as_ref().map_or(0, |s| s.bytes_len())
-    }
-}
-
 #[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, Default, Hash)]
 pub struct ProposalShortId([u8; 10]);
 

pool/Cargo.toml

@@ -20,6 +20,7 @@ serde_derive = "1.0"
 log = "0.4"
 fnv = "1.0.3"
 crossbeam-channel = "0.3"
+occupied-capacity = { path = "../util/occupied-capacity" }
 
 [dev-dependencies]
 env_logger = "0.6"

pool/src/txs_pool/types.rs

@@ -7,6 +7,7 @@ use ckb_core::BlockNumber;
 use ckb_verification::TransactionError;
 use fnv::{FnvHashMap, FnvHashSet};
 use linked_hash_map::LinkedHashMap;
+use occupied_capacity::OccupiedCapacity;
 use serde_derive::{Deserialize, Serialize};
 use std::collections::VecDeque;
 use std::hash::Hash;
@@ -99,26 +100,21 @@ pub struct PoolEntry {
     pub transaction: Transaction,
     /// refs count
     pub refs_count: usize,
-    /// Size estimate
-    pub size_estimate: usize,
+    /// Bytes size
+    pub bytes_size: usize,
 }
 
 impl PoolEntry {
     /// Create new transaction pool entry
     pub fn new(tx: Transaction, count: usize) -> PoolEntry {
         PoolEntry {
-            size_estimate: estimate_transaction_size(&tx),
+            bytes_size: tx.occupied_capacity(),
             transaction: tx,
             refs_count: count,
         }
     }
 }
 
-/// TODO guessing this needs implementing
-fn estimate_transaction_size(_tx: &Transaction) -> usize {
-    0
-}
-
 #[derive(Default, Debug)]
 pub struct Edges<K: Hash + Eq, V: Copy + Eq + Hash> {
     inner: FnvHashMap<K, Option<V>>,

util/occupied-capacity-derive/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "occupied-capacity-derive"
+version = "0.5.0-pre"
+authors = ["Nervos Core Dev <[email protected]>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "0.4"
+quote = "0.6"
+syn = "0.15"

util/occupied-capacity-derive/src/lib.rs

@@ -0,0 +1,98 @@
+extern crate proc_macro;
+
+use proc_macro2::TokenStream;
+use quote::{quote, quote_spanned};
+use syn::spanned::Spanned;
+use syn::{
+    parse_macro_input, parse_quote, Data, DeriveInput, Fields, GenericParam, Generics, Index,
+};
+
+#[proc_macro_derive(OccupiedCapacity)]
+pub fn derive_occupied_capacity(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    // Parse the input tokens into a syntax tree.
+    let input = parse_macro_input!(input as DeriveInput);
+
+    // Used in the quasi-quotation below as `#name`.
+    let name = input.ident;
+
+    // Add a bound `T: OccupiedCapacity` to every type parameter T.
+    let generics = add_trait_bounds(input.generics);
+    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+
+    let sum = occupied_capacity_sum(&input.data);
+
+    let expanded = quote! {
+        // The generated impl.
+        impl #impl_generics ::occupied_capacity::OccupiedCapacity for #name #ty_generics #where_clause {
+            fn occupied_capacity(&self) -> usize {
+                #sum
+            }
+        }
+    };
+
+    // Hand the output tokens back to the compiler.
+    proc_macro::TokenStream::from(expanded)
+}
+
+// Add a bound `T: OccupiedCapacity` to every type parameter T.
+fn add_trait_bounds(mut generics: Generics) -> Generics {
+    for param in &mut generics.params {
+        if let GenericParam::Type(ref mut type_param) = *param {
+            type_param
+                .bounds
+                .push(parse_quote!(::occupied_capacity::OccupiedCapacity));
+        }
+    }
+    generics
+}
+
+// Generate an expression to sum up the heap size of each field.
+fn occupied_capacity_sum(data: &Data) -> TokenStream {
+    match *data {
+        Data::Struct(ref data) => {
+            match data.fields {
+                Fields::Named(ref fields) => {
+                    // Expands to an expression like
+                    //
+                    //     0 + self.x.occupied_capacity() + self.y.occupied_capacity() + self.z.occupied_capacity()
+                    //
+                    // but using fully qualified function call syntax.
+                    //
+                    // We take some care to use the span of each `syn::Field` as
+                    // the span of the corresponding `occupied_capacity`
+                    // call. This way if one of the field types does not
+                    // implement `OccupiedCapacity` then the compiler's error message
+                    // underlines which field it is.
+                    let recurse = fields.named.iter().map(|f| {
+                        let name = &f.ident;
+                        quote_spanned! {f.span()=>
+                            ::occupied_capacity::OccupiedCapacity::occupied_capacity(&self.#name)
+                        }
+                    });
+                    quote! {
+                        0 #(+ #recurse)*
+                    }
+                }
+                Fields::Unnamed(ref fields) => {
+                    // Expands to an expression like
+                    //
+                    //     0 + self.0.occupied_capacity() + self.1.occupied_capacity() + self.2.occupied_capacity()
+                    let recurse = fields.unnamed.iter().enumerate().map(|(i, f)| {
+                        let index = Index::from(i);
+                        quote_spanned! {f.span()=>
+                            ::occupied_capacity::OccupiedCapacity::occupied_capacity(&self.#index)
+                        }
+                    });
+                    quote! {
+                        0 #(+ #recurse)*
+                    }
+                }
+                Fields::Unit => {
+                    // Unit structs cannot own more than 0 bytes of heap memory.
+                    quote!(0)
+                }
+            }
+        }
+        Data::Enum(_) | Data::Union(_) => unimplemented!(),
+    }
+}

util/occupied-capacity/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "occupied-capacity"
+version = "0.5.0-pre"
+authors = ["Nervos Core Dev <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+numext-fixed-hash = { version = "0.1", features = ["support_rand", "support_heapsize", "support_serde"] }
+occupied-capacity-derive = { path = "../occupied-capacity-derive" }