chore: Update RocksDB to 8.5.3

.github/workflows/rust.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Install rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: 1.67.1
+          toolchain: 1.71.1
           components: rustfmt
           profile: minimal
           override: true
@@ -34,7 +34,7 @@ jobs:
       - name: Install rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: 1.67.1
+          toolchain: 1.71.1
           components: clippy
           profile: minimal
           override: true
@@ -98,7 +98,7 @@ jobs:
       - name: Install rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: 1.67.1
+          toolchain: 1.71.1
           target: ${{ matrix.target }}
           profile: minimal
           override: true

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "ckb-rocksdb"
 description = "Rust wrapper for Facebook's RocksDB embeddable database"
-version = "0.20.0"
+version = "0.21.0"
 edition = "2018"
 authors = ["Tyler Neely <[email protected]>", "David Greenberg <[email protected]>", "Nervos Core Dev <[email protected]>"]
 license = "Apache-2.0"
@@ -31,12 +31,12 @@ march-native = ["librocksdb-sys/march-native"]
 
 [dependencies]
 libc = "0.2"
-librocksdb-sys = { package = "ckb-librocksdb-sys", path = "librocksdb-sys", version = "=8.1.1" }
+librocksdb-sys = { package = "ckb-librocksdb-sys", path = "librocksdb-sys", version = "=8.5.3" }
 tempfile = "3"
 
 [dev-dependencies]
 trybuild = "1.0"
-criterion = "0.4"
+criterion = "0.5"
 
 [[bench]]
 name = "transaction"

librocksdb-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ckb-librocksdb-sys"
-version = "8.1.1"
+version = "8.5.3"
 edition = "2018"
 authors = ["Karl Hobley <[email protected]>", "Arkadiy Paronyan <[email protected]>", "Nervos Core Dev <[email protected]>"]
 license = "MIT/Apache-2.0/BSD-3-Clause"
@@ -40,15 +40,15 @@ bzip2 = []
 
 [dependencies]
 libc = "0.2"
-pkg-config = "0.3.25"
+pkg-config = "0.3"
 
 [dev-dependencies]
 const-cstr = "0.3"
 uuid = { version = "1.0", features = ["v4"] }
 
 [build-dependencies]
 cc = { version = "1.0", features = ["parallel"] }
-bindgen = "0.63.0"
-glob = "0.3.0"
+bindgen = "0.68.1"
+glob = "0.3.1"
 pkg-config = "0.3"
-rust-ini = "0.18"
+rust-ini = "0.19"

librocksdb-sys/build_version.cc

@@ -8,9 +8,18 @@
 
 // The build script may replace these values with real values based
 // on whether or not GIT is available and the platform settings
-static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:6a436150417120a3f9732d65a2a5c2b8d19b60fc";
-static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v8.1.1";
-static const std::string rocksdb_build_date = "rocksdb_build_date:2023-04-06 09:38:52";
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:f32521662acf3352397d438b732144c7813bbbec";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v8.5.3";
+#define HAS_GIT_CHANGES 0
+#if HAS_GIT_CHANGES == 0
+// If HAS_GIT_CHANGES is 0, the GIT date is used.
+// Use the time the branch/tag was last modified
+static const std::string rocksdb_build_date = "rocksdb_build_date:2023-09-01 13:58:39";
+#else
+// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
+// Use the time the build was created.
+static const std::string rocksdb_build_date = "rocksdb_build_date:2023-09-21 10:21:22";
+#endif
 
 extern "C" {
 

librocksdb-sys/rocksdb_lib_sources.txt

@@ -249,6 +249,8 @@ util/stderr_logger.cc
 util/string_util.cc
 util/thread_local.cc
 util/threadpool_imp.cc
+util/udt_util.cc
+util/write_batch_util.cc
 util/xxhash.cc
 utilities/agg_merge/agg_merge.cc
 utilities/backup/backup_engine.cc

librocksdb-sys/tests/ffi.rs

@@ -1117,7 +1117,7 @@ fn ffi() {
                 rocksdb_slicetransform_create_fixed_prefix(3),
             );
             rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
-            rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
+            rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16, 0, 0, 0, 0);
             rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
 
             db = rocksdb_open(options, dbname, &mut err);

rust-toolchain

@@ -1 +1 @@
-1.67.1
+1.71.1

src/checkpoint.rs

@@ -15,7 +15,7 @@
 
 use crate::ffi;
 use crate::ops::*;
-///! Implementation of bindings to RocksDB Checkpoint[1] API
+/// Implementation of bindings to RocksDB Checkpoint[1] API
 ///
 /// [1]: https://github.com/facebook/rocksdb/wiki/Checkpoints
 use crate::{Error, DB};

src/db_options.rs

@@ -2466,14 +2466,18 @@ impl Options {
     /// # Examples
     ///
     /// ```
-    /// use ckb_rocksdb::{Options, PlainTableFactoryOptions};
+    /// use ckb_rocksdb::{KeyEncodingType, Options, PlainTableFactoryOptions};
     ///
     /// let mut opts = Options::default();
     /// let factory_opts = PlainTableFactoryOptions {
     ///   user_key_length: 0,
     ///   bloom_bits_per_key: 20,
     ///   hash_table_ratio: 0.75,
     ///   index_sparseness: 16,
+    ///   huge_page_tlb_size: 0,
+    ///   encoding_type: KeyEncodingType::Plain,
+    ///   full_scan_mode: false,
+    ///   store_index_in_file: false,
     /// };
     ///
     /// opts.set_plain_table_factory(&factory_opts);
@@ -2486,6 +2490,10 @@ impl Options {
                 options.bloom_bits_per_key,
                 options.hash_table_ratio,
                 options.index_sparseness,
+                options.huge_page_tlb_size,
+                options.encoding_type as c_char,
+                c_uchar::from(options.full_scan_mode),
+                c_uchar::from(options.store_index_in_file),
             );
         }
     }
@@ -2569,7 +2577,7 @@ impl Options {
 
             // Must have valid UTF-8 format.
             let s = CStr::from_ptr(value).to_str().unwrap().to_owned();
-            libc::free(value as *mut c_void);
+            ffi::rocksdb_free(value as *mut c_void);
             Some(s)
         }
     }
@@ -3447,6 +3455,16 @@ pub enum MemtableFactory {
     },
 }
 
+/// Used in [`PlainTableFactoryOptions`].
+#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)]
+pub enum KeyEncodingType {
+    /// Always write full keys.
+    #[default]
+    Plain = 0,
+    /// Find opportunities to write the same prefix for multiple rows.
+    Prefix = 1,
+}
+
 /// Used with DBOptions::set_plain_table_factory.
 /// See https://github.com/facebook/rocksdb/wiki/PlainTable-Format.
 ///
@@ -3455,11 +3473,19 @@ pub enum MemtableFactory {
 ///  bloom_bits_per_key: 10
 ///  hash_table_ratio: 0.75
 ///  index_sparseness: 16
+///  huge_page_tlb_size: 0
+///  encoding_type: KeyEncodingType::Plain
+///  full_scan_mode: false
+///  store_index_in_file: false
 pub struct PlainTableFactoryOptions {
     pub user_key_length: u32,
     pub bloom_bits_per_key: i32,
     pub hash_table_ratio: f64,
     pub index_sparseness: usize,
+    pub huge_page_tlb_size: usize,
+    pub encoding_type: KeyEncodingType,
+    pub full_scan_mode: bool,
+    pub store_index_in_file: bool,
 }
 
 #[derive(Debug, Copy, Clone, PartialEq)]

src/db_vector.rs

@@ -46,7 +46,7 @@ impl AsRef<[u8]> for DBVector {
 impl Drop for DBVector {
     fn drop(&mut self) {
         unsafe {
-            libc::free(self.base as *mut c_void);
+            ffi::rocksdb_free(self.base as *mut c_void);
         }
     }
 }

src/ffi_util.rs

@@ -25,7 +25,7 @@ pub fn error_message(ptr: *const c_char) -> String {
     let cstr = unsafe { CStr::from_ptr(ptr as *const _) };
     let s = String::from_utf8_lossy(cstr.to_bytes()).into_owned();
     unsafe {
-        libc::free(ptr as *mut c_void);
+        ffi::rocksdb_free(ptr as *mut c_void);
     }
     s
 }

src/lib.rs

@@ -103,9 +103,9 @@ pub use crate::db_iterator::{DBIterator, DBRawIterator, Direction, IteratorMode}
 pub use crate::db_options::{
     BlockBasedIndexType, BlockBasedOptions, BottommostLevelCompaction, Cache, CompactOptions,
     CuckooTableOptions, DBCompactionStyle, DBCompressionType, DBPath, DBRecoveryMode,
-    DataBlockIndexType, Env, FifoCompactOptions, FlushOptions, IngestExternalFileOptions, LogLevel,
-    MemtableFactory, Options, PlainTableFactoryOptions, ReadOptions, UniversalCompactOptions,
-    UniversalCompactionStopStyle, WriteOptions,
+    DataBlockIndexType, Env, FifoCompactOptions, FlushOptions, IngestExternalFileOptions,
+    KeyEncodingType, LogLevel, MemtableFactory, Options, PlainTableFactoryOptions, ReadOptions,
+    UniversalCompactOptions, UniversalCompactionStopStyle, WriteOptions,
 };
 pub use crate::db_pinnable_slice::DBPinnableSlice;
 pub use crate::db_vector::DBVector;

src/ops/multi_get.rs

@@ -191,40 +191,40 @@ where
 }
 
 pub trait BatchedMultiGetCF<R> {
-    fn batched_multi_get_cf_full<K, I>(
+    fn batched_multi_get_cf_full<'a, K, I>(
         &self,
         cf: &ColumnFamily,
         keys: I,
         sorted_input: bool,
         readopts: Option<&R>,
     ) -> Vec<Result<Option<DBPinnableSlice>, Error>>
     where
-        K: AsRef<[u8]>,
-        I: IntoIterator<Item = K>;
+        K: AsRef<[u8]> + 'a + ?Sized,
+        I: IntoIterator<Item = &'a K>;
 
-    fn batched_multi_get_cf<K, I>(
+    fn batched_multi_get_cf<'a, K, I>(
         &self,
         cf: &ColumnFamily,
         keys: I,
         sorted_input: bool,
     ) -> Vec<Result<Option<DBPinnableSlice>, Error>>
     where
-        K: AsRef<[u8]>,
-        I: IntoIterator<Item = K>,
+        K: AsRef<[u8]> + 'a + ?Sized,
+        I: IntoIterator<Item = &'a K>,
     {
         self.batched_multi_get_cf_full(cf, keys, sorted_input, None)
     }
 
-    fn batched_multi_get_cf_opt<K, I>(
+    fn batched_multi_get_cf_opt<'a, K, I>(
         &self,
         cf: &ColumnFamily,
         keys: I,
         sorted_input: bool,
         readopts: &R,
     ) -> Vec<Result<Option<DBPinnableSlice>, Error>>
     where
-        K: AsRef<[u8]>,
-        I: IntoIterator<Item = K>,
+        K: AsRef<[u8]> + 'a + ?Sized,
+        I: IntoIterator<Item = &'a K>,
     {
         self.batched_multi_get_cf_full(cf, keys, sorted_input, Some(readopts))
     }
@@ -234,16 +234,16 @@ impl<T> BatchedMultiGetCF<ReadOptions> for T
 where
     T: Handle<ffi::rocksdb_t> + super::Read,
 {
-    fn batched_multi_get_cf_full<K, I>(
+    fn batched_multi_get_cf_full<'a, K, I>(
         &self,
         cf: &ColumnFamily,
         keys: I,
         sorted_input: bool,
         readopts: Option<&ReadOptions>,
     ) -> Vec<Result<Option<DBPinnableSlice>, Error>>
     where
-        K: AsRef<[u8]>,
-        I: IntoIterator<Item = K>,
+        K: AsRef<[u8]> + 'a + ?Sized,
+        I: IntoIterator<Item = &'a K>,
     {
         let mut default_readopts = None;
         let ro_handle = match ReadOptions::input_or_default(readopts, &mut default_readopts) {
@@ -258,11 +258,13 @@ where
             }
         };
 
-        let (keys, keys_sizes): (Vec<Box<[u8]>>, Vec<_>) = keys
+        let (ptr_keys, keys_sizes): (Vec<_>, Vec<_>) = keys
             .into_iter()
-            .map(|k| (Box::from(k.as_ref()), k.as_ref().len()))
+            .map(|k| {
+                let k = k.as_ref();
+                (k.as_ptr() as *const c_char, k.len())
+            })
             .unzip();
-        let ptr_keys: Vec<_> = keys.iter().map(|k| k.as_ptr() as *const c_char).collect();
 
         let mut pinned_values = vec![ptr::null_mut(); ptr_keys.len()];
         let mut errors = vec![ptr::null_mut(); ptr_keys.len()];

src/ops/property.rs

@@ -118,7 +118,7 @@ fn property_value_impl<R>(
         ))),
     };
     unsafe {
-        libc::free(value as *mut c_void);
+        ffi::rocksdb_free(value as *mut c_void);
     }
     result
 }

tests/fail/checkpoint_outlive_db.stderr

@@ -1,9 +1,10 @@
 error[E0597]: `db` does not live long enough
-  --> $DIR/checkpoint_outlive_db.rs:9:25
+  --> tests/fail/checkpoint_outlive_db.rs:9:25
    |
 7  |     let _checkpoint = {
    |         ----------- borrow later stored here
 8  |         let db = DB::open_default("foo").unwrap();
+   |             -- binding `db` declared here
 9  |         Checkpoint::new(&db)
    |                         ^^^ borrowed value does not live long enough
 10 |     };

tests/fail/iterator/db.stderr

@@ -1,9 +1,10 @@
 error[E0597]: `db` does not live long enough
-  --> $DIR/db.rs:9:9
+  --> tests/fail/iterator/db.rs:9:9
    |
 7  |     let _iter = {
    |         ----- borrow later stored here
 8  |         let db = DB::open_default("foo").unwrap();
+   |             -- binding `db` declared here
 9  |         db.iterator(IteratorMode::Start)
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ borrowed value does not live long enough
 10 |     };

tests/fail/iterator/optimistic_transaction_db.stderr

@@ -1,9 +1,10 @@
 error[E0597]: `db` does not live long enough
-  --> $DIR/optimistic_transaction_db.rs:9:9
+  --> tests/fail/iterator/optimistic_transaction_db.rs:9:9
    |
 7  |     let _iter = {
    |         ----- borrow later stored here
 8  |         let db = OptimisticTransactionDB::open_default("foo").unwrap();
+   |             -- binding `db` declared here
 9  |         db.iterator(IteratorMode::Start)
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ borrowed value does not live long enough
 10 |     };