feat(iceberg): bump iceberg-rust #19206
Conversation
Cargo.toml
Outdated
| parquet = { version = "53", features = ["async"] } | ||
| parquet = { version = "53.2", features = ["async"] } |
There was a problem hiding this comment.
why to upgrade parquet? It's ok to me though.
There was a problem hiding this comment.
Because the upstream iceberg-rust upgrades it, so we do.
There was a problem hiding this comment.
There was a problem hiding this comment.
But the actual version needs to be 53.2. see Cargo.lock
There was a problem hiding this comment.
I'm not sure what you mean. If we change back to 53 in cargo.toml, it's possible to revert Cargo.lock.
Not suggest to revert the upgrade. Just FYI:
> cargo update -p [email protected] --precise 53.0.0
Updating crates.io index
Removing brotli v7.0.0
Downgrading parquet v53.2.0 -> v53.0.0
diff --git a/Cargo.toml b/Cargo.toml
index fb1c838f44..7c77a043dd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -161,7 +161,7 @@ deltalake = { version = "0.20.1", features = [
itertools = "0.13.0"
jsonbb = "0.1.4"
lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "2682b85" }
-parquet = { version = "53.2", features = ["async"] }
+parquet = { version = "53", features = ["async"] }
thiserror-ext = "0.1.2"
tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" }
tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [
diff --git a/Cargo.lock b/Cargo.lock
index 12fc55f45a..e23bffc0c9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2240,17 +2240,6 @@ dependencies = [
"brotli-decompressor",
]
-[[package]]
-name = "brotli"
-version = "7.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor",
-]
-
[[package]]
name = "brotli-decompressor"
version = "4.0.1"
@@ -6320,7 +6309,7 @@ dependencies = [
"once_cell",
"opendal 0.50.1",
"ordered-float 4.1.1",
- "parquet 53.2.0",
+ "parquet 53.0.0",
"paste",
"rand",
"reqwest 0.12.4",
@@ -6405,7 +6394,7 @@ dependencies = [
"once_cell",
"opendal 0.49.2",
"ordered-float 3.9.1",
- "parquet 53.2.0",
+ "parquet 53.0.0",
"prometheus",
"regex",
"reqwest 0.11.20",
@@ -8792,7 +8781,7 @@ dependencies = [
"arrow-schema 52.2.0",
"arrow-select 52.2.0",
"base64 0.22.0",
- "brotli 6.0.0",
+ "brotli",
"bytes",
"chrono",
"flate2",
@@ -8815,9 +8804,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "53.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e"
+checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8"
dependencies = [
"ahash 0.8.11",
"arrow-array 53.2.0",
@@ -8828,7 +8817,7 @@ dependencies = [
"arrow-schema 53.2.0",
"arrow-select 53.2.0",
"base64 0.22.0",
- "brotli 7.0.0",
+ "brotli",
"bytes",
"chrono",
"flate2",
@@ -9851,7 +9840,7 @@ dependencies = [
"indoc",
"libc",
"memoffset",
- "parking_lot 0.12.1",
+ "parking_lot 0.11.2",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
@@ -10549,7 +10538,7 @@ dependencies = [
"memcomparable",
"opendal 0.49.2",
"parking_lot 0.12.1",
- "parquet 53.2.0",
+ "parquet 53.0.0",
"paste",
"prometheus",
"prost 0.13.1",
@@ -11084,7 +11073,7 @@ dependencies = [
"opensearch",
"openssl",
"parking_lot 0.12.1",
- "parquet 53.2.0",
+ "parquet 53.0.0",
"paste",
"pg_bigdecimal",
"postgres-openssl",There was a problem hiding this comment.
I mean iceberg-rust uses some structs that are newly added in parquet 52.3, but these structs don't exist in 52.0
There was a problem hiding this comment.
If we use 53.0.0, we will have a compile error.
error[E0432]: unresolved import
parquet::file::metadata::ParquetMetaDataReader
--> /Users/dylan/.cargo/git/checkouts/iceberg-rust-0821e0c4f5c01b19-shallow/13a561f/crates/iceberg/src/arrow/reader.rs:38:48
|
38 | use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
| ^^^^^^^^^^^^^^^^^^^^^
| |
| noParquetMetaDataReaderinfile::metadata
| help: a similar name exists in the module:ParquetMetaDataWriter
There was a problem hiding this comment.
😄 interesting. This sounds like a bug of iceberg-rust. They wrongly specify the minimal version
|
@xxhZs It seems iceberg-rust has some breaking change that will cause position delete can't work. Please take a look together. |
I hereby agree to the terms of the RisingWave Labs, Inc. Contributor License Agreement.
What's changed and what's your intention?
Checklist
./risedev check(or alias,./risedev c)Documentation
Release note
If this PR includes changes that directly affect users or other significant modifications relevant to the community, kindly draft a release note to provide a concise summary of these changes. Please prioritize highlighting the impact these changes will have on users.