From 70d70095acb615efad5d5747089f3ebabc92163e Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Thu, 23 Feb 2023 16:24:02 +0000 Subject: [PATCH 1/2] Update arrow 34 --- Cargo.toml | 7 +++ datafusion-cli/Cargo.lock | 45 +++++++------------ datafusion-cli/Cargo.toml | 6 +++ .../core/src/physical_plan/sorts/sort.rs | 2 +- datafusion/proto/Cargo.toml | 2 +- 5 files changed, 30 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 96bbc377eeee..f7e09e2e46f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,3 +51,10 @@ opt-level = 3 overflow-checks = false panic = 'unwind' rpath = false + +[patch.crates-io] +arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 4b3f57d6dd0d..1397d25c66e2 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -69,8 +69,7 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3724c874f1517cf898cd1c3ad18ab5071edf893c48e73139ab1e16cf0f2affe" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "ahash", "arrow-arith", @@ -92,8 +91,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958823b8383ca14d0a2e973de478dd7674cd9f72837f8c41c132a0fda6a4e5e" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -107,8 +105,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db670eab50e76654065b5aed930f4367101fcddcb2223802007d1e0b4d5a2579" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "ahash", "arrow-buffer", @@ -123,8 +120,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f0e01c931882448c0407bd32311a624b9f099739e94e786af68adc97016b5f2" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "half", "num", @@ -133,8 +129,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf35d78836c93f80d9362f3ccb47ff5e2c5ecfc270ff42cdf1ef80334961d44" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -149,8 +144,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6aa7c2531d89d01fed8c469a9b1bf97132a0bdf70b4724fe4bbb4537a50880" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -168,8 +162,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea50db4d1e1e4c2da2bfdea7b6d2722eef64267d5ab680d815f7ae42428057f5" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-buffer", "arrow-schema", @@ -180,8 +173,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4042fe6585155d1ec28a8e4937ec901a3ca7a19a22b9f6cd3f551b935cd84f5" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -194,8 +186,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c907c4ab4f26970a3719dc06e78e8054a01d0c96da3664d23b941e201b33d2b" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -213,8 +204,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e131b447242a32129efc7932f58ed8931b42f35d8701c1a08f9f524da13b1d3c" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -227,8 +217,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b591ef70d76f4ac28dd7666093295fece0e5f9298f49af51ea49c001e1635bb6" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "ahash", "arrow-array", @@ -242,14 +231,12 @@ dependencies = [ [[package]] name = "arrow-schema" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb327717d87eb94be5eff3b0cb8987f54059d343ee5235abf7f143c85f54cfc8" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" [[package]] name = "arrow-select" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d3c389d1cea86793934f31594f914c8547d82e91e3411d4833ad0aac3266a7" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -261,8 +248,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee67790496dd310ddbf5096870324431e89aa76453e010020ac29b1184d356" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -1774,8 +1760,7 @@ dependencies = [ [[package]] name = "parquet" version = "33.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b076829801167d889795cd1957989055543430fa1469cb1f6e32b789bfc764" +source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" dependencies = [ "ahash", "arrow-array", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index d5cdc5e641fd..b2c92f10ea4d 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -41,3 +41,9 @@ parking_lot = { version = "0.12" } rustyline = "10.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] } url = "2.2" + +[patch.crates-io] +arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } diff --git a/datafusion/core/src/physical_plan/sorts/sort.rs b/datafusion/core/src/physical_plan/sorts/sort.rs index 9bacf9fad6ff..c3fc06206ca1 100644 --- a/datafusion/core/src/physical_plan/sorts/sort.rs +++ b/datafusion/core/src/physical_plan/sorts/sort.rs @@ -1054,7 +1054,7 @@ mod tests { #[tokio::test] async fn test_sort_fetch_memory_calculation() -> Result<()> { // This test mirrors down the size from the example above. - let avg_batch_size = 5336; + let avg_batch_size = 6000; let partitions = 4; // A tuple of (fetch, expect_spillage) diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 6746325abc99..9d39eb592fbe 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -60,4 +60,4 @@ tokio = "1.18" [build-dependencies] # Pin these dependencies so that the generated output is deterministic pbjson-build = { version = "=0.5.1" } -prost-build = { version = "=0.11.6" } +prost-build = { version = "=0.11.7" } From e50e3fcf30b6672ae260c717d766213e37c508e2 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Mon, 27 Feb 2023 12:05:42 +0000 Subject: [PATCH 2/2] Remove pin --- Cargo.toml | 7 --- benchmarks/Cargo.toml | 4 +- datafusion-cli/Cargo.lock | 75 +++++++++++++++++------------ datafusion-cli/Cargo.toml | 8 +-- datafusion-examples/Cargo.toml | 6 +-- datafusion/common/Cargo.toml | 4 +- datafusion/core/Cargo.toml | 4 +- datafusion/expr/Cargo.toml | 2 +- datafusion/jit/Cargo.toml | 2 +- datafusion/optimizer/Cargo.toml | 2 +- datafusion/physical-expr/Cargo.toml | 6 +-- datafusion/proto/Cargo.toml | 2 +- datafusion/row/Cargo.toml | 2 +- datafusion/sql/Cargo.toml | 2 +- parquet-test-utils/Cargo.toml | 2 +- test-utils/Cargo.toml | 2 +- 16 files changed, 66 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f7e09e2e46f8..96bbc377eeee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,10 +51,3 @@ opt-level = 3 overflow-checks = false panic = 'unwind' rpath = false - -[patch.crates-io] -arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index b494db7ec47b..7c951b5aabb3 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -33,14 +33,14 @@ simd = ["datafusion/simd"] snmalloc = ["snmalloc-rs"] [dependencies] -arrow = "33.0.0" +arrow = "34.0.0" datafusion = { path = "../datafusion/core", version = "18.0.0", features = ["scheduler"] } env_logger = "0.10" futures = "0.3" mimalloc = { version = "0.1", optional = true, default-features = false } num_cpus = "1.13.0" object_store = "0.5.4" -parquet = "33.0.0" +parquet = "34.0.0" parquet-test-utils = { path = "../parquet-test-utils/", version = "0.1.0" } rand = "0.8.4" serde = { version = "1.0.136", features = ["derive"] } diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 1397d25c66e2..ff2e464a22fa 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -68,8 +68,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" dependencies = [ "ahash", "arrow-arith", @@ -90,8 +91,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" dependencies = [ "arrow-array", "arrow-buffer", @@ -104,8 +106,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" dependencies = [ "ahash", "arrow-buffer", @@ -119,8 +122,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b4ec72eda7c0207727df96cf200f539749d736b21f3e782ece113e18c1a0a7" dependencies = [ "half", "num", @@ -128,8 +132,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" dependencies = [ "arrow-array", "arrow-buffer", @@ -143,8 +148,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" dependencies = [ "arrow-array", "arrow-buffer", @@ -161,8 +167,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" dependencies = [ "arrow-buffer", "arrow-schema", @@ -172,8 +179,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" dependencies = [ "arrow-array", "arrow-buffer", @@ -185,8 +193,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" dependencies = [ "arrow-array", "arrow-buffer", @@ -203,8 +212,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,8 +226,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" dependencies = [ "ahash", "arrow-array", @@ -230,13 +241,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc" [[package]] name = "arrow-select" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" dependencies = [ "arrow-array", "arrow-buffer", @@ -247,8 +260,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" dependencies = [ "arrow-array", "arrow-buffer", @@ -1759,8 +1773,9 @@ dependencies = [ [[package]] name = "parquet" -version = "33.0.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=57f79c03a8dee9d8bf8601bf555aa271746913fe#57f79c03a8dee9d8bf8601bf555aa271746913fe" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" dependencies = [ "ahash", "arrow-array", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index b2c92f10ea4d..e84cb228584d 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.62" readme = "README.md" [dependencies] -arrow = "33.0.0" +arrow = "34.0.0" async-trait = "0.1.41" clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "18.0.0" } @@ -41,9 +41,3 @@ parking_lot = { version = "0.12" } rustyline = "10.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] } url = "2.2" - -[patch.crates-io] -arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } -parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "57f79c03a8dee9d8bf8601bf555aa271746913fe" } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 5bbf6cce5093..fd178e852c25 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,9 +34,9 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow = "33.0.0" -arrow-flight = { version = "33.0.0", features = ["flight-sql-experimental"] } -arrow-schema = "33.0.0" +arrow = "34.0.0" +arrow-flight = { version = "34.0.0", features = ["flight-sql-experimental"] } +arrow-schema = "34.0.0" async-trait = "0.1.41" dashmap = "5.4" datafusion = { path = "../datafusion/core" } diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 7bb77dd670a2..fe8977aa2cfd 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -40,11 +40,11 @@ pyarrow = ["pyo3", "arrow/pyarrow"] [dependencies] apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true } -arrow = { version = "33.0.0", default-features = false } +arrow = { version = "34.0.0", default-features = false } chrono = { version = "0.4", default-features = false } cranelift-module = { version = "0.92.0", optional = true } num_cpus = "1.13.0" object_store = { version = "0.5.4", default-features = false, optional = true } -parquet = { version = "33.0.0", default-features = false, optional = true } +parquet = { version = "34.0.0", default-features = false, optional = true } pyo3 = { version = "0.18.0", optional = true } sqlparser = "0.30" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 53fcc3bee4a6..bf0ab92d732a 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -60,7 +60,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } apache-avro = { version = "0.14", optional = true } -arrow = { version = "33.0.0", features = ["prettyprint"] } +arrow = { version = "34.0.0", features = ["prettyprint"] } async-compression = { version = "0.3.14", features = ["bzip2", "gzip", "xz", "futures-io", "tokio"], optional = true } async-trait = "0.1.41" bytes = "1.1" @@ -86,7 +86,7 @@ num-traits = { version = "0.2", optional = true } num_cpus = "1.13.0" object_store = "0.5.3" parking_lot = "0.12" -parquet = { version = "33.0.0", features = ["arrow", "async"] } +parquet = { version = "34.0.0", features = ["arrow", "async"] } paste = "^1.0" percent-encoding = "2.2.0" pin-project-lite = "^0.2.7" diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 8d017401c0b6..f4c5707ccb28 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "33.0.0", default-features = false } +arrow = { version = "34.0.0", default-features = false } datafusion-common = { path = "../common", version = "18.0.0" } log = "^0.4" sqlparser = "0.30" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index 6402dbc84b0c..92ce3bf7c27a 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { version = "33.0.0", default-features = false } +arrow = { version = "34.0.0", default-features = false } cranelift = "0.89.0" cranelift-jit = "0.89.0" cranelift-module = "0.89.0" diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 74b34c2b1f58..9b26a6429283 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "33.0.0", features = ["prettyprint"] } +arrow = { version = "34.0.0", features = ["prettyprint"] } async-trait = "0.1.41" chrono = { version = "0.4.23", default-features = false } datafusion-common = { path = "../common", version = "18.0.0" } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 44d9bf9e5996..062ecafff4a6 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -43,9 +43,9 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "33.0.0", features = ["prettyprint"] } -arrow-buffer = "33.0.0" -arrow-schema = "33.0.0" +arrow = { version = "34.0.0", features = ["prettyprint"] } +arrow-buffer = "34.0.0" +arrow-schema = "34.0.0" blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4.23", default-features = false } diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 9d39eb592fbe..a09296ca2953 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -40,7 +40,7 @@ default = [] json = ["pbjson", "serde", "serde_json"] [dependencies] -arrow = "33.0.0" +arrow = "34.0.0" chrono = { version = "0.4", default-features = false } datafusion = { path = "../core", version = "18.0.0" } datafusion-common = { path = "../common", version = "18.0.0" } diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml index 344d110d6223..ee16e76535db 100644 --- a/datafusion/row/Cargo.toml +++ b/datafusion/row/Cargo.toml @@ -37,7 +37,7 @@ path = "src/lib.rs" jit = ["datafusion-jit"] [dependencies] -arrow = "33.0.0" +arrow = "34.0.0" datafusion-common = { path = "../common", version = "18.0.0" } datafusion-jit = { path = "../jit", version = "18.0.0", optional = true } paste = "^1.0" diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index cc37b041db7b..b3441f89201b 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow-schema = "33.0.0" +arrow-schema = "34.0.0" datafusion-common = { path = "../common", version = "18.0.0" } datafusion-expr = { path = "../expr", version = "18.0.0" } log = "^0.4" diff --git a/parquet-test-utils/Cargo.toml b/parquet-test-utils/Cargo.toml index 2083e527d28f..0d41aa823de1 100644 --- a/parquet-test-utils/Cargo.toml +++ b/parquet-test-utils/Cargo.toml @@ -25,4 +25,4 @@ edition = "2021" [dependencies] datafusion = { path = "../datafusion/core" } object_store = "0.5.4" -parquet = "33.0.0" +parquet = "34.0.0" diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 4aeaef068fb4..b7863a385f05 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -23,7 +23,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "33.0.0", features = ["prettyprint"] } +arrow = { version = "34.0.0", features = ["prettyprint"] } datafusion-common = { path = "../datafusion/common" } env_logger = "0.10.0" rand = "0.8"