Minor: Move ObjectStoreRegistry to datafusion_execution crate

datafusion/core/src/datasource/listing/mod.rs

@@ -59,7 +59,7 @@ pub struct PartitionedFile {
     pub partition_values: Vec<ScalarValue>,
     /// An optional file range for a more fine-grained parallel execution
     pub range: Option<FileRange>,
-    /// An optional field for user defined per object metadata  
+    /// An optional field for user defined per object metadata
     pub extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,
 }
 
@@ -103,3 +103,52 @@ impl From<ObjectMeta> for PartitionedFile {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use datafusion_execution::object_store::ObjectStoreRegistry;
+    use object_store::local::LocalFileSystem;
+
+    use super::*;
+
+    #[test]
+    fn test_object_store_listing_url() {
+        let listing = ListingTableUrl::parse("file:///").unwrap();
+        let store = listing.object_store();
+        assert_eq!(store.as_str(), "file:///");
+
+        let listing = ListingTableUrl::parse("s3://bucket/").unwrap();
+        let store = listing.object_store();
+        assert_eq!(store.as_str(), "s3://bucket/");
+    }
+
+    #[test]
+    fn test_get_by_url_hdfs() {
+        let sut = ObjectStoreRegistry::default();
+        sut.register_store("hdfs", "localhost:8020", Arc::new(LocalFileSystem::new()));
+        let url = ListingTableUrl::parse("hdfs://localhost:8020/key").unwrap();
+        sut.get_by_url(&url).unwrap();
+    }
+
+    #[test]
+    fn test_get_by_url_s3() {
+        let sut = ObjectStoreRegistry::default();
+        sut.register_store("s3", "bucket", Arc::new(LocalFileSystem::new()));
+        let url = ListingTableUrl::parse("s3://bucket/key").unwrap();
+        sut.get_by_url(&url).unwrap();
+    }
+
+    #[test]
+    fn test_get_by_url_file() {
+        let sut = ObjectStoreRegistry::default();
+        let url = ListingTableUrl::parse("file:///bucket/key").unwrap();
+        sut.get_by_url(&url).unwrap();
+    }
+
+    #[test]
+    fn test_get_by_url_local() {
+        let sut = ObjectStoreRegistry::default();
+        let url = ListingTableUrl::parse("../").unwrap();
+        sut.get_by_url(&url).unwrap();
+    }
+}

datafusion/core/src/datasource/mod.rs

@@ -25,10 +25,12 @@ pub mod file_format;
 pub mod listing;
 pub mod listing_table_factory;
 pub mod memory;
-pub mod object_store;
 pub mod streaming;
 pub mod view;
 
+// backwards compatibility
+pub use datafusion_execution::object_store;
+
 use futures::Stream;
 
 pub use self::datasource::TableProvider;

datafusion/core/src/execution/runtime_env.rs

@@ -23,9 +23,11 @@ use crate::{
     execution::disk_manager::{DiskManager, DiskManagerConfig},
 };
 
-use crate::datasource::object_store::ObjectStoreRegistry;
-use crate::execution::memory_pool::{GreedyMemoryPool, MemoryPool, UnboundedMemoryPool};
 use datafusion_common::DataFusionError;
+use datafusion_execution::{
+    memory_pool::{GreedyMemoryPool, MemoryPool, UnboundedMemoryPool},
+    object_store::ObjectStoreRegistry,
+};
 use object_store::ObjectStore;
 use std::fmt::{Debug, Formatter};
 use std::path::PathBuf;

datafusion/execution/Cargo.toml

@@ -34,10 +34,13 @@ path = "src/lib.rs"
 
 
 [dependencies]
+dashmap = "5.4.0"
 datafusion-common = { path = "../common", version = "19.0.0" }
 datafusion-expr = { path = "../expr", version = "19.0.0" }
 hashbrown = { version = "0.13", features = ["raw"] }
 log = "^0.4"
+object_store = "0.5.4"
 parking_lot = "0.12"
 rand = "0.8"
 tempfile = "3"
+url = "2.2"

datafusion/execution/src/lib.rs

@@ -17,4 +17,5 @@
 
 pub mod disk_manager;
 pub mod memory_pool;
+pub mod object_store;
 pub mod registry;

datafusion/core/src/datasource/object_store.rs → datafusion/execution/src/object_store.rs

@@ -217,22 +217,12 @@ impl ObjectStoreRegistry {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::datasource::listing::ListingTableUrl;
-    use std::sync::Arc;
 
     #[test]
     fn test_object_store_url() {
-        let listing = ListingTableUrl::parse("file:///").unwrap();
-        let store = listing.object_store();
-        assert_eq!(store.as_str(), "file:///");
-
         let file = ObjectStoreUrl::parse("file://").unwrap();
         assert_eq!(file.as_str(), "file:///");
 
-        let listing = ListingTableUrl::parse("s3://bucket/").unwrap();
-        let store = listing.object_store();
-        assert_eq!(store.as_str(), "s3://bucket/");
-
         let url = ObjectStoreUrl::parse("s3://bucket").unwrap();
         assert_eq!(url.as_str(), "s3://bucket/");
 
@@ -255,34 +245,4 @@ mod tests {
             ObjectStoreUrl::parse("s3://username:password@host:123/foo").unwrap_err();
         assert_eq!(err.to_string(), "Execution error: ObjectStoreUrl must only contain scheme and authority, got: /foo");
     }
-
-    #[test]
-    fn test_get_by_url_hdfs() {
-        let sut = ObjectStoreRegistry::default();
-        sut.register_store("hdfs", "localhost:8020", Arc::new(LocalFileSystem::new()));
-        let url = ListingTableUrl::parse("hdfs://localhost:8020/key").unwrap();
-        sut.get_by_url(&url).unwrap();
-    }
-
-    #[test]
-    fn test_get_by_url_s3() {
-        let sut = ObjectStoreRegistry::default();
-        sut.register_store("s3", "bucket", Arc::new(LocalFileSystem::new()));
-        let url = ListingTableUrl::parse("s3://bucket/key").unwrap();
-        sut.get_by_url(&url).unwrap();
-    }
-
-    #[test]
-    fn test_get_by_url_file() {
-        let sut = ObjectStoreRegistry::default();
-        let url = ListingTableUrl::parse("file:///bucket/key").unwrap();
-        sut.get_by_url(&url).unwrap();
-    }
-
-    #[test]
-    fn test_get_by_url_local() {
-        let sut = ObjectStoreRegistry::default();
-        let url = ListingTableUrl::parse("../").unwrap();
-        sut.get_by_url(&url).unwrap();
-    }
 }