From 965968cda0d3d6cace62905b5af5437960ea851a Mon Sep 17 00:00:00 2001
From: Adrian Ehrsam <adrian.ehrsam@bmsuisse.ch>
Date: Wed, 13 Mar 2024 06:56:58 +0100
Subject: [PATCH] we need GIL, no?

---
 python/src/lib.rs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/python/src/lib.rs b/python/src/lib.rs
index 1b81eeab6f..635b377cd2 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -1385,6 +1385,20 @@ impl From<&PyAddAction> for Add {
     }
 }
 
+struct GilIterator {
+    reader: ArrowArrayStreamReader,
+}
+
+impl<'a> Iterator for GilIterator {
+    type Item = RecordBatch;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let reader = &mut self.reader;
+
+        Python::with_gil(|_| reader.next().map_or(None, |v| Some(v.unwrap())))
+    }
+}
+
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
 fn write_to_deltalake(
@@ -1405,7 +1419,7 @@ fn write_to_deltalake(
     custom_metadata: Option<HashMap<String, String>>,
 ) -> PyResult<()> {
     py.allow_threads(|| {
-        let batches = data.0.map(|batch| batch.unwrap());
+        let batches = GilIterator { reader: data.0 }; // The pyarrow reader can be backed by a python object
         let save_mode = mode.parse().map_err(PythonError::from)?;
 
         let options = storage_options.clone().unwrap_or_default();