diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index 58519014b4..e46425a7a7 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -325,11 +325,7 @@ def _to_df_internal(self) -> pd.DataFrame: def _to_arrow_internal(self) -> pyarrow.Table: """Return dataset as pyarrow Table synchronously""" - - # write to temp parquet and then load it as pyarrow table from disk - with tempfile.TemporaryDirectory() as temp_dir: - self.to_spark_df().write.parquet(temp_dir, mode="overwrite") - return pq.read_table(temp_dir) + return pyarrow.Table.from_pandas(self._to_df_internal()) def persist(self, storage: SavedDatasetStorage, allow_overwrite: bool = False): """