Skip to content
This repository has been archived by the owner on Mar 11, 2024. It is now read-only.

🐛 fix type check when writing df #55

Merged
merged 1 commit into from
Jan 30, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions dagster_polars/io_managers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,32 +269,32 @@ def dump_to_path(
else:
assert obj is not None, "output should not be None if it's type is not Optional"
if not annotation_for_storage_metadata(typing_type):
if typing_type == pl.DataFrame:
if typing_type in POLARS_EAGER_FRAME_ANNOTATIONS:
obj = cast(pl.DataFrame, obj)
df = obj
self.write_df_to_path(context=context, df=df, path=path)
elif typing_type == pl.LazyFrame:
elif typing_type in POLARS_LAZY_FRAME_ANNOTATIONS:
obj = cast(pl.LazyFrame, obj)
df = obj
self.sink_df_to_path(context=context, df=df, path=path)
else:
raise NotImplementedError
raise NotImplementedError(f"dump_df_to_path for {typing_type} is not implemented")
else:
if not annotation_is_typing_optional(typing_type):
frame_type = get_args(typing_type)[0]
else:
frame_type = get_args(get_args(typing_type)[0])[0]

if frame_type == pl.DataFrame:
if frame_type in POLARS_EAGER_FRAME_ANNOTATIONS:
obj = cast(Tuple[pl.DataFrame, Dict[str, Any]], obj)
df, metadata = obj
self.write_df_to_path(context=context, df=df, path=path, metadata=metadata)
elif frame_type == pl.LazyFrame:
elif frame_type in POLARS_LAZY_FRAME_ANNOTATIONS:
obj = cast(Tuple[pl.LazyFrame, Dict[str, Any]], obj)
df, metadata = obj
self.sink_df_to_path(context=context, df=df, path=path, metadata=metadata)
else:
raise NotImplementedError
raise NotImplementedError(f"dump_df_to_path for {typing_type} is not implemented")

def load_from_path(
self, context: InputContext, path: "UPath", partition_key: Optional[str] = None
Expand Down
Loading