Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: upgrade Ray to 2.6 and fix security dependabots #2403

Merged
merged 8 commits into from
Jul 31, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion awswrangler/distributed/ray/_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def map(self, func: Callable[..., MapOutputType], _: Optional["BaseClient"], *ar
return list(func(*arg) for arg in zip(itertools.repeat(None), *args))


@ray.remote # type: ignore[attr-defined]
@ray.remote
class AsyncActor:
async def run_concurrent(self, func: Callable[..., MapOutputType], *args: Any) -> MapOutputType:
return func(*args)
Expand Down
4 changes: 2 additions & 2 deletions awswrangler/distributed/ray/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ def _estimate_avail_cpus(cur_pg: Optional[PlacementGroup]) -> int:
Args:
cur_pg: The current placement group, if any.
"""
cluster_cpus = int(ray.cluster_resources().get("CPU", 1)) # type: ignore[attr-defined]
cluster_gpus = int(ray.cluster_resources().get("GPU", 0)) # type: ignore[attr-defined]
cluster_cpus = int(ray.cluster_resources().get("CPU", 1))
cluster_gpus = int(ray.cluster_resources().get("GPU", 0))

# If we're in a placement group, we shouldn't assume the entire cluster's
# resources are available for us to use. Estimate an upper bound on what's
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ def __init__(
import ray
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy

self._local_scheduling = NodeAffinitySchedulingStrategy(ray.get_runtime_context().get_node_id(), soft=False) # type: ignore[attr-defined]
self._local_scheduling = NodeAffinitySchedulingStrategy(ray.get_runtime_context().get_node_id(), soft=False)

dataset_kwargs = reader_args.pop("dataset_kwargs", {})
try:
Expand All @@ -225,7 +225,7 @@ def __init__(
# Try to infer dataset schema by passing dummy table through UDF.
dummy_table = schema.empty_table()
try:
inferred_schema = _block_udf(dummy_table).schema # type: ignore[union-attr]
inferred_schema = _block_udf(dummy_table).schema
inferred_schema = inferred_schema.with_metadata(schema.metadata)
except Exception: # pylint: disable=broad-except
_logger.debug(
Expand Down
4 changes: 2 additions & 2 deletions awswrangler/s3/_read_deltalake.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def _set_default_storage_options_kwargs(
@_utils.check_optional_dependency(deltalake, "deltalake")
@apply_configs
def read_deltalake(
path: Optional[str] = None,
path: str,
version: Optional[int] = None,
partitions: Optional[List[Tuple[str, str, Any]]] = None,
columns: Optional[List[str]] = None,
Expand All @@ -54,7 +54,7 @@ def read_deltalake(

Parameters
----------
path: Optional[str]
path: str
The path of the DeltaTable.
version: Optional[int]
The version of the DeltaTable.
Expand Down
Loading