[Data] [Docs] Standardize API Refs for Input/Output (#37017)

Standardize API refs for some of the I/O APIs. --------- Signed-off-by: amogkam <[email protected]> Signed-off-by: Amog Kamsetty <[email protected]> Co-authored-by: angelinalg <[email protected]>
ray-project · Jul 7, 2023 · 957f9b7 · 957f9b7
1 parent dfee3f0
commit 957f9b7
Show file tree

Hide file tree

Showing 11 changed files with 1,128 additions and 643 deletions.
diff --git a/doc/source/data/api/input_output.rst b/doc/source/data/api/input_output.rst
@@ -227,6 +227,9 @@ Partitioning API
    datasource.PathPartitionEncoder
    datasource.PathPartitionParser
    datasource.PathPartitionFilter
+   datasource.FileExtensionFilter
+
+.. _metadata_provider:
 
 MetadataProvider API
 --------------------
@@ -240,3 +243,16 @@ MetadataProvider API
    datasource.DefaultFileMetadataProvider
    datasource.DefaultParquetMetadataProvider
    datasource.FastFileMetadataProvider
+
+
+.. _block_write_path_provider:
+
+BlockWritePathProvider API
+--------------------------
+
+.. autosummary::
+   :toctree: doc/
+
+   datasource.BlockWritePathProvider
+   datasource.DefaultBlockWritePathProvider
+
diff --git a/doc/source/data/performance-tips.rst b/doc/source/data/performance-tips.rst
@@ -18,6 +18,8 @@ If your transformation isn't vectorized, there's no performance benefit.
 Optimizing reads
 ----------------
 
+.. _read_parallelism:
+
 Tuning read parallelism
 ~~~~~~~~~~~~~~~~~~~~~~~
 

diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py
diff --git a/python/ray/data/examples/data/different-extensions/data.csv b/python/ray/data/examples/data/different-extensions/data.csv
@@ -0,0 +1,2 @@
+a,b
+0,1
diff --git a/python/ray/data/examples/data/different-extensions/data.json b/python/ray/data/examples/data/different-extensions/data.json
diff --git a/python/ray/data/examples/data/iris.tfrecords.gz b/python/ray/data/examples/data/iris.tfrecords.gz
diff --git a/python/ray/data/examples/data/year=2022/month=09/sales.csv b/python/ray/data/examples/data/year=2022/month=09/sales.csv
@@ -0,0 +1,2 @@
+order_number,quantity
+10107,30
diff --git a/python/ray/data/examples/data/year=2022/month=09/sales.json b/python/ray/data/examples/data/year=2022/month=09/sales.json
@@ -0,0 +1,4 @@
+{
+    "order_number": 10107,
+    "quantity": 30
+}
diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py
diff --git a/python/ray/data/tests/test_consumption.py b/python/ray/data/tests/test_consumption.py
@@ -494,9 +494,6 @@ def test_convert_types(ray_start_regular_shared):
 def test_from_items(ray_start_regular_shared):
     ds = ray.data.from_items(["hello", "world"])
     assert extract_values("item", ds.take()) == ["hello", "world"]
-
-    ds = ray.data.from_items([{"hello": "world"}], output_arrow_format=True)
-    assert ds.take() == [{"hello": "world"}]
     assert isinstance(next(ds.iter_batches(batch_format=None)), pa.Table)
 
 

diff --git a/rllib/offline/dataset_writer.py b/rllib/offline/dataset_writer.py
@@ -71,9 +71,7 @@ def write(self, sample_batch: SampleBatchType):
         # Todo: We should flush at the end of sampling even if this
         # condition was not reached.
         if len(self.samples) >= self.max_num_samples_per_file:
-            ds = data.from_items(self.samples, output_arrow_format=True).repartition(
-                num_blocks=1, shuffle=False
-            )
+            ds = data.from_items(self.samples).repartition(num_blocks=1, shuffle=False)
             if self.format == "json":
                 ds.write_json(self.path, try_create_dir=True)
             elif self.format == "parquet":