Add documentation to functional API (all fn.*) + New documentation layout

README.rst

@@ -57,8 +57,8 @@ Table of Contents
 Installing or building DALI
 ---------------------------
 
-DALI is preinstalled in the `NVIDIA GPU Cloud <https://ngc.nvidia.com>`_ TensorFlow, PyTorch,
-and MXNet containers in versions 18.07 and later.
+DALI is preinstalled in the TensorFlow, PyTorch, and MXNet containers in versions 18.07 and
+later on `NVIDIA GPU Cloud <https://ngc.nvidia.com>`_.
 
 Latest DALI release as well as weekly and nightly developer builds can be installed using pip.
 Detailed instructions can be found in the documentation |docs_install|_.

dali/operators/decoder/image_decoder.cc

@@ -185,9 +185,9 @@ image format, it will decode the entire image and crop the selected ROI.
 
 .. note::
   ROI decoding is currently not compatible with hardware-based decoding. Using
-  :meth:`nvidia.dali.ops.ImageDecoderCrop` automatically disables hardware accelerated
-  decoding. To use the hardware decoder, use the :meth:`nvidia.dali.ops.ImageDecoder` and
-  :meth:`nvidia.dali.ops.Crop` operators instead.
+  :meth:`nvidia.dali.fn.image_decoder` automatically disables hardware accelerated
+  decoding. To use the hardware decoder, use the :meth:`nvidia.dali.fn.image_decoder` and
+  :meth:`nvidia.dali.fn.crop` operators instead.
 
 The output of the decoder is in *HWC* layout.
 
@@ -213,9 +213,9 @@ image format, it will decode the entire image and crop the selected ROI.
 
 .. note::
   ROI decoding is currently not compatible with hardware-based decoding. Using
-  :meth:`nvidia.dali.ops.ImageDecoderRandomCrop` automatically disables hardware accelerated
-  decoding. To use the hardware decoder, use the :meth:`nvidia.dali.ops.ImageDecoder` and
-  :meth:`nvidia.dali.ops.RandomResizedCrop` operators instead.
+  :meth:`nvidia.dali.fn.image_decoder_random_crop` automatically disables hardware accelerated
+  decoding. To use the hardware decoder, use the :meth:`nvidia.dali.fn.image_decoder` and
+  :meth:`nvidia.dali.fn.random_crop` operators instead.
 
 The output of the decoder is in *HWC* layout.
 
@@ -257,7 +257,7 @@ and shape is incompatible with the named arguments specified above.
 The slice arguments should provide as many dimensions as specified by the ``axis_names`` or ``axes``
 arguments.
 
-By default, the :meth:`nvidia.dali.ops.ImageDecoderSlice` operator uses normalized coordinates
+By default, the :meth:`nvidia.dali.fn.image_decoder_slice` operator uses normalized coordinates
 and "WH" order for the slice arguments.
 
 When possible, the argument uses the ROI decoding APIs (for example, *libjpeg-turbo* and *nvJPEG*)
@@ -266,9 +266,9 @@ image format, it will decode the entire image and crop the selected ROI.
 
 .. note::
   ROI decoding is currently not compatible with hardware-based decoding. Using
-  :meth:`nvidia.dali.ops.ImageDecoderSlice` automatically disables hardware accelerated decoding.
-  To use the hardware decoder, use the :meth:`nvidia.dali.ops.ImageDecoder` and
-  :meth:`nvidia.dali.ops.Slice` operators instead.
+  :meth:`nvidia.dali.fn.image_decoder_slice` automatically disables hardware accelerated decoding.
+  To use the hardware decoder, use the :meth:`nvidia.dali.fn.image_decoder` and
+  :meth:`nvidia.dali.fn.slice` operators instead.
 
 The output of the decoder is in the *HWC* layout.
 
@@ -280,6 +280,21 @@ Please note that GPU acceleration for JPEG 2000 decoding is only available for C
   .NumInput(1, 3)
   .NumOutput(1)
   .AddParent("ImageDecoderAttr")
-  .AddParent("SliceAttr");
+  .AddParent("SliceAttr")
+  .InputDox(0, "data", "TensorList", R"code(Batch that contains the input data.)code")
+  .InputDox(1, "anchor", "1D TensorList of float or int",
+            R"code(Input that contains normalized or absolute coordinates for the starting
+point of the slice (x0, x1, x2, …).
+
+Integer coordinates are interpreted as absolute coordinates, while float coordinates can be
+interpreted as absolute or relative coordinates, depending on the value of
+``normalized_anchor``.)code")
+  .InputDox(2, "shape", "1D TensorList of float or int",
+            R"code(Input that contains normalized or absolute coordinates for the dimensions
+of the slice (s0, s1, s2, …).
+
+Integer coordinates are interpreted as absolute coordinates, while float coordinates can be
+interpreted as absolute or relative coordinates, depending on the value of
+``normalized_shape``.)code");
 
 }  // namespace dali

dali/operators/generic/slice/slice.cc

@@ -45,7 +45,7 @@ and shape is incompatible with the named arguments specified above.
 The slice arguments should provide as many dimensions as specified by the ``axis_names`` or ``axes``
 arguments.
 
-By default, the :meth:`nvidia.dali.ops.Slice` operator uses normalized coordinates and ``WH``
+By default, the :meth:`nvidia.dali.fn.slice` operator uses normalized coordinates and ``WH``
 order for the slice arguments.)code")
     .NumInput(1, 3)
     .InputDevice(1, 3, InputDevice::CPU)

dali/operators/geometry/affine_transforms/combine_transforms.cc

@@ -36,7 +36,7 @@ using affine_mat_t = mat<mat_dim, mat_dim, T>;
 
 DALI_SCHEMA(transforms__Combine)
   .DocStr(R"code(Combines two or more affine transforms.
-  
+
 By default, the transforms are combined such that applying the resulting transform to a point is equivalent to
  applying the input transforms in the order as listed.
 

dali/operators/geometry/affine_transforms/transform_translation.cc

@@ -40,7 +40,7 @@ DALI_SCHEMA(TransformTranslation)  // Deprecated in 0.28.0dev
 .. warning::
 
    This operator was renamed and moved to dedicated submodule.
-   Please use :class:`transforms.Translation` instead.
+   Please use :meth:`transforms.translation` instead.
 
 )code")
   .AddArg(
@@ -50,7 +50,7 @@ DALI_SCHEMA(TransformTranslation)  // Deprecated in 0.28.0dev
 The number of dimensions of the transform is inferred from this argument.)code",
     DALI_FLOAT_VEC, true)
   .MakeDocHidden()
-  .Deprecate("transforms.Translation")
+  .Deprecate("transforms.translation")
   .NumInput(0, 1)
   .NumOutput(1)
   .AddParent("TransformAttr");

dali/operators/image/color/old_color_twist.cc

@@ -284,7 +284,7 @@ DALI_SCHEMA(OldColorTwist)
 
 .. note::
     This is an old implementation which uses NPP.)code")
-    .Deprecate("ColorTwist")
+    .Deprecate("color_twist")
     .NumInput(1)
     .NumOutput(1)
     .AddOptionalArg("hue", R"code(Hue change, in degrees.)code", 0.f, true)

dali/operators/image/crop/bbox_crop.cc

@@ -152,7 +152,7 @@ the bounding boxes.
 **Outputs: 0**: anchor, 1: shape, 2: bboxes (, 3: labels, 4: bboxes_indices)
 
 The resulting crop parameters are provided as two separate outputs, ``anchor`` and ``shape``,
-that can be fed directly to the :meth:`nvidia.dali.ops.Slice` operator to complete the cropping
+that can be fed directly to the :meth:`nvidia.dali.fn.slice` operator to complete the cropping
 of the original image. ``anchor`` and ``shape`` contain the starting coordinates and dimensions
 for the crop in the ``[x, y, (z)]`` and ``[w, h, (d)]`` formats, respectively. The coordinates can
 be represented in absolute or relative terms, and the represetnation depends on whether
@@ -161,7 +161,7 @@ the fixed ``crop_shape`` was used.
 .. note::
   Both ``anchor`` and ``shape`` are returned as a ``float``, even if they represent absolute
   coordinates due to providing ``crop_shape`` argument. In order for them to be interpreted
-  correctly by :meth:`nvidia.dali.ops.Slice`, ``normalized_anchor`` and ``normalized_shape``
+  correctly by :meth:`nvidia.dali.fn.slice`, ``normalized_anchor`` and ``normalized_shape``
   should be set to False.
 
 

dali/operators/python_function/python_function.cc

@@ -37,7 +37,7 @@ DALI_SCHEMA(PythonFunction)
 This operator can be used to execute custom Python code in the DALI pipeline.
 The function receives the data from DALI as NumPy arrays in case of CPU operators or
 as CuPy arrays for GPU operators. It is expected to return the results in the same format. For
-a more universal data format, see :meth:`nvidia.dali.ops.DLTensorPythonFunction`.
+a more universal data format, see :meth:`nvidia.dali.fn.dl_tensor_python_function`.
 The function should not modify input tensors.
 
 .. warning::
@@ -62,7 +62,7 @@ for CPU and GPU backend, respectively.)code", false);
 DALI_SCHEMA(TorchPythonFunction)
         .DocStr(R"code(Executes a function that is operating on Torch tensors.
 
-This class is analogous to :meth:`nvidia.dali.ops.PythonFunction` but the tensor data is handled
+This class is analogous to :meth:`nvidia.dali.fn.python_function` but the tensor data is handled
 as PyTorch tensors.)code")
         .NumInput(0, 256)
         .AllowSequences()

dali/operators/random/coin_flip_cpu.cc

@@ -49,7 +49,7 @@ sample is generated.
     .NumInput(0, 1)
     .NumOutput(1)
     .AddParent("random__CoinFlip")
-    .Deprecate("random.CoinFlip");  // Deprecated in 0.30
+    .Deprecate("random.coin_flip");  // Deprecated in 0.30
 
 
 DALI_REGISTER_OPERATOR(CoinFlip, CoinFlip<CPUBackend>, CPU);

dali/operators/random/normal_distribution_cpu.cc

@@ -48,7 +48,7 @@ sample is generated.
     .NumInput(0, 1)
     .NumOutput(1)
     .AddParent("random__Normal")
-    .Deprecate("random.Normal");  // Deprecated in 0.30
+    .Deprecate("random.normal");  // Deprecated in 0.30
 
 
 DALI_REGISTER_OPERATOR(NormalDistribution, NormalDistribution<CPUBackend>, CPU);

dali/operators/random/uniform_distribution_cpu.cc

@@ -62,7 +62,7 @@ This argument is mutually exclusive with ``values``.)code",
 This argument is mutually exclusive with ``range``.)code",
       nullptr, true)
     .AddParent("RNGAttr")
-    .Deprecate("random.Uniform");  // Deprecated in 0.30
+    .Deprecate("random.uniform");  // Deprecated in 0.30
 
 
 DALI_REGISTER_OPERATOR(Uniform, UniformDistribution<CPUBackend>, CPU);

dali/operators/reader/video_reader_resize_op.cc

@@ -34,7 +34,7 @@ The video streams can be in most of the container file formats. FFmpeg is used t
 containers and returns a batch of sequences with shape ``(N, F, H, W, C)``, with N being
 the batch size, and F the number of frames in the sequence.
 
-This operator combines the features of :meth:`nvidia.dali.ops.VideoDecoder` and :meth:`nvidia.dali.ops.Resize`.
+This operator combines the features of :meth:`nvidia.dali.fn.video_reader` and :meth:`nvidia.dali.fn.resize`.
 
 .. note::
   The decoder supports only constant frame-rate videos.

dali/operators/ssd/random_crop.cc

@@ -34,7 +34,7 @@ cropped and valid bounding boxes and valid labels are returned.)code")
   .NumInput(3)   // [img, bbox, label]
   .NumOutput(3)  // [img, bbox, label]
   .AddOptionalArg("num_attempts", R"code(Number of attempts.)code", 1)
-  .Deprecate("RandomBBoxCrop");  // deprecated in DALI 0.30
+  .Deprecate("random_bbox_crop");  // deprecated in DALI 0.30
 
 /*
  * # This function is from https://github.com/kuangliu/pytorch-ssd.

dali/pipeline/operator/builtin/external_source.cc

@@ -86,11 +86,8 @@ and bandwidth.)code", false)
   .MakeInternal();
 
 DALI_SCHEMA(ExternalSource)
-  .DocStr(R"code(Allows externally provided data to be passed as an input to the pipeline,
-see :meth:`nvidia.dali.ops.ExternalSource.__init__`,
-see :meth:`nvidia.dali.fn.external_source`,
-:meth:`nvidia.dali.pipeline.Pipeline.feed_input` and
-:meth:`nvidia.dali.pipeline.Pipeline.iter_setup`.
+  .DocStr(R"code(Allows externally provided data to be passed as an input to the pipeline.
+
 Currently this operator is not supported in TensorFlow. It is worth noting that fed inputs
 should match the number of dimensions expected by the next operator in the pipeline
 (e.g. HWC will expect 3-dimensional tensors

dali/pipeline/operator/op_schema.h

@@ -156,7 +156,7 @@ graph even if its outputs are not used.)code", false);
    * @param doc
    * @param append_kwargs_section
    */
-  DLL_PUBLIC inline OpSchema &CallDocStr(const string &doc, bool append_kwargs_section = true) {
+  DLL_PUBLIC inline OpSchema &CallDocStr(const string &doc, bool append_kwargs_section = false) {
     DALI_ENFORCE(!doc.empty(), "The custom docstring for __call__ should not be empty.");
 
     DALI_ENFORCE(!input_dox_set_,
@@ -894,9 +894,9 @@ graph even if its outputs are not used.)code", false);
   // Custom docstring, if not empty should be used in place of input_dox_ descriptions
   std::string call_dox_ = {};
 
-  // Whether to append kwargs section to __call__ docstring. On by default,
-  // can be turned off for call_dox_ specified manually
-  bool append_kwargs_section_ = true;
+  // Whether to append kwargs section to __call__ docstring. Off by default,
+  // can be turned on for call_dox_ specified manually
+  bool append_kwargs_section_ = false;
 
   SpecFunc output_fn_, in_place_fn_, additional_outputs_fn_;
 

dali/python/nvidia/dali/data_node.py

@@ -114,7 +114,7 @@ def __bool__(self):
                 " graph of operations for DALI Pipeline. It should not be used for truth evaluation"
                 " in regular Python context. Bool conversion in Pipeline can be achieved"
                 " with \"Cast\" operator. To see what operations are allowed on DataNodes to"
-                " represent computations in DALI Pipeline see the \"Mathematical expressions\""
+                " represent computations in DALI Pipeline see the \"Mathematical Expressions\""
                 " section of DALI documentation."))
 
 

dali/python/nvidia/dali/external_source.py

@@ -189,7 +189,7 @@ class ExternalSource():
 The simplest and preferred way is to specify a ``source``, which can be a callable or iterable.
 
 .. warning::
-    :meth:`nvidia.dali.ops.ExternalSource` operator is not compatible with TensorFlow integration.
+    :meth:`nvidia.dali.fn.external_source` operator is not compatible with TensorFlow integration.
 
 .. note::
     To return a batch of copies of the same tensor, use :func:`nvidia.dali.types.Constant`,
@@ -475,7 +475,7 @@ def external_source(source = None, num_outputs = None, *, cycle = None, name = N
 provided memory is copied to the internal buffer.
 
 .. warning::
-    :meth:`nvidia.dali.ops.ExternalSource` operator is not compatible with TensorFlow integration.
+    :meth:`nvidia.dali.fn.external_source` operator is not compatible with TensorFlow integration.
 
 .. note::
     To return a batch of copies of the same tensor, use :func:`nvidia.dali.types.Constant`,

dali/python/nvidia/dali/fn.py

@@ -61,10 +61,10 @@ def _to_snake_case(pascal):
     out = _handle_special_case(out)
     return out
 
-def _wrap_op_fn(op_class, wrapper_name):
-    def op_wrapper(*inputs, **arguments):
+def _wrap_op_fn(op_class, wrapper_name, wrapper_doc):
+    def op_wrapper(*inputs, **kwargs):
         import nvidia.dali.ops
-        init_args, call_args = nvidia.dali.ops._separate_kwargs(arguments)
+        init_args, call_args = nvidia.dali.ops._separate_kwargs(kwargs)
 
         default_dev = nvidia.dali.ops._choose_device(inputs)
         if default_dev == "gpu" and init_args.get("device") == "cpu":
@@ -77,17 +77,18 @@ def op_wrapper(*inputs, **arguments):
 
     op_wrapper.__name__ = wrapper_name
     op_wrapper.__qualname__ = wrapper_name
-    op_wrapper.__doc__ = "see :class:`{0}.{1}`".format(op_class.__module__, op_class.__name__)
+    op_wrapper.__doc__ = wrapper_doc
     return op_wrapper
 
-def _wrap_op(op_class, submodule, parent_module=None):
+def _wrap_op(op_class, submodule, parent_module, wrapper_doc):
     """Wrap the DALI Operator with fn API and insert the function into appropriate module.
 
     Args:
         op_class: Op class to wrap
         submodule: Additional submodule (scope)
         parent_module (str): If set to None, the wrapper is placed in nvidia.dali.fn module,
             otherwise in a specified parent module.
+        wrapper_doc (str): Documentation of the wrapper function
     """
     schema = _b.TryGetSchema(op_class.__name__)
     make_hidden = schema.IsDocHidden() if schema else False
@@ -98,7 +99,7 @@ def _wrap_op(op_class, submodule, parent_module=None):
         fn_module = sys.modules[parent_module]
     module = _internal.get_submodule(fn_module, submodule)
     if not hasattr(module, wrapper_name):
-        wrap_func = _wrap_op_fn(op_class, wrapper_name)
+        wrap_func = _wrap_op_fn(op_class, wrapper_name, wrapper_doc)
         setattr(module, wrapper_name, wrap_func)
         if submodule:
             wrap_func.__module__ = module.__name__