Split video directory by subset in datumaro format.

CHANGELOG.md

@@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add ExtractedMask and update importers who can use it to use it
   (<https://github.com/openvinotoolkit/datumaro/pull/1480>)
 
+### Bug fixes
+- Split the video directory into subsets to avoid overwriting
+  (<https://github.com/openvinotoolkit/datumaro/pull/1485>)
+
 ## May 2024 Release 1.6.1
 ### Enhancements
 - Prevent AcLauncher for OpenVINO 2024.0

docs/source/docs/data-formats/formats/datumaro.md

@@ -58,6 +58,14 @@ A Datumaro dataset directory should have the following structure:
     │       ├── <image_name2.ext>
     │       └── ...
     ├── videos/  # directory to store video files
+    │   ├── <subset_name_1>/
+    │   │   ├── <video_name1.ext>
+    │   │   ├── <video_name2.ext>
+    │   │   └── ...
+    │   └── <subset_name_2> /
+    │       ├── <video_name1.ext>
+    │       ├── <video_name2.ext>
+    │       └── ...
     └── annotations/
         ├── <subset_name_1>.json
         ├── <subset_name_2>.json

src/datumaro/components/exporter.py

@@ -400,12 +400,14 @@ def save_video(
         item: DatasetItem,
         *,
         basedir: Optional[str] = None,
+        subdir: Optional[str] = None,
         fname: Optional[str] = None,
     ):
         if not item.media or not isinstance(item.media, VideoFrame):
             log.warning("Item '%s' has no video", item.id)
             return
         basedir = self._video_dir if basedir is None else basedir
+        basedir = osp.join(basedir, subdir) if subdir is not None else basedir
         fname = self.make_video_filename(item) if fname is None else fname
 
         path = osp.join(basedir, fname)

src/datumaro/plugins/data_formats/datumaro/base.py

@@ -190,7 +190,9 @@ def _parse_item(self, item_desc: Dict) -> Optional[DatasetItem]:
             if media and video_frame_info:
                 raise MediaTypeError("Dataset cannot contain multiple media types")
             if video_frame_info:
-                video_path = osp.join(self._video_dir, video_frame_info.get("video_path"))
+                video_path = osp.join(
+                    self._video_dir, self._subset, video_frame_info.get("video_path")
+                )
                 if video_path not in self._videos:
                     self._videos[video_path] = Video(video_path)
                 video = self._videos[video_path]

src/datumaro/plugins/data_formats/datumaro/exporter.py

@@ -185,7 +185,7 @@ def context_save_media(
             if context.save_media:
                 fname = context.make_video_filename(item)
                 if not osp.exists(fname):
-                    context.save_video(item, fname=fname)
+                    context.save_video(item, fname=fname, subdir=item.subset)
                 item.media = VideoFrame(Video(fname), video_frame.index)
 
             yield

src/datumaro/plugins/data_formats/kitti/base.py

@@ -12,7 +12,7 @@
 from datumaro.components.dataset_base import DatasetItem, SubsetBase
 from datumaro.components.importer import ImportContext
 from datumaro.components.media import Image
-from datumaro.util.image import find_images, load_image
+from datumaro.util.image import find_images, lazy_image
 from datumaro.util.meta_file_util import has_meta_file, parse_meta_file
 
 from .format import KittiLabelMap, KittiPath, KittiTask, make_kitti_categories, parse_label_map

src/datumaro/plugins/data_formats/mots.py

@@ -19,7 +19,7 @@
 from datumaro.components.exporter import Exporter
 from datumaro.components.importer import ImportContext, Importer
 from datumaro.components.media import Image
-from datumaro.util.image import find_images, load_image, save_image
+from datumaro.util.image import find_images, lazy_image, save_image
 from datumaro.util.mask_tools import merge_masks
 from datumaro.util.meta_file_util import has_meta_file, parse_meta_file