Add merge delete and populate

CHANGELOG.md

@@ -4,31 +4,38 @@
 
 ### Infrastructure
 
-- Additional documentation. #690
-- Clean up following pre-commit checks. #688
-- Add Mixin class to centralize `fetch_nwb` functionality. #692, #734
-- Refactor restriction use in `delete_downstream_merge` #703
-- Add `cautious_delete` to Mixin class
-    - Initial implementation. #711, #762
-    - More robust caching of join to downstream tables. #806
-    - Overwrite datajoint `delete` method to use `cautious_delete`. #806
-    - Reverse join order for session summary. #821
-    - Add temporary logging of use to `common_usage`. #811, #821
-- Add `deprecation_factory` to facilitate table migration. #717
-- Add Spyglass logger. #730
-- IntervalList: Add secondary key `pipeline` #742
-- Increase pytest coverage for `common`, `lfp`, and `utils`. #743
-- Update docs to reflect new notebooks. #776
-- Add overview of Spyglass to docs. #779
-- Update linting for Black 24. #808
-- Steamline dependency management. #822
-- Add catch errors during `populate_all_common`, log in `common_usage`. #XXX
-- Merge UUIDs #XXX
-    - Revise Merge table uuid generation to include source.
-    - Remove mutual exclusivity logic due to new UUID generation.
+- Docs:
+    - Additional documentation. #690
+    - Add overview of Spyglass to docs. #779
+    - Update docs to reflect new notebooks. #776
+- Mixin:
+    - Add Mixin class to centralize `fetch_nwb` functionality. #692, #734
+    - Refactor restriction use in `delete_downstream_merge` #703
+    - Add `cautious_delete` to Mixin class
+        - Initial implementation. #711, #762
+        - More robust caching of join to downstream tables. #806
+        - Overwrite datajoint `delete` method to use `cautious_delete`. #806
+        - Reverse join order for session summary. #821
+        - Add temporary logging of use to `common_usage`. #811, #821
+- Merge Tables:
+    - UUIDs: Revise Merge table uuid generation to include source. #824
+    - UUIDs: Remove mutual exclusivity logic due to new UUID generation. #824
+    - Add method for `merge_populate`. #824
+- Linting:
+    - Clean up following pre-commit checks. #688
+    - Update linting for Black 24. #808
+- Misc:
+    - Add `deprecation_factory` to facilitate table migration. #717
+    - Add Spyglass logger. #730
+    - Increase pytest coverage for `common`, `lfp`, and `utils`. #743
+    - Steamline dependency management. #822
 
 ### Pipelines
 
+- Common:
+    - `IntervalList`: Add secondary key `pipeline` #742
+    - Add `common_usage` table. #811, #821, #824
+    - Add catch errors during `populate_all_common`. #824
 - Spike sorting:
     - Add SpikeSorting V1 pipeline. #651
     - Move modules into spikesorting.v0 #807

src/spyglass/linearization/__init__.py

@@ -1 +1,3 @@
-from spyglass.linearization.merge import LinearizedPositionOutput
+# CB: Circular import if only importing PositionOutput
+
+# from spyglass.linearization.merge import LinearizedPositionOutput

src/spyglass/utils/dj_merge_tables.py

@@ -3,13 +3,14 @@
 from inspect import getmodule
 from itertools import chain as iter_chain
 from pprint import pprint
+from time import time
 from typing import Union
 
 import datajoint as dj
 from datajoint.condition import make_condition
 from datajoint.errors import DataJointError
 from datajoint.preview import repr_html
-from datajoint.utils import from_camel_case, get_master, to_camel_case
+from datajoint.utils import from_camel_case, to_camel_case
 from IPython.core.display import HTML
 
 from spyglass.utils.logging import logger
@@ -248,6 +249,9 @@ def _merge_repr(cls, restriction: str = True) -> dj.expression.Union:
                 return_empties=False,  # motivated by SpikeSortingOutput.Import
             )
         ]
+        if not parts:
+            logger.warning("No parts found. Try adjusting restriction.")
+            return
 
         attr_dict = {  # NULL for non-numeric, 0 for numeric
             attr.name: "0" if attr.numeric else "NULL"
@@ -758,31 +762,32 @@ def merge_fetch(self, restriction: str = True, *attrs, **kwargs) -> list:
             )
         return results[0] if len(results) == 1 else results
 
-    @classmethod
-    def merge_populate(source: str, key=None):
-        raise NotImplementedError(
-            "CBroz: In the future, this command will support executing "
-            + "part_parent `make` and then inserting all entries into Merge"
-        )
+    def merge_populate(self, source: str, keys=None):
+        """Populate the merge table with entries from the source table."""
+        logger.warning("CBroz: Not fully tested. Use with caution.")
+        parent_class = self.merge_get_parent_class(source)
+        if not keys:
+            keys = parent_class.key_source
+        parent_class.populate(keys)
+        successes = (parent_class & keys).fetch("KEY", as_dict=True)
+        self.insert(successes)
 
     def delete(self, force_permission=False, *args, **kwargs):
         """Alias for cautious_delete, overwrites datajoint.table.Table.delete"""
-        raise NotImplementedError(
-            "Please use delete_downstream_merge or cautious_delete "
-            + "to clear merge entries."
-        )
-        # for part in self.merge_get_part(
-        #     restriction=self.restriction,
-        #     multi_source=True,
-        #     return_empties=False,
-        # ):
-        #     part.delete(force_permission=force_permission, *args, **kwargs)
+        for part in self.merge_get_part(
+            restriction=self.restriction,
+            multi_source=True,
+            return_empties=False,
+        ):
+            part.delete(force_permission=force_permission, *args, **kwargs)
 
     def super_delete(self, *args, **kwargs):
         """Alias for datajoint.table.Table.delete.
 
         Added to support MRO of SpyglassMixin"""
         logger.warning("!! Using super_delete. Bypassing cautious_delete !!")
+
+        self._log_use(start=time(), super_delete=True)
         super().delete(*args, **kwargs)
 
 

src/spyglass/utils/dj_mixin.py

@@ -365,7 +365,7 @@ def _usage_table(self):
 
         return CautiousDelete()
 
-    def _log_use(self, start, merge_deletes=None):
+    def _log_use(self, start, merge_deletes=None, super_delete=False):
         """Log use of cautious_delete."""
         if isinstance(merge_deletes, QueryExpression):
             merge_deletes = merge_deletes.fetch(as_dict=True)
@@ -374,15 +374,13 @@ def _log_use(self, start, merge_deletes=None):
             dj_user=dj.config["database.user"],
             origin=self.full_table_name,
         )
+        restr_str = "Super delete: " if super_delete else ""
+        restr_str += "".join(self.restriction) if self.restriction else "None"
         try:
             self._usage_table.insert1(
                 dict(
                     **safe_insert,
-                    restriction=(
-                        "".join(self.restriction)[255:]  # handle list
-                        if self.restriction
-                        else "None"
-                    ),
+                    restriction=restr_str[:255],
                     merge_deletes=merge_deletes,
                 )
             )
@@ -455,4 +453,5 @@ def delete(self, force_permission=False, *args, **kwargs):
     def super_delete(self, *args, **kwargs):
         """Alias for datajoint.table.Table.delete."""
         logger.warning("!! Using super_delete. Bypassing cautious_delete !!")
+        self._log_use(start=time(), super_delete=True)
         super().delete(*args, **kwargs)