Part table not caught by delete_downstream_parts

[MichaelCoulter]

im trying to perform this delete:

del_key = {"nwb_file_name": 'RS1020240627_.nwb'}
sgs.SortGroup.delete(del_key)

and i get this error. any help is appreciated.

Error stack

[17:09:06][INFO] Spyglass: Building part-parent cache for SortGroup.
	Found 6 downstream part tables
12-Aug-24 17:09:06 Building part-parent cache for SortGroup.
	Found 6 downstream part tables
[17:09:16][WARNING] Spyglass: Skipping unimported: `ms_decoding`.`__clusterless_decoding_fig_u_r_l_1_d`
12-Aug-24 17:09:16 Skipping unimported: `ms_decoding`.`__clusterless_decoding_fig_u_r_l_1_d`
[17:09:16][WARNING] Spyglass: Skipping unimported: `ms_decoding`.`__clusterless_decoding_figurl_1_d`
12-Aug-24 17:09:16 Skipping unimported: `ms_decoding`.`__clusterless_decoding_figurl_1_d`
[17:09:16][WARNING] Spyglass: Skipping unimported: `ms_decoding`.`__clusterless_ahead_behind`
12-Aug-24 17:09:16 Skipping unimported: `ms_decoding`.`__clusterless_ahead_behind`
[17:09:16][WARNING] Spyglass: Skipping unimported: `ms_valid_decodes`.`clusterless_valid_decode_selection`
12-Aug-24 17:09:16 Skipping unimported: `ms_valid_decodes`.`clusterless_valid_decode_selection`
[17:09:16][WARNING] Spyglass: Skipping unimported: `ms_decoding`.`__clusterless_ahead_behind_distance`
12-Aug-24 17:09:16 Skipping unimported: `ms_decoding`.`__clusterless_ahead_behind_distance`
[17:09:17][WARNING] Spyglass: Skipping unimported: `ms_decoding`.`__sorted_spikes_decoding_figurl_1_d`
12-Aug-24 17:09:17 Skipping unimported: `ms_decoding`.`__sorted_spikes_decoding_figurl_1_d`
[17:09:17][WARNING] Spyglass: Skipping unimported: `ms_place_fields`.`sorted_decoding_group__control_encoding`
12-Aug-24 17:09:17 Skipping unimported: `ms_place_fields`.`sorted_decoding_group__control_encoding`
[17:09:17][WARNING] Spyglass: Skipping unimported: `ms_place_fields`.`sorted_decoding_group__test_encoding`
12-Aug-24 17:09:17 Skipping unimported: `ms_place_fields`.`sorted_decoding_group__test_encoding`
[17:09:17][WARNING] Spyglass: Skipping unimported: `ms_place_fields`.`sorted_decoding_group__stimulus_encoding`
12-Aug-24 17:09:17 Skipping unimported: `ms_place_fields`.`sorted_decoding_group__stimulus_encoding`

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In [7], line 5
      3 import spyglass.spikesorting.v1 as sgs
      4 del_key = {"nwb_file_name": 'RS1020240627_.nwb'}
----> 5 sgs.SortGroup.delete(del_key)

File ~/spyglass/src/spyglass/utils/dj_mixin.py:695, in SpyglassMixin.delete(self, *args, **kwargs)
    693 def delete(self, *args, **kwargs):
    694     """Alias for cautious_delete, overwrites datajoint.table.Table.delete"""
--> 695     self.cautious_delete(*args, **kwargs)

File ~/spyglass/src/spyglass/utils/dj_mixin.py:662, in SpyglassMixin.cautious_delete(self, force_permission, dry_run, *args, **kwargs)
    659 if not force_permission or dry_run:
    660     self._check_delete_permission()
--> 662 down_fts = self.delete_downstream_parts(
    663     dry_run=True,
    664     disable_warning=True,
    665 )
    667 if dry_run:
    668     return (
    669         down_fts,
    670         IntervalList(),  # cleanup func relies on downstream deletes
    671         external["raw"].unused(),
    672         external["analysis"].unused(),
    673     )

File ~/spyglass/src/spyglass/utils/dj_mixin.py:431, in SpyglassMixin.delete_downstream_parts(self, restriction, dry_run, reload_cache, disable_warning, return_graph, verbose, **kwargs)
    428 _ = self._part_masters  # load cache before loading graph
    429 restriction = restriction or self.restriction or True
--> 431 restr_graph = RestrGraph(
    432     seed_table=self,
    433     leaves={self.full_table_name: restriction},
    434     direction="down",
    435     cascade=True,
    436     verbose=verbose,
    437 )
    439 if return_graph:
    440     return restr_graph

File ~/spyglass/src/spyglass/utils/dj_graph.py:599, in RestrGraph.__init__(self, seed_table, leaves, destinations, direction, cascade, verbose, **kwargs)
    597 for dir in dir_list:
    598     self._log_truncate(f"Start {dir:<4} : {self.leaves}")
--> 599     self.cascade(direction=dir)
    600     self.cascaded = False
    601     self.visited -= self.leaves

File ~/spyglass/src/spyglass/utils/dj_graph.py:745, in RestrGraph.cascade(self, show_progress, direction, warn)
    741     restr = self._get_restr(table)
    742     self._log_truncate(
    743         f"Start  {direction:<4}: {self._camel(table)}, {restr}"
    744     )
--> 745     self.cascade1(table, restr, direction=direction)
    747 self.cascaded = True  # Mark here so next step can use `restr_ft`
    748 self.cascade_files()

File ~/spyglass/src/spyglass/utils/dj_graph.py:456, in AbstractGraph.cascade1(self, table, restriction, direction, replace, count, **kwargs)
    453 if next_restr == ["False"]:  # Stop cascade if empty restriction
    454     continue
--> 456 self.cascade1(
    457     table=next_table,
    458     restriction=next_restr,
    459     direction=direction,
    460     replace=replace,
    461     count=count + 1,
    462 )

File ~/spyglass/src/spyglass/utils/dj_graph.py:456, in AbstractGraph.cascade1(self, table, restriction, direction, replace, count, **kwargs)
    453 if next_restr == ["False"]:  # Stop cascade if empty restriction
    454     continue
--> 456 self.cascade1(
    457     table=next_table,
    458     restriction=next_restr,
    459     direction=direction,
    460     replace=replace,
    461     count=count + 1,
    462 )

    [... skipping similar frames: AbstractGraph.cascade1 at line 456 (6 times)]

File ~/spyglass/src/spyglass/utils/dj_graph.py:456, in AbstractGraph.cascade1(self, table, restriction, direction, replace, count, **kwargs)
    453 if next_restr == ["False"]:  # Stop cascade if empty restriction
    454     continue
--> 456 self.cascade1(
    457     table=next_table,
    458     restriction=next_restr,
    459     direction=direction,
    460     replace=replace,
    461     count=count + 1,
    462 )

File ~/spyglass/src/spyglass/utils/dj_graph.py:446, in AbstractGraph.cascade1(self, table, restriction, direction, replace, count, **kwargs)
    443     self._log_truncate(f"{reason}: {self._camel(next_table)}")
    444     continue
--> 446 next_restr = self._bridge_restr(
    447     table1=table,
    448     table2=next_table,
    449     restr=restriction,
    450     **data,
    451 )
    453 if next_restr == ["False"]:  # Stop cascade if empty restriction
    454     continue

File ~/spyglass/src/spyglass/utils/dj_graph.py:324, in AbstractGraph._bridge_restr(self, table1, table2, restr, direction, attr_map, aliased, **kwargs)
    322 # May return empty table if outside imported and outside spyglass
    323 ft1 = self._get_ft(table1) & restr
--> 324 ft2 = self._get_ft(table2)
    326 if len(ft1) == 0 or len(ft2) == 0:
    327     return ["False"]

File ~/spyglass/src/spyglass/utils/dj_graph.py:259, in AbstractGraph._get_ft(self, table, with_restr, warn)
    256 else:
    257     restr = True
--> 259 if not (ft := self._get_node(table).get("ft")):
    260     ft = FreeTable(self.connection, table)
    261     self._set_node(table, "ft", ft)

File ~/spyglass/src/spyglass/utils/dj_graph.py:180, in AbstractGraph._get_node(self, table)
    178 table = ensure_names(table)
    179 if not (node := self.graph.nodes.get(table)):
--> 180     raise ValueError(
    181         f"Table {table} not found in graph."
    182         + "\n\tPlease import this table and rerun"
    183     )
    184 return node

ValueError: Table `mua_v1`.`__mua_events_v1` not found in graph.
	Please import this table and rerun

[samuelbray32]

Did you try importing MuaEventsV1 and then running again?

[MichaelCoulter]

thanks, i was able to get past that error and now i have this one. i did import "SortedSpikesGroup" but i still get this error. im not sure how to import the parts table SortedSpikesGroup.Unit

Error stack

---------------------------------------------------------------------------
DataJointError                            Traceback (most recent call last)
Cell In [14], line 5
      3 import spyglass.spikesorting.v1 as sgs
      4 del_key = {"nwb_file_name": 'RS1020240627_.nwb'}
----> 5 sgs.SortGroup.delete(del_key)

File ~/spyglass/src/spyglass/utils/dj_mixin.py:695, in SpyglassMixin.delete(self, *args, **kwargs)
    693 def delete(self, *args, **kwargs):
    694     """Alias for cautious_delete, overwrites datajoint.table.Table.delete"""
--> 695     self.cautious_delete(*args, **kwargs)

File ~/spyglass/src/spyglass/utils/dj_mixin.py:675, in SpyglassMixin.cautious_delete(self, force_permission, dry_run, *args, **kwargs)
    667 if dry_run:
    668     return (
    669         down_fts,
    670         IntervalList(),  # cleanup func relies on downstream deletes
    671         external["raw"].unused(),
    672         external["analysis"].unused(),
    673     )
--> 675 if not self._commit_downstream_delete(down_fts, start=start, **kwargs):
    676     return  # Abort delete based on user input
    678 super().delete(*args, **kwargs)  # Confirmation here

File ~/spyglass/src/spyglass/utils/dj_mixin.py:377, in SpyglassMixin._commit_downstream_delete(self, down_fts, start, **kwargs)
    371 if (
    372     self._test_mode
    373     or not safemode
    374     or user_choice("Commit deletes?", default="no") == "yes"
    375 ):
    376     for down_ft in down_fts:  # safemode off b/c already checked
--> 377         down_ft.delete(safemode=False, **kwargs)
    378 else:
    379     logger.info("Delete aborted.")

File ~/anaconda3/envs/spyglass2/lib/python3.9/site-packages/datajoint/table.py:574, in Table.delete(self, transaction, safemode, force_parts)
    572             if transaction:
    573                 self.connection.cancel_transaction()
--> 574             raise DataJointError(
    575                 "Attempt to delete part table {part} before deleting from "
    576                 "its master {master} first.".format(part=part, master=master)
    577             )
    579 # Confirm and commit
    580 if delete_count == 0:

DataJointError: Attempt to delete part table `spikesorting_group_v1`.`sorted_spikes_group__units` before deleting from its master `spikesorting_group_v1`.`sorted_spikes_group` first.

[CBroz1]

Hi @MichaelCoulter - Thanks for reporting. The part delete error is one that would be avoided entirely by my pending PR to DataJoint. I'm hoping we can get that merged this week.

In the short term, this error is asking you to delete from SortedSpikesGroup first, and then run your previous delete.

[samuelbray32]

Closing as resolved by #1035 (further cleanup in case from #1081 once merged)