Fixes: better fill in enchanced mediaitems from symlinks and modify p…

…rogram to use id for queue checking (rivenmedia#541)

* Fix: media-enhancer creates blank items for missing seasons and episodes

* Fix: Check for items in queue by _id instead of comparing objects to prevent issues.

* Fix: Remove erronous returns

* Feat: Try and scrape entire show/season again if backoff timer allows when in partially completed

* Fix: Allow shows and seasons to return scraped state if they're scraped so they can proceed to the next phase

---------

Co-authored-by: Administrator <[email protected]>

src/program/db/db_functions.py

@@ -69,8 +69,9 @@ def _run_thread_with_db_item(fn, service, program, input_item: MediaItem | None)
         with db.Session() as session:
             if isinstance(input_item, (Movie, Show, Season, Episode)):
                 item = input_item
-                if not _check_for_and_run_insertion_required(session, item): 
-                    item = _get_item_from_db(session, item)
+                if not _check_for_and_run_insertion_required(session, item):
+                    pass
+                item = _get_item_from_db(session, item)
 
                 #session.merge(item)
                 for res in fn(item):

src/program/libraries/symlink.py

@@ -90,11 +90,13 @@ def process_shows(directory: Path, item_type: str, is_anime: bool = False) -> Sh
         show_item = Show({'imdb_id': imdb_id.group(), 'title': title.group(1)})
         if is_anime:
             show_item.is_anime = True
+        seasons = {}
         for season in os.listdir(directory / show):
             if not (season_number := re.search(r'(\d+)', season)):
                 logger.log("NOT_FOUND", f"Can't extract season number at path {directory / show / season}")
                 continue
             season_item = Season({'number': int(season_number.group())})
+            episodes = {}
             for episode in os.listdir(directory / show / season):
                 if not (episode_number := re.search(r's\d+e(\d+)', episode)):
                     logger.log("NOT_FOUND", f"Can't extract episode number at path {directory / show / season / episode}")
@@ -110,6 +112,13 @@ def process_shows(directory: Path, item_type: str, is_anime: bool = False) -> Sh
                     episode_item.set("update_folder", "updated")
                 if is_anime:
                     episode_item.is_anime = True
-                season_item.add_episode(episode_item)
-            show_item.add_season(season_item)
-        yield show_item
+                #season_item.add_episode(episode_item)
+                episodes[int(episode_number.group(1))] = episode_item
+            if len(episodes) > 0:
+                for i in range(1, max(episodes.keys())+1):
+                    season_item.add_episode(episodes.get(i, Episode({'number': i})))
+                seasons[int(season_number.group())] = season_item
+        if len(seasons) > 0:
+            for i in range(1, max(seasons.keys())+1):
+                show_item.add_season(seasons.get(i, Season({'number': i})))
+        yield show_item

src/program/media/item.py

@@ -340,14 +340,14 @@ def _determine_state(self):
         if len(self.episodes) > 0:
             if all(episode.state == States.Completed for episode in self.episodes):
                 return States.Completed
-            if any(episode.state == States.Completed for episode in self.episodes):
-                return States.PartiallyCompleted
             if all(episode.state == States.Symlinked for episode in self.episodes):
                 return States.Symlinked
             if all(episode.file and episode.folder for episode in self.episodes):
                 return States.Downloaded
             if self.is_scraped():
                 return States.Scraped
+            if any(episode.state == States.Completed for episode in self.episodes):
+                return States.PartiallyCompleted
             if any(episode.state == States.Indexed for episode in self.episodes):
                 return States.Indexed
             if any(episode.state == States.Requested for episode in self.episodes):
@@ -493,18 +493,17 @@ def get_season_index_by_id(self, item_id):
     def _determine_state(self):
         if all(season.state == States.Completed for season in self.seasons):
             return States.Completed
-
-        if any(
-            season.state in (States.Completed, States.PartiallyCompleted)
-            for season in self.seasons
-        ):
-            return States.PartiallyCompleted
         if all(season.state == States.Symlinked for season in self.seasons):
             return States.Symlinked
         if all(season.state == States.Downloaded for season in self.seasons):
             return States.Downloaded
         if self.is_scraped():
             return States.Scraped
+        if any(
+            season.state in (States.Completed, States.PartiallyCompleted)
+            for season in self.seasons
+        ):
+            return States.PartiallyCompleted
         if any(season.state == States.Indexed for season in self.seasons):
             return States.Indexed
         if any(season.state == States.Requested for season in self.seasons):

src/program/program.py

@@ -193,9 +193,9 @@ def _retry_library(self) -> None:
         with db.Session() as session:
             items_to_submit = session.execute(select(MediaItem).where( (MediaItem.last_state!="Completed" ) & ( (MediaItem.type == 'show') | (MediaItem.type == 'movie') )).order_by(MediaItem.scraped_at.desc())).unique().scalars().all()
             session.expunge_all()
-        logger.log("PROGRAM", f"Found {len(items_to_submit)} items to retry")
-        for item in items_to_submit:
-            self._push_event_queue(Event(emitted_by=self.__class__, item=item))
+            logger.log("PROGRAM", f"Found {len(items_to_submit)} items to retry")
+            for item in items_to_submit:
+                self._push_event_queue(Event(emitted_by=self.__class__, item=item))
 
     def _schedule_functions(self) -> None:
         """Schedule each service based on its update interval."""
@@ -237,25 +237,36 @@ def _schedule_services(self) -> None:
                 coalesce=False,
             )
             logger.log("PROGRAM", f"Scheduled {service_cls.__name__} to run every {update_interval} seconds.")
-
+    def _id_in_queue(self, id):
+        for i in self.queued_items:
+            if i._id == id:
+                return True 
+        return False
+    def _id_in_running_items(self, id):
+        for i in self.running_items:
+            if i._id == id:
+                return True
+        return False
     def _push_event_queue(self, event):
         with self.mutex:
             if( not event.item in self.queued_items and not event.item in self.running_items):
-                if ( isinstance(event.item, Show) 
-                        and (any( [s for s in event.item.seasons if s in self.queued_items or s in self.running_items]) 
-                        or any([e for e in [s.episodes for s in event.item.seasons] if e in self.queued_items or e in self.running_items]) ) 
-                        ):
-                    return 
-                if isinstance(event.item, Season) and any( [e for e in event.item.episodes if e in self.queued_items or e in self.running_items] ):
-                    return
-                if hasattr(event.item, "parent") and event.item.parent in self.queued_items :
-                    return
-                if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and event.item.parent.parent in self.queued_items :
-                    return
-                if hasattr(event.item, "parent") and event.item.parent in self.running_items :
-                    return
-                if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and event.item.parent.parent in self.running_items :
-                    return
+                if hasattr(event.item, "_id"):
+                    if isinstance(event.item, Show):
+                        for s in event.item.seasons:
+                            if self._id_in_queue(s._id) or self._id_in_running_items(s._id):
+                                return
+                            for e in s.episodes:
+                                if self._id_in_queue(e._id) or self._id_in_running_items(e._id):
+                                    return
+
+                    if isinstance(event.item, Season):
+                        for e in event.item.episodes:
+                            if self._id_in_queue(e._id) or self._id_in_running_items(e._id):
+                                return
+                    if hasattr(event.item, "parent") and ( self._id_in_queue(event.item.parent._id) or self._id_in_running_items(event.item.parent._id) ):
+                        return
+                    if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and ( self._id_in_queue(event.item.parent.parent._id) or self._id_in_running_items(event.item.parent.parent._id)):
+                        return
                 self.queued_items.append(event.item)
                 self.event_queue.put(event)
                 if not isinstance(event.item, (Show, Movie, Episode, Season)):
@@ -282,7 +293,10 @@ def add_to_running(self, item, service_name):
         if item is None:
             return
         if item not in self.running_items:
-            self.running_items.append(item)
+            if isinstance(item, MediaItem) and not self._id_in_running_items(item._id):
+                self.running_items.append(item)
+            elif not isinstance(item, MediaItem):
+                self.running_items.append(item)
             logger.log("PROGRAM", f"Item {item.log_string} started running section {service_name}" )
 
     def _process_future_item(self, future: Future, service: Service, orig_item: MediaItem) -> None:

src/program/scrapers/__init__.py

@@ -55,7 +55,7 @@ def yield_incomplete_children(self, item: MediaItem) -> Union[List[Season], List
         return None
 
     def partial_state(self, item: MediaItem) -> bool:
-        if item.state != States.PartiallyCompleted:
+        if item.state != States.PartiallyCompleted or self.can_we_scrape(item):
             return False
         if isinstance(item, Show):
             sres = [s for s in item.seasons if s.state != States.Completed and s.is_released and self.should_submit(s)]