fix: content services now only output new items that arent in the db.…

… tidied some initial startup logging.

src/controllers/webhooks.py

@@ -7,6 +7,7 @@
 from program.media.item import MediaItem
 from requests import RequestException
 from utils.logger import logger
+from program.db.db_functions import _ensure_item_exists_in_db
 
 from .models.overseerr import OverseerrWebhook
 
@@ -19,53 +20,57 @@
 @router.post("/overseerr")
 async def overseerr(request: Request) -> Dict[str, Any]:
     """Webhook for Overseerr"""
-    response = await request.json()
-
-    if response.get("subject") == "Test Notification":
-        logger.log("API", "Received test notification, Overseerr configured properly")
-        return {"success": True}
-
     try:
+        response = await request.json()
+        if response.get("subject") == "Test Notification":
+            logger.log("API", "Received test notification, Overseerr configured properly")
+            return {"status": "success"}
         req = OverseerrWebhook.model_validate(response)
-    except pydantic.ValidationError:
-        return {"success": False, "message": "Invalid request"}
-    except Exception as e:
+    except (Exception, pydantic.ValidationError) as e:
         logger.error(f"Failed to process request: {e}")
-        return {"success": False, "message": "Failed to process request"}
-
+        return {"status": "error", "message": str(e)}
 
-    imdb_id = req.media.imdbId
+    imdb_id = get_imdbid_from_overseerr(req)
     if not imdb_id:
-        try:
-            _type = req.media.media_type
-            if _type == "tv":
-                _type = "show"
-            imdb_id = get_imdbid_from_tmdb(req.media.tmdbId, type=_type)
-            if not imdb_id or not imdb_id.startswith("tt"):
-                imdb_id = get_imdbid_from_tvdb(req.media.tvdbId, type=_type)
-            if not imdb_id or not imdb_id.startswith("tt"):
-                logger.error(f"Failed to get imdb_id from Overseerr: {req.media.tmdbId}")
-                return {"success": False, "message": "Failed to get imdb_id from Overseerr", "title": req.subject}
-        except RequestException:
-            logger.error(f"Failed to get imdb_id from Overseerr: {req.media.tmdbId}")
-            return {"success": False, "message": "Failed to get imdb_id from Overseerr", "title": req.subject}
+        logger.error(f"Failed to get imdb_id from Overseerr: {req.media.tmdbId}")
+        return {"status": "error", "message": "Failed to get imdb_id from Overseerr"}
 
     overseerr: Overseerr = request.app.program.all_services[Overseerr]
     if not overseerr.initialized:
         logger.error("Overseerr not initialized")
-        return {"success": False, "message": "Overseerr not initialized", "title": req.subject}
+        return {"status": "error", "message": "Overseerr not initialized"}
+
+    try:
+        new_item = MediaItem({"imdb_id": imdb_id, "requested_by": "overseerr", "requested_id": req.request.request_id})
+    except Exception as e:
+        logger.error(f"Failed to create item for {imdb_id}: {e}")
+        return {"status": "error", "message": str(e)}
 
-    if imdb_id in overseerr.recurring_items:
-        logger.log("API", "Request already in queue", {"imdb_id": imdb_id})
-        return {"success": True, "message": "Request already in queue", "title": req.subject}
+    if _ensure_item_exists_in_db(new_item) or imdb_id in overseerr.recurring_items:
+        logger.log("API", "Request already in queue or already exists in the database")
+        return {"status": "success"}
     else:
         overseerr.recurring_items.add(imdb_id)
 
     try:
-        new_item = MediaItem({"imdb_id": imdb_id, "requested_by": "overseerr"})
         request.app.program.em.add_item(new_item)
-    except Exception:
-        logger.error(f"Failed to create item from imdb_id: {imdb_id}")
-        return {"success": False, "message": "Failed to create item from imdb_id", "title": req.subject}
+    except Exception as e:
+        logger.error(f"Failed to add item for {imdb_id}: {e}")
+
+    return {"status": "success"}
 
-    return {"success": True, "message": f"Added {imdb_id} to queue"}
+
+def get_imdbid_from_overseerr(req: OverseerrWebhook) -> str:
+    """Get the imdb_id from the Overseerr webhook"""
+    imdb_id = req.media.imdbId
+    if not imdb_id:
+        try:
+            _type = req.media.media_type
+            if _type == "tv":
+                _type = "show"
+            imdb_id = get_imdbid_from_tmdb(req.media.tmdbId, type=_type)
+            if not imdb_id or not imdb_id.startswith("tt"):
+                imdb_id = get_imdbid_from_tvdb(req.media.tvdbId, type=_type)
+        except RequestException:
+            pass
+    return imdb_id

src/program/content/listrr.py

@@ -1,5 +1,4 @@
 """Listrr content module"""
-
 from typing import Generator
 
 from program.indexers.trakt import get_imdbid_from_tmdb
@@ -8,6 +7,7 @@
 from requests.exceptions import HTTPError
 from utils.logger import logger
 from utils.request import get, ping
+from program.db.db_functions import _filter_existing_items
 
 
 class Listrr:
@@ -21,14 +21,12 @@ def __init__(self):
         self.initialized = self.validate()
         if not self.initialized:
             return
-        self.not_found_ids = []
-        self.recurring_items = set()
+        self.recurring_items: set[str] = set()
         logger.success("Listrr initialized!")
 
     def validate(self) -> bool:
         """Validate Listrr settings."""
         if not self.settings.enabled:
-            logger.debug("Listrr is set to disabled.")
             return False
         if self.settings.api_key == "" or len(self.settings.api_key) != 64:
             logger.error("Listrr api key is not set or invalid.")
@@ -60,15 +58,22 @@ def validate(self) -> bool:
 
     def run(self) -> Generator[MediaItem, None, None]:
         """Fetch new media from `Listrr`"""
-        items_to_yield = []
-        self.not_found_ids.clear()
-        movie_items = self._get_items_from_Listrr("Movies", self.settings.movie_lists)
-        show_items = self._get_items_from_Listrr("Shows", self.settings.show_lists)
-        for imdb_id in movie_items + show_items:
-            if imdb_id not in self.recurring_items:
-                self.recurring_items.add(imdb_id)
-                items_to_yield.append(MediaItem({"imdb_id": imdb_id, "requested_by": self.key}))
-        yield items_to_yield
+        try:
+            movie_items = self._get_items_from_Listrr("Movies", self.settings.movie_lists)
+            show_items = self._get_items_from_Listrr("Shows", self.settings.show_lists)
+        except Exception as e:
+            logger.error(f"Failed to fetch items from Listrr: {e}")
+            return
+
+        listrr_items = movie_items + show_items
+        non_existing_items = _filter_existing_items(listrr_items)
+        new_non_recurring_items = [item for item in non_existing_items if item.imdb_id not in self.recurring_items]
+        self.recurring_items.update([item.imdb_id for item in new_non_recurring_items])
+
+        if new_non_recurring_items:
+            logger.info(f"Fetched {len(new_non_recurring_items)} new items from Listrr")
+
+        yield new_non_recurring_items
 
     def _get_items_from_Listrr(self, content_type, content_lists) -> list[MediaItem]:  # noqa: C901, PLR0912
         """Fetch unique IMDb IDs from Listrr for a given type and list of content."""
@@ -95,8 +100,6 @@ def _get_items_from_Listrr(self, content_type, content_lists) -> list[MediaItem]
                             imdb_id = get_imdbid_from_tmdb(item["tmDbId"])
                             if imdb_id:
                                 unique_ids.add(imdb_id)
-                        else:
-                            self.not_found_ids.append(item["id"])
                 except HTTPError as e:
                     if e.response.status_code in [400, 404, 429, 500]:
                         break

src/program/content/mdblist.py

@@ -7,6 +7,7 @@
 from utils.logger import logger
 from utils.ratelimiter import RateLimiter, RateLimitExceeded
 from utils.request import get, ping
+from program.db.db_functions import _filter_existing_items
 
 
 class Mdblist:
@@ -18,14 +19,12 @@ def __init__(self):
         self.initialized = self.validate()
         if not self.initialized:
             return
-        self.recurring_items = set()
         self.requests_per_2_minutes = self._calculate_request_time()
         self.rate_limiter = RateLimiter(self.requests_per_2_minutes, 120, True)
         logger.success("mdblist initialized")
 
     def validate(self):
         if not self.settings.enabled:
-            logger.debug("Mdblist is set to disabled.")
             return False
         if self.settings.api_key == "" or len(self.settings.api_key) != 25:
             logger.error("Mdblist api key is not set.")
@@ -55,15 +54,17 @@ def run(self) -> Generator[MediaItem, None, None]:
                         items = list_items_by_url(list, self.settings.api_key)
                     for item in items:
                         # Check if the item is already completed in the media container
-                        if item.imdb_id and item.imdb_id not in self.recurring_items:
-                            self.recurring_items.add(item.imdb_id)
+                        if item.imdb_id:
                             items_to_yield.append(MediaItem(
                                 {"imdb_id": item.imdb_id, "requested_by": self.key}
                             ))
-
         except RateLimitExceeded:
             pass
-        yield items_to_yield
+
+        non_existing_items = _filter_existing_items(items_to_yield)
+        if len(non_existing_items) > 0:
+            logger.info(f"Found {len(non_existing_items)} new items to fetch")
+        yield non_existing_items
 
     def _calculate_request_time(self):
         limits = my_limits(self.settings.api_key).limits

src/program/content/overseerr.py

@@ -9,6 +9,7 @@
 from urllib3.exceptions import MaxRetryError, NewConnectionError
 from utils.logger import logger
 from utils.request import delete, get, ping, post
+from program.db.db_functions import _filter_existing_items
 
 
 class Overseerr:
@@ -22,12 +23,11 @@ def __init__(self):
         self.run_once = False
         if not self.initialized:
             return
-        self.recurring_items = set()
+        self.recurring_items: set[str] = set()
         logger.success("Overseerr initialized!")
 
     def validate(self) -> bool:
         if not self.settings.enabled:
-            logger.debug("Overseerr is set to disabled.")
             return False
         if self.settings.api_key == "" or len(self.settings.api_key) != 68:
             logger.error("Overseerr api key is not set.")
@@ -53,57 +53,56 @@ def validate(self) -> bool:
 
     def run(self):
         """Fetch new media from `Overseerr`"""
-        if self.settings.use_webhook and not self.run_once:
-            logger.info("Webhook is enabled, but running Overseerr once before switching to webhook.")
-            self.run_once = True
-        elif self.run_once:
+        if self.settings.use_webhook and self.run_once:
             return
 
+        overseerr_items: list[MediaItem] = self.get_media_requests()
+        non_existing_items = _filter_existing_items(overseerr_items)
+        new_non_recurring_items = [item for item in non_existing_items if item.imdb_id not in self.recurring_items]
+        self.recurring_items.update([item.imdb_id for item in new_non_recurring_items])
+
+        if self.settings.use_webhook:
+            logger.debug("Webhook is enabled. Running Overseerr once before switching to webhook.")
+            self.run_once = True
+
+        if new_non_recurring_items:
+            logger.info(f"Fetched {len(new_non_recurring_items)} new items from Overseerr")
+
+        yield new_non_recurring_items
+
+    def get_media_requests(self) -> list[MediaItem]:
+        """Get media requests from `Overseerr`"""
         try:
-            response = get(
-                self.settings.url + f"/api/v1/request?take={10000}&filter=approved&sort=added",
-                additional_headers=self.headers,
-            )
+            response = get(self.settings.url + f"/api/v1/request?take={10000}&filter=approved&sort=added", additional_headers=self.headers)
+            if not response.is_ok:
+                logger.error(f"Failed to fetch requests from overseerr: {response.data}")
+                return []
         except (ConnectionError, RetryError, MaxRetryError) as e:
             logger.error(f"Failed to fetch requests from overseerr: {str(e)}")
-            return
+            return []
         except Exception as e:
             logger.error(f"Unexpected error during fetching requests: {str(e)}")
-            return
+            return []
 
-        if not response.is_ok or not hasattr(response.data, "pageInfo") or getattr(response.data.pageInfo, "results", 0) == 0:
-            return
+        if not hasattr(response.data, "pageInfo") or getattr(response.data.pageInfo, "results", 0) == 0:
+            return []
 
         # Lets look at approved items only that are only in the pending state
         pending_items = [
-            item
-            for item in response.data.results
+            item for item in response.data.results
             if item.status == 2 and item.media.status == 3
         ]
-        items_to_yield = []
-        for item in pending_items:
-            try:
-                mediaId: int = int(item.media.id)
-                if not item.media.imdbId:
-                    imdb_id = self.get_imdb_id(item.media)
-                else:
-                    imdb_id = item.media.imdbId
-                if not imdb_id or imdb_id in self.recurring_items:
-                    continue
-                self.recurring_items.add(imdb_id)
-                media_item = MediaItem({"imdb_id": imdb_id, "requested_by": self.key, "overseerr_id": mediaId, "requested_id": item.id})
-                if media_item:
-                    items_to_yield.append(media_item)
-                else:
-                    logger.log("NOT_FOUND", f"Failed to create media item for {imdb_id}")
-            except Exception as e:
-                logger.error(f"Error processing item {item}: {str(e)}")
-                continue
 
-        if self.settings.use_webhook:
-            self.run_once = True
+        return [
+            MediaItem({
+                "imdb_id": self.get_imdb_id(item.media),
+                "requested_by": self.key,
+                "overseerr_id": item.media.id,
+                "requested_id": item.id
+            })
+            for item in pending_items
+        ]
 
-        yield items_to_yield
 
     def get_imdb_id(self, data) -> str:
         """Get imdbId for item from overseerr"""