diff --git a/poetry.lock b/poetry.lock index fb511dd3..76249e78 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1437,13 +1437,13 @@ dev = ["importlib-metadata", "tox"] [[package]] name = "pyright" -version = "1.1.371" +version = "1.1.373" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.371-py3-none-any.whl", hash = "sha256:cce52e42ff73943243e7e5e24f2a59dee81b97d99f4e3cf97370b27e8a1858cd"}, - {file = "pyright-1.1.371.tar.gz", hash = "sha256:777b508b92dda2db476214c400ce043aad8d8f3dd0e10d284c96e79f298308b5"}, + {file = "pyright-1.1.373-py3-none-any.whl", hash = "sha256:b805413227f2c209f27b14b55da27fe5e9fb84129c9f1eb27708a5d12f6f000e"}, + {file = "pyright-1.1.373.tar.gz", hash = "sha256:f41bcfc8b9d1802b09921a394d6ae1ce19694957b628bc657629688daf8a83ff"}, ] [package.dependencies] @@ -1455,13 +1455,13 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "8.3.1" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, - {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] diff --git a/src/controllers/webhooks.py b/src/controllers/webhooks.py index f5db0205..b3136fcc 100644 --- a/src/controllers/webhooks.py +++ b/src/controllers/webhooks.py @@ -3,7 +3,7 @@ import pydantic from fastapi import APIRouter, Request from program.content.overseerr import Overseerr -from program.indexers.trakt import get_imdbid_from_tmdb +from program.indexers.trakt import get_imdbid_from_tmdb, get_imdbid_from_tvdb from program.media.item import MediaItem from requests import RequestException from utils.logger import logger @@ -37,13 +37,18 @@ async def overseerr(request: Request) -> Dict[str, Any]: imdb_id = req.media.imdbId if not imdb_id: try: - imdb_id = get_imdbid_from_tmdb(req.media.tmdbId) + _type = req.media.media_type + if _type == "tv": + _type = "show" + imdb_id = get_imdbid_from_tmdb(req.media.tmdbId, type=_type) + if not imdb_id or not imdb_id.startswith("tt"): + imdb_id = get_imdbid_from_tvdb(req.media.tvdbId, type=_type) + if not imdb_id or not imdb_id.startswith("tt"): + logger.error(f"Failed to get imdb_id from Overseerr: {req.media.tmdbId}") + return {"success": False, "message": "Failed to get imdb_id from Overseerr", "title": req.subject} except RequestException: - logger.error(f"Failed to get imdb_id from TMDB: {req.media.tmdbId}") - return {"success": False, "message": "Failed to get imdb_id from TMDB", "title": req.subject} - if not imdb_id: - logger.error(f"Failed to get imdb_id from TMDB: {req.media.tmdbId}") - return {"success": False, "message": "Failed to get imdb_id from TMDB", "title": req.subject} + logger.error(f"Failed to get imdb_id from Overseerr: {req.media.tmdbId}") + return {"success": False, "message": "Failed to get imdb_id from Overseerr", "title": req.subject} overseerr: Overseerr = request.app.program.services[Overseerr] if not overseerr.initialized: diff --git a/src/program/content/overseerr.py b/src/program/content/overseerr.py index 553d8e05..624ebd48 100644 --- a/src/program/content/overseerr.py +++ b/src/program/content/overseerr.py @@ -61,7 +61,7 @@ def run(self): try: response = get( - self.settings.url + f"/api/v1/request?take={10000}&filter=approved", + self.settings.url + f"/api/v1/request?take={10000}&filter=approved&sort=added", additional_headers=self.headers, ) except (ConnectionError, RetryError, MaxRetryError) as e: @@ -134,8 +134,11 @@ def get_imdb_id(self, data) -> str: for id_attr, fetcher in alternate_ids: external_id_value = getattr(response.data.externalIds, id_attr, None) if external_id_value: + _type = data.media_type + if _type == "tv": + _type = "show" try: - new_imdb_id: Union[str, None] = fetcher(external_id_value) + new_imdb_id: Union[str, None] = fetcher(external_id_value, type=_type) if not new_imdb_id: continue return new_imdb_id diff --git a/src/program/content/plex_watchlist.py b/src/program/content/plex_watchlist.py index b622c059..47b9ef4b 100644 --- a/src/program/content/plex_watchlist.py +++ b/src/program/content/plex_watchlist.py @@ -1,12 +1,18 @@ """Plex Watchlist Module""" +from types import SimpleNamespace +import xml.etree.ElementTree as ET from typing import Generator, Union from program.media.item import Episode, MediaItem, Movie, Season, Show from program.settings.manager import settings_manager +from program.indexers.trakt import get_imdbid_from_tvdb from requests import HTTPError from utils.logger import logger from utils.request import get, ping +from plexapi.myplex import MyPlexAccount +from requests import Session +import xmltodict class PlexWatchlist: @@ -17,6 +23,8 @@ def __init__(self): self.rss_enabled = False self.settings = settings_manager.settings.content.plex_watchlist self.token = settings_manager.settings.updaters.plex.token + self.account = None + self.session = Session() self.initialized = self.validate() if not self.initialized: return @@ -30,24 +38,29 @@ def validate(self): if not self.token: logger.error("Plex token is not set!") return False + try: + self.account = MyPlexAccount(self.session, token=self.token) + except Exception as e: + logger.error(f"Unable to authenticate Plex account: {e}") + return False if self.settings.rss: for rss_url in self.settings.rss: try: response = ping(rss_url) response.response.raise_for_status() self.rss_enabled = True - return True except HTTPError as e: if e.response.status_code == 404: logger.warning(f"Plex RSS URL {rss_url} is Not Found. Please check your RSS URL in settings.") + return False else: logger.warning( f"Plex RSS URL {rss_url} is not reachable (HTTP status code: {e.response.status_code})." ) + return False except Exception as e: logger.error(f"Failed to validate Plex RSS URL {rss_url}: {e}", exc_info=True) - logger.warning("None of the provided RSS URLs are reachable. Falling back to using user Watchlist.") - return False + return False return True def run(self) -> Generator[Union[Movie, Show, Season, Episode], None, None]: @@ -76,38 +89,49 @@ def _get_items_from_rss(self) -> Generator[MediaItem, None, None]: """Fetch media from Plex RSS Feeds.""" for rss_url in self.settings.rss: try: - response = get(rss_url, timeout=60) - if not response.is_ok: + response = self.session.get(rss_url, timeout=60) + if not response.ok: logger.error(f"Failed to fetch Plex RSS feed from {rss_url}: HTTP {response.status_code}") continue - if not hasattr(response.data, "items"): - logger.error("Invalid response or missing items attribute in response data.") - continue - results = response.data.items - for item in results: - yield self._extract_imdb_ids(item.guids) + + xmltodict_data = xmltodict.parse(response.content) + items = xmltodict_data.get("rss", {}).get("channel", {}).get("item", []) + for item in items: + guid_text = item.get("guid", {}).get("#text", "") + guid_id = guid_text.split("//")[-1] if guid_text else "" + if not guid_id or guid_id in self.recurring_items: + continue + if guid_id.startswith("tt") and guid_id not in self.recurring_items: + yield guid_id + elif guid_id: + imdb_id: str = get_imdbid_from_tvdb(guid_id) + if not imdb_id or not imdb_id.startswith("tt") or imdb_id in self.recurring_items: + continue + yield imdb_id + else: + logger.log("NOT_FOUND", f"Failed to extract IMDb ID from {item['title']}") + continue except Exception as e: logger.error(f"An unexpected error occurred while fetching Plex RSS feed from {rss_url}: {e}") + continue def _get_items_from_watchlist(self) -> Generator[MediaItem, None, None]: """Fetch media from Plex watchlist""" - filter_params = "includeFields=title,year,ratingkey&includeElements=Guid&sort=watchlistedAt:desc" - url = f"https://metadata.provider.plex.tv/library/sections/watchlist/all?X-Plex-Token={self.token}&{filter_params}" - response = get(url) - if not response.is_ok or not hasattr(response.data, "MediaContainer"): - logger.error("Invalid response or missing MediaContainer in response data.") - return - media_container = getattr(response.data, "MediaContainer", None) - if not media_container or not hasattr(media_container, "Metadata"): - logger.log("NOT_FOUND", "MediaContainer is missing Metadata attribute.") - return - for item in media_container.Metadata: - if hasattr(item, "ratingKey") and item.ratingKey: - imdb_id = self._ratingkey_to_imdbid(item.ratingKey) - if imdb_id: + # filter_params = "includeFields=title,year,ratingkey&includeElements=Guid&sort=watchlistedAt:desc" + # url = f"https://metadata.provider.plex.tv/library/sections/watchlist/all?X-Plex-Token={self.token}&{filter_params}" + # response = get(url) + items = self.account.watchlist() + for item in items: + if hasattr(item, "guids") and item.guids: + imdb_id = next((guid.id.split("//")[-1] for guid in item.guids if guid.id.startswith("imdb://")), None) + if imdb_id and imdb_id in self.recurring_items: + continue + elif imdb_id.startswith("tt"): yield imdb_id + else: + logger.log("NOT_FOUND", f"Unable to extract IMDb ID from {item.title} ({item.year}) with data id: {imdb_id}") else: - logger.log("NOT_FOUND", f"{item.title} ({item.year}) is missing ratingKey attribute from Plex") + logger.log("NOT_FOUND", f"{item.title} ({item.year}) is missing guids attribute from Plex") @staticmethod def _ratingkey_to_imdbid(ratingKey: str) -> str: diff --git a/src/program/db/db_functions.py b/src/program/db/db_functions.py index f706528a..1f7bb23b 100644 --- a/src/program/db/db_functions.py +++ b/src/program/db/db_functions.py @@ -4,28 +4,50 @@ from program.types import Event from sqlalchemy import func, select from sqlalchemy.orm import joinedload +from sqlalchemy.exc import NoResultFound, IntegrityError, InvalidRequestError from utils.logger import logger from .db import db -def _ensure_item_exists_in_db(item:MediaItem) -> bool: +def _ensure_item_exists_in_db(item: MediaItem) -> bool: if isinstance(item, (Movie, Show)): with db.Session() as session: - return session.execute(select(func.count(MediaItem._id)).where(MediaItem.imdb_id==item.imdb_id)).scalar_one() != 0 + return session.execute(select(func.count(MediaItem._id)).where(MediaItem.imdb_id == item.imdb_id)).scalar_one() != 0 return item._id is not None def _get_item_type_from_db(item: MediaItem) -> str: with db.Session() as session: - if item._id is None: - return session.execute(select(MediaItem.type).where( (MediaItem.imdb_id==item.imdb_id ) & ( (MediaItem.type == "show") | (MediaItem.type == "movie") ) )).scalar_one() - return session.execute(select(MediaItem.type).where(MediaItem._id==item._id)).scalar_one() - + try: + if item._id is None: + return session.execute(select(MediaItem.type).where((MediaItem.imdb_id == item.imdb_id) & ((MediaItem.type == "show") | (MediaItem.type == "movie")))).scalar_one() + return session.execute(select(MediaItem.type).where(MediaItem._id == item._id)).scalar_one() + except NoResultFound as e: + logger.exception(f"No Result Found in db for {item.log_string} with id {item._id}: {e}") + except Exception as e: + logger.exception(f"Failed to get item type from db for item: {item.log_string} with id {item._id} - {e}") + def _store_item(item: MediaItem): if isinstance(item, (Movie, Show, Season, Episode)) and item._id is not None: with db.Session() as session: - session.merge(item) - session.commit() + try: + session.merge(item) + session.commit() + except IntegrityError as e: + logger.exception(f"IntegrityError: {e}. Attempting to update existing item.") + logger.warning(f"Attempting rollback of session for item: {item.log_string}") + session.rollback() + existing_item = session.query(MediaItem).filter_by(_id=item._id).one() + for key, value in item.__dict__.items(): + if key != '_sa_instance_state' and key != '_id': + setattr(existing_item, key, value) + logger.warning(f"Committing changes to existing item: {item.log_string}") + session.commit() + except InvalidRequestError as e: + logger.exception(f"InvalidRequestError: {e}. Could not update existing item.") + session.rollback() + except Exception as e: + logger.exception(f"Failed to update existing item: {item.log_string} - {e}") else: with db.Session() as session: _check_for_and_run_insertion_required(session, item) diff --git a/src/program/downloaders/realdebrid.py b/src/program/downloaders/realdebrid.py index fc4503b4..346b2de6 100644 --- a/src/program/downloaders/realdebrid.py +++ b/src/program/downloaders/realdebrid.py @@ -103,6 +103,8 @@ def validate(self) -> bool: def run(self, item: MediaItem) -> bool: """Download media item from real-debrid.com""" return_value = False + if not item: + return return_value if self.is_cached(item) and not self._is_downloaded(item): self._download_item(item) return_value = True @@ -129,7 +131,7 @@ def log_item(item: MediaItem) -> None: def is_cached(self, item: MediaItem) -> bool: """Check if item is cached on real-debrid.com""" - if not item.get("streams", {}): + if not item.get("streams", []): return False def _chunked(lst: List, n: int) -> Generator[List, None, None]: @@ -140,9 +142,13 @@ def _chunked(lst: List, n: int) -> Generator[List, None, None]: logger.log("DEBRID", f"Processing {len(item.streams)} streams for {item.log_string}") processed_stream_hashes = set() - filtered_streams = [stream.infohash for stream in item.streams if stream.infohash and stream.infohash not in processed_stream_hashes] + filtered_streams = [ + stream.infohash for stream in item.streams + if stream.infohash and stream.infohash not in processed_stream_hashes + and not stream.blacklisted + ] if not filtered_streams: - logger.log("NOT_FOUND", f"No streams found from filtering: {item.log_string}") + logger.log("NOT_FOUND", f"No streams found from filtering out processed and blacklisted hashes for: {item.log_string}") return False for stream_chunk in _chunked(filtered_streams, 5): @@ -152,27 +158,34 @@ def _chunked(lst: List, n: int) -> Generator[List, None, None]: if response.is_ok and response.data and isinstance(response.data, dict): if self._evaluate_stream_response(response.data, processed_stream_hashes, item): return True + processed_stream_hashes.update(stream_chunk) except Exception as e: logger.exception(f"Error checking cache for streams: {str(e)}", exc_info=True) continue + if item.type == "movie" or item.type == "episode": + for hash in filtered_streams: + stream = next((stream for stream in item.streams if stream.infohash == hash), None) + if stream and not stream.blacklisted: + stream.blacklisted = True + logger.log("NOT_FOUND", f"No wanted cached streams found for {item.log_string} out of {len(filtered_streams)}") return False - def _evaluate_stream_response(self, data, processed_stream_hashes, item): + def _evaluate_stream_response(self, data: dict, processed_stream_hashes: set, item: MediaItem) -> bool: """Evaluate the response data from the stream availability check.""" for stream_hash, provider_list in data.items(): - if stream_hash in processed_stream_hashes: + stream = next((stream for stream in item.streams if stream.infohash == stream_hash), None) + if not stream or stream.blacklisted: continue + if not provider_list or not provider_list.get("rd"): + stream.blacklisted = True continue - processed_stream_hashes.add(stream_hash) + if self._process_providers(item, provider_list, stream_hash): logger.debug(f"Finished processing providers - selecting {stream_hash} for downloading") return True - else: - stream = next(stream for stream in item.streams if stream.infohash == stream_hash) - stream.blacklisted = True return False def _process_providers(self, item: MediaItem, provider_list: dict, stream_hash: str) -> bool: diff --git a/src/program/indexers/trakt.py b/src/program/indexers/trakt.py index b139071e..1730e89a 100644 --- a/src/program/indexers/trakt.py +++ b/src/program/indexers/trakt.py @@ -1,6 +1,7 @@ """Trakt updater module""" from datetime import datetime, timedelta +from types import SimpleNamespace from typing import Generator, List, Optional, Union from program.media.item import Episode, MediaItem, Movie, Season, Show @@ -174,22 +175,49 @@ def find_first(preferred_types, data): data = next((d for d in response.data if d.type in ["show", "movie", "season"]), None) return _map_item_from_data(getattr(data, data.type), data.type) if data else None -def get_imdbid_from_tmdb(tmdb_id: str) -> Optional[str]: + +def get_imdbid_from_tmdb(tmdb_id: str, type: str = "movie") -> Optional[str]: """Wrapper for trakt.tv API search method.""" - url = f"https://api.trakt.tv/search/tmdb/{tmdb_id}?extended=full" + url = f"https://api.trakt.tv/search/tmdb/{tmdb_id}" # ?extended=full response = get(url, additional_headers={"trakt-api-version": "2", "trakt-api-key": CLIENT_ID}) if not response.is_ok or not response.data: return None - imdb_id = get_imdb_id_from_list(response.data) - if imdb_id: + imdb_id = get_imdb_id_from_list(response.data, id_type="tmdb", _id=tmdb_id, type=type) + if imdb_id and imdb_id.startswith("tt"): return imdb_id logger.error(f"Failed to fetch imdb_id for tmdb_id: {tmdb_id}") return None -def get_imdb_id_from_list(namespaces): - for ns in namespaces: - if ns.type == "movie": - return ns.movie.ids.imdb - elif ns.type == "show": - return ns.show.ids.imdb + +def get_imdbid_from_tvdb(tvdb_id: str, type: str = "show") -> Optional[str]: + """Wrapper for trakt.tv API search method.""" + url = f"https://api.trakt.tv/search/tvdb/{tvdb_id}" + response = get(url, additional_headers={"trakt-api-version": "2", "trakt-api-key": CLIENT_ID}) + if not response.is_ok or not response.data: + return None + imdb_id = get_imdb_id_from_list(response.data, id_type="tvdb", _id=tvdb_id, type=type) + if imdb_id and imdb_id.startswith("tt"): + return imdb_id + logger.error(f"Failed to fetch imdb_id for tvdb_id: {tvdb_id}") return None + + +def get_imdb_id_from_list(namespaces: List[SimpleNamespace], id_type: str = None, _id: str = None, type: str = None) -> Optional[str]: + """Get the imdb_id from the list of namespaces.""" + if not any([id_type, _id, type]): + return None + + for ns in namespaces: + if type == "movie" and hasattr(ns, 'movie') and hasattr(ns.movie, 'ids') and hasattr(ns.movie.ids, 'imdb'): + if str(getattr(ns.movie.ids, id_type)) == str(_id): + return ns.movie.ids.imdb + elif type == "show" and hasattr(ns, 'show') and hasattr(ns.show, 'ids') and hasattr(ns.show.ids, 'imdb'): + if str(getattr(ns.show.ids, id_type)) == str(_id): + return ns.show.ids.imdb + elif type == "season" and hasattr(ns, 'season') and hasattr(ns.season, 'ids') and hasattr(ns.season.ids, 'imdb'): + if str(getattr(ns.season.ids, id_type)) == str(_id): + return ns.season.ids.imdb + elif type == "episode" and hasattr(ns, 'episode') and hasattr(ns.episode, 'ids') and hasattr(ns.episode.ids, 'imdb'): + if str(getattr(ns.episode.ids, id_type)) == str(_id): + return ns.episode.ids.imdb + return None \ No newline at end of file diff --git a/src/program/program.py b/src/program/program.py index 2db0b2c5..92750fe6 100644 --- a/src/program/program.py +++ b/src/program/program.py @@ -270,11 +270,11 @@ def _push_event_queue(self, event): if not isinstance(event.item, (Show, Movie, Episode, Season)): logger.log("NEW", f"Added {event.item.log_string} to the queue") else: - logger.log("DISCOVERY", f"Re-added {event.item.log_string} to the queue" ) + logger.log("DISCOVERY", f"Re-added {event.item.log_string} to the queue") return True - logger.debug(f"Item {event.item.log_string} is already in the queue or running, skipping.") - return False + logger.debug(f"Item {event.item.log_string} is already in the queue or running, skipping.") + return False def _pop_event_queue(self, event): with self.mutex: diff --git a/src/program/symlink.py b/src/program/symlink.py index c640fe07..a51b0eb2 100644 --- a/src/program/symlink.py +++ b/src/program/symlink.py @@ -255,15 +255,9 @@ def _symlink(self, item: Union[Movie, Episode]) -> bool: logger.error(f"OS error when creating symlink for {item.log_string}: {e}") return False - if not os.path.islink(destination): - logger.error(f"Symlink validation failed: {destination} is not a symlink for {item.log_string}") - return False if os.readlink(destination) != source: logger.error(f"Symlink validation failed: {destination} does not point to {source} for {item.log_string}") return False - if not os.path.isfile(destination): - logger.error(f"Symlink validation failed: {destination} is not a valid file for {item.log_string}") - return False item.set("symlinked", True) item.set("symlinked_at", datetime.now())