Check for socket timeout for yt-dl

Potential workaround for #1469. Seems to work better than 37db56e

musicbot/downloader.py

@@ -4,8 +4,6 @@
 import functools
 import youtube_dl
 
-from .exceptions import YouTubeDLIsAwful
-
 from concurrent.futures import ThreadPoolExecutor
 
 log = logging.getLogger(__name__)
@@ -22,7 +20,9 @@
     'no_warnings': True,
     'default_search': 'auto',
     'source_address': '0.0.0.0',
-    'usenetrc': True
+    'usenetrc': True,
+    'socket_timeout': 1,
+    'retries': 100
 }
 
 # Fuck your useless bugreports message that gets two link embeds and confuses users
@@ -38,9 +38,6 @@
 
 class Downloader:
     def __init__(self, download_folder=None):
-        # Temporary dirty patch for YT throttling
-        ytdl_format_options['progress_hooks'] = [self.hook]
-
         self.thread_pool = ThreadPoolExecutor(max_workers=2)
         self.unsafe_ytdl = youtube_dl.YoutubeDL(ytdl_format_options)
         self.safe_ytdl = youtube_dl.YoutubeDL(ytdl_format_options)
@@ -60,13 +57,6 @@ def __init__(self, download_folder=None):
     def ytdl(self):
         return self.safe_ytdl
 
-    def hook(self, data):
-        if data['status'] == 'downloading':
-            if not data['speed']:
-                return
-            if 200000 > data['speed']:  # if speed drops below 200KiB/s
-                raise YouTubeDLIsAwful('borked')
-
     async def extract_info(self, loop, *args, on_error=None, retry_on_error=False, **kwargs):
         """
             Runs ytdl.extract_info within the threadpool. Returns a future that will fire when it's done.

musicbot/entry.py

@@ -5,7 +5,7 @@
 
 from enum import Enum
 from .constructs import Serializable
-from .exceptions import ExtractionError, YouTubeDLIsAwful
+from .exceptions import ExtractionError
 from .utils import get_header, md5sum
 
 log = logging.getLogger(__name__)
@@ -224,9 +224,6 @@ async def _really_download(self, *, hash=False):
             try:
                 result = await self.playlist.downloader.extract_info(self.playlist.loop, self.url, download=True)
                 break
-            except YouTubeDLIsAwful:
-                log.warning('Detected low download speed (< 200KiB/s). Trying to restart...')
-                continue
             except Exception as e:
                 raise ExtractionError(e)
 

musicbot/exceptions.py

@@ -40,10 +40,6 @@ class FFmpegError(MusicbotException):
 class FFmpegWarning(MusicbotException):
     pass
 
-# YouTube-DL started downloading really slowly
-class YouTubeDLIsAwful(MusicbotException):
-    pass
-
 # Some issue retrieving something from Spotify's API
 class SpotifyError(MusicbotException):
     pass

musicbot/playlist.py

@@ -82,11 +82,11 @@ async def add_entry(self, song_url, **meta):
 
         # TODO: Extract this to its own function
         if info['extractor'] in ['generic', 'Dropbox']:
+            log.debug('Detected a generic extractor, or Dropbox')
             try:
                 headers = await get_header(self.bot.aiosession, info['url'])
                 content_type = headers.get('CONTENT-TYPE')
                 log.debug("Got content type {}".format(content_type))
-
             except Exception as e:
                 log.warning("Failed to get content type for url {} ({})".format(song_url, e))
                 content_type = None
@@ -97,9 +97,9 @@ async def add_entry(self, song_url, **meta):
                         # How does a server say `application/ogg` what the actual fuck
                         raise ExtractionError("Invalid content type \"%s\" for url %s" % (content_type, song_url))
 
-                elif content_type.startswith('text/html'):
+                elif content_type.startswith('text/html') and info['extractor'] == 'generic':
                     log.warning("Got text/html for content-type, this might be a stream.")
-                    # TODO: Check for shoutcast/icecast
+                    return await self.add_stream_entry(song_url, info=info, **meta)  # TODO: Check for shoutcast/icecast
 
                 elif not content_type.startswith(('audio/', 'video/')):
                     log.warning("Questionable content-type \"{}\" for url {}".format(content_type, song_url))