Skip to content

Commit

Permalink
[skeb] fix '429 Too Many Requests' errors (#5766)
Browse files Browse the repository at this point in the history
Introduce '_handle_429' method to make it easier for Extractors to react
to 429 errors regardless of 'sleep-429' settings.
  • Loading branch information
mikf committed Jun 20, 2024
1 parent 60b4541 commit 11421cf
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 11 deletions.
9 changes: 7 additions & 2 deletions gallery_dl/extractor/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class Extractor():
browser = None
request_interval = 0.0
request_interval_min = 0.0
request_interval_429 = 60.0
request_timestamp = 0.0

def __init__(self, match):
Expand Down Expand Up @@ -203,7 +204,9 @@ def request(self, url, method="GET", session=None,
self.log.warning("Cloudflare CAPTCHA")
break

if code == 429 and self._interval_429:
if code == 429 and self._handle_429(response):
continue
elif code == 429 and self._interval_429:
pass
elif code not in retry_codes and code < 500:
break
Expand Down Expand Up @@ -231,6 +234,8 @@ def request(self, url, method="GET", session=None,

raise exception.HttpError(msg, response)

_handle_429 = util.false

def wait(self, seconds=None, until=None, adjust=1.0,
reason="rate limit"):
now = time.time()
Expand Down Expand Up @@ -324,7 +329,7 @@ def _init_options(self):
self.request_interval_min,
)
self._interval_429 = util.build_duration_func(
self.config("sleep-429", 60),
self.config("sleep-429", self.request_interval_429),
)

if self._retries < 0:
Expand Down
19 changes: 10 additions & 9 deletions gallery_dl/extractor/skeb.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"""Extractors for https://skeb.jp/"""

from .common import Extractor, Message
from .. import text, exception
from .. import text
import itertools


Expand All @@ -31,14 +31,15 @@ def _init(self):
if "Authorization" not in self.session.headers:
self.headers["Authorization"] = "Bearer null"

def request(self, url, **kwargs):
while True:
try:
return Extractor.request(self, url, **kwargs)
except exception.HttpError as exc:
if exc.status == 429 and "request_key" in exc.response.cookies:
continue
raise
def _handle_429(self, response):
if "request_key" in response.cookies:
return True

request_key = text.extr(
response.text, "request_key=", ";")
if request_key:
self.cookies.set("request_key", request_key, domain="skeb.jp")
return True

def items(self):
metadata = self.metadata()
Expand Down

0 comments on commit 11421cf

Please sign in to comment.