Skip to content

Commit

Permalink
[deviantart] handle CloudFront blocks in general (#5363)
Browse files Browse the repository at this point in the history
This was already done for non-OAuth requests (#655)
but CF is now blocking OAuth API requests as well.
  • Loading branch information
mikf committed Mar 23, 2024
1 parent 31e7ca7 commit 925123e
Showing 1 changed file with 20 additions and 11 deletions.
31 changes: 20 additions & 11 deletions gallery_dl/extractor/deviantart.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,16 @@ def _init(self):
else:
self.commit_journal = None

def request(self, url, **kwargs):
if "fatal" not in kwargs:
kwargs["fatal"] = False
while True:
response = Extractor.request(self, url, **kwargs)
if response.status_code != 403 or \
b"Request blocked." not in response.content:
return response
self.wait(seconds=300, reason="CloudFront block")

def skip(self, num):
self.offset += num
return num
Expand Down Expand Up @@ -462,18 +472,12 @@ def _extract_comments(self, target_id, target_type="deviation"):

def _limited_request(self, url, **kwargs):
"""Limits HTTP requests to one every 2 seconds"""
kwargs["fatal"] = None
diff = time.time() - DeviantartExtractor._last_request
if diff < 2.0:
self.sleep(2.0 - diff, "request")

while True:
response = self.request(url, **kwargs)
if response.status_code != 403 or \
b"Request blocked." not in response.content:
DeviantartExtractor._last_request = time.time()
return response
self.wait(seconds=180)
response = self.request(url, **kwargs)
DeviantartExtractor._last_request = time.time()
return response

def _fetch_premium(self, deviation):
try:
Expand Down Expand Up @@ -1418,9 +1422,14 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs):
self.authenticate(None if public else self.refresh_token_key)
kwargs["headers"] = self.headers
response = self.extractor.request(url, **kwargs)
data = response.json()
status = response.status_code

try:
data = response.json()
except ValueError:
self.log.error("Unable to parse API response")
data = {}

status = response.status_code
if 200 <= status < 400:
if self.delay > self.delay_min:
self.delay -= 1
Expand Down

0 comments on commit 925123e

Please sign in to comment.