[newgrounds] update and fix pagination (#2456)

gallery_dl/extractor/newgrounds.py

@@ -236,16 +236,20 @@ def _video_fallback(formats):
             yield fmt[1][0]["src"]
 
     def _pagination(self, kind):
-        root = self.user_root
+        url = "{}/{}".format(self.user_root, kind)
+        params = {
+            "page": 1,
+            "isAjaxRequest": "1",
+        }
         headers = {
-            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Referer": url,
             "X-Requested-With": "XMLHttpRequest",
-            "Referer": root,
         }
-        url = "{}/{}/page/1".format(root, kind)
 
         while True:
-            with self.request(url, headers=headers, fatal=False) as response:
+            with self.request(
+                    url, params=params, headers=headers,
+                    fatal=False) as response:
                 try:
                     data = response.json()
                 except ValueError:
@@ -256,14 +260,17 @@ def _pagination(self, kind):
                     msg = ", ".join(text.unescape(e) for e in data["errors"])
                     raise exception.StopExtraction(msg)
 
-            for year in data["sequence"]:
-                for item in data["years"][str(year)]["items"]:
+            for year, items in data["items"].items():
+                for item in items:
                     page_url = text.extract(item, 'href="', '"')[0]
-                    yield text.urljoin(root, page_url)
+                    if page_url[0] == "/":
+                        page_url = self.root + page_url
+                    yield page_url
 
-            if not data["more"]:
+            more = data.get("load_more")
+            if not more or len(more) < 8:
                 return
-            url = text.urljoin(root, data["more"])
+            params["page"] += 1
 
 
 class NewgroundsImageExtractor(NewgroundsExtractor):
@@ -301,9 +308,9 @@ class NewgroundsImageExtractor(NewgroundsExtractor):
             "count": 2,
         }),
         # "adult" rated (#2456)
-        ("https://www.newgrounds.com/art/view/bdoneart/vampire-booty", {
+        ("https://www.newgrounds.com/art/view/kekiiro/red", {
             "options": (("username", None),),
-            "count": 0,
+            "count": 1,
         }),
     )
 
@@ -374,7 +381,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor):
         # "adult" rated (#2456)
         ("https://www.newgrounds.com/portal/view/717744", {
             "options": (("username", None),),
-            "count": 0,
+            "count": 1,
         }),
     )
 
@@ -470,25 +477,28 @@ def posts(self):
         )
 
     def _pagination(self, kind):
-        num = 1
+        url = "{}/favorites/{}".format(self.user_root, kind)
+        params = {
+            "page": 1,
+            "isAjaxRequest": "1",
+        }
         headers = {
-            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Referer": url,
             "X-Requested-With": "XMLHttpRequest",
-            "Referer": self.user_root,
         }
 
         while True:
-            url = "{}/favorites/{}/{}".format(self.user_root, kind, num)
-            response = self.request(url, headers=headers)
+            response = self.request(url, params=params, headers=headers)
             if response.history:
                 return
 
-            favs = self._extract_favorites(response.text)
+            data = response.json()
+            favs = self._extract_favorites(data.get("component") or "")
             yield from favs
 
             if len(favs) < 24:
                 return
-            num += 1
+            params["page"] += 1
 
     def _extract_favorites(self, page):
         return [