[deviantart] expand nested comment replies (#4653)

gallery_dl/extractor/deviantart.py

@@ -234,7 +234,7 @@ def prepare(self, deviation):
 
         if self.comments:
             deviation["comments"] = (
-                self.api.comments(deviation["deviationid"], target="deviation")
+                self._extract_comments(deviation["deviationid"], "deviation")
                 if deviation["stats"]["comments"] else ()
             )
 
@@ -401,6 +401,28 @@ def _update_token(self, deviation, content):
                 binascii.b2a_base64(payload).rstrip(b"=\n").decode())
         )
 
+    def _extract_comments(self, target_id, target_type="deviation"):
+        results = None
+        comment_ids = [None]
+
+        while comment_ids:
+            comments = self.api.comments(
+                target_id, target_type, comment_ids.pop())
+
+            if results:
+                results.extend(comments)
+            else:
+                results = comments
+
+            # parent comments, i.e. nodes with at least one child
+            parents = {c["parentid"] for c in comments}
+            # comments with more than one reply
+            replies = {c["commentid"] for c in comments if c["replies"]}
+            # add comment UUIDs with replies that are not parent to any node
+            comment_ids.extend(replies - parents)
+
+        return results
+
     def _limited_request(self, url, **kwargs):
         """Limits HTTP requests to one every 2 seconds"""
         kwargs["fatal"] = None
@@ -704,7 +726,7 @@ def prepare(self, deviation):
         deviation["stats"] = {"comments": comments_count}
         if self.comments:
             deviation["comments"] = (
-                self.api.comments(deviation["statusid"], target="status")
+                self._extract_comments(deviation["statusid"], "status")
                 if comments_count else ()
             )
 
@@ -1078,11 +1100,17 @@ def collections_folders(self, username, offset=0):
                   "mature_content": self.mature}
         return self._pagination_list(endpoint, params)
 
-    def comments(self, id, target, offset=0):
+    def comments(self, target_id, target_type="deviation",
+                 comment_id=None, offset=0):
         """Fetch comments posted on a target"""
-        endpoint = "/comments/{}/{}".format(target, id)
-        params = {"maxdepth": "5", "offset": offset, "limit": 50,
-                  "mature_content": self.mature}
+        endpoint = "/comments/{}/{}".format(target_type, target_id)
+        params = {
+            "commentid"     : comment_id,
+            "maxdepth"      : "5",
+            "offset"        : offset,
+            "limit"         : 50,
+            "mature_content": self.mature,
+        }
         return self._pagination_list(endpoint, params=params, key="thread")
 
     def deviation(self, deviation_id, public=None):

test/results/deviantart.py

@@ -547,23 +547,27 @@
     "#options" : {"comments": True},
     "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+",
 
-    "comments": list,
+    "comments": "len:44",
 },
 
 {
-    "#url"     : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466",
-    "#comment" : "wixmp URL rewrite",
+    "#url"     : "https://www.deviantart.com/justatest235723/art/Blue-811519058",
+    "#comment" : "nested comments (#4653)",
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
-    "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/[^/]+/[^.]+\.jpg\?token=",
+    "#options" : {
+        "original": False,
+        "comments": True,
+    },
+
+    "comments": "len:20",
 },
 
 {
     "#url"     : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466",
     "#comment" : "wixmp URL rewrite /intermediary/",
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
-    "#options" : {"jwt": False},
     "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/intermediary/f/[^/]+/[^.]+\.jpg",
 },