[deviantart] add 'avatar' and 'background' extractors (#4995)

mikf · Jan 2, 2024 · 4f36714 · 4f36714
1 parent ee65f3d
commit 4f36714
Show file tree

Hide file tree

Showing 4 changed files with 154 additions and 12 deletions.
diff --git a/docs/configuration.rst b/docs/configuration.rst
@@ -1401,7 +1401,13 @@ Description
     when processing a user profile.
 
     Possible values are
-    ``"gallery"``, ``"scraps"``, ``"journal"``, ``"favorite"``, ``"status"``.
+    ``"avatar"``,
+    ``"background"``,
+    ``"gallery"``,
+    ``"scraps"``,
+    ``"journal"``,
+    ``"favorite"``,
+    ``"status"``.
 
     It is possible to use ``"all"`` instead of listing all values separately.
 

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
@@ -148,7 +148,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>DeviantArt</td>
     <td>https://www.deviantart.com/</td>
-    <td>Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
+    <td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
     <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
 </tr>
 <tr>

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
@@ -38,7 +38,7 @@ class DeviantartExtractor(Extractor):
 
     def __init__(self, match):
         Extractor.__init__(self, match)
-        self.user = match.group(1) or match.group(2)
+        self.user = (match.group(1) or match.group(2)).lower()
         self.offset = 0
 
     def _init(self):
@@ -104,7 +104,6 @@ def items(self):
                     raise exception.StopExtraction()
                 else:
                     self.subcategory = "group-" + self.subcategory
-                    self.user = self.user.lower()
                     self.group = True
 
         for deviation in self.deviations():
@@ -513,11 +512,13 @@ def initialize(self):
     def items(self):
         base = "{}/{}/".format(self.root, self.user)
         return self._dispatch_extractors((
-            (DeviantartGalleryExtractor , base + "gallery"),
-            (DeviantartScrapsExtractor  , base + "gallery/scraps"),
-            (DeviantartJournalExtractor , base + "posts"),
-            (DeviantartStatusExtractor  , base + "posts/statuses"),
-            (DeviantartFavoriteExtractor, base + "favourites"),
+            (DeviantartAvatarExtractor    , base + "avatar"),
+            (DeviantartBackgroundExtractor, base + "banner"),
+            (DeviantartGalleryExtractor   , base + "gallery"),
+            (DeviantartScrapsExtractor    , base + "gallery/scraps"),
+            (DeviantartJournalExtractor   , base + "posts"),
+            (DeviantartStatusExtractor    , base + "posts/statuses"),
+            (DeviantartFavoriteExtractor  , base + "favourites"),
         ), ("gallery",))
 
 
@@ -538,6 +539,47 @@ def deviations(self):
         return self._folder_urls(folders, "gallery", DeviantartFolderExtractor)
 
 
+class DeviantartAvatarExtractor(DeviantartExtractor):
+    """Extractor for an artist's avatar"""
+    subcategory = "avatar"
+    archive_fmt = "a_{_username}_{index}"
+    pattern = BASE_PATTERN + r"/avatar"
+    example = "https://www.deviantart.com/USER/avatar/"
+
+    def deviations(self):
+        profile = self.api.user_profile(self.user.lower())
+        if profile:
+            url = profile["user"]["usericon"]
+            return ({
+                "author"         : profile["user"],
+                "category"       : "avatar",
+                "index"          : text.parse_int(url.rpartition("?")[2]),
+                "is_deleted"     : False,
+                "is_downloadable": False,
+                "published_time" : 0,
+                "title"          : "avatar",
+                "content"        : {
+                    "src": url.replace("/avatars/", "/avatars-big/", 1),
+                },
+            },)
+        return ()
+
+
+class DeviantartBackgroundExtractor(DeviantartExtractor):
+    """Extractor for an artist's banner"""
+    subcategory = "background"
+    archive_fmt = "b_{index}"
+    pattern = BASE_PATTERN + r"/ba(?:nner|ckground)"
+    example = "https://www.deviantart.com/USER/banner/"
+
+    def deviations(self):
+        try:
+            return (self.api.user_profile(self.user.lower())
+                    ["cover_deviation"]["cover_deviation"],)
+        except Exception:
+            return ()
+
+
 class DeviantartFolderExtractor(DeviantartExtractor):
     """Extractor for deviations inside an artist's gallery folder"""
     subcategory = "folder"

diff --git a/test/results/deviantart.py b/test/results/deviantart.py
@@ -14,16 +14,23 @@
     "#url"     : "https://www.deviantart.com/shimoda7",
     "#category": ("", "deviantart", "user"),
     "#class"   : deviantart.DeviantartUserExtractor,
-    "#pattern" : "/shimoda7/gallery$",
+    "#urls"    : "https://www.deviantart.com/shimoda7/gallery",
 },
 
 {
     "#url"     : "https://www.deviantart.com/shimoda7",
     "#category": ("", "deviantart", "user"),
     "#class"   : deviantart.DeviantartUserExtractor,
     "#options" : {"include": "all"},
-    "#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$",
-    "#count"   : 5,
+    "#urls"    : (
+        "https://www.deviantart.com/shimoda7/avatar",
+        "https://www.deviantart.com/shimoda7/banner",
+        "https://www.deviantart.com/shimoda7/gallery",
+        "https://www.deviantart.com/shimoda7/gallery/scraps",
+        "https://www.deviantart.com/shimoda7/posts",
+        "https://www.deviantart.com/shimoda7/posts/statuses",
+        "https://www.deviantart.com/shimoda7/favourites",
+    ),
 },
 
 {
@@ -195,6 +202,93 @@
     "#class"   : deviantart.DeviantartGalleryExtractor,
 },
 
+{
+    "#url"     : "https://deviantart.com/shimoda7/avatar",
+    "#category": ("", "deviantart", "avatar"),
+    "#class"   : deviantart.DeviantartAvatarExtractor,
+    "#urls"        : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4",
+    "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f",
+
+    "author"         : {
+        "type"    : "premium",
+        "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4",
+        "userid"  : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
+        "username": "shimoda7",
+    },
+    "content"        : {
+        "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
+    },
+    "da_category"    : "avatar",
+    "date"           : "dt:1970-01-01 00:00:00",
+    "extension"      : "jpg",
+    "filename"       : "avatar_by_shimoda7-d4",
+    "index"          : 4,
+    "index_base36"   : "4",
+    "is_deleted"     : False,
+    "is_downloadable": False,
+    "is_original"    : True,
+    "published_time" : 0,
+    "target"         : {
+        "extension": "jpg",
+        "filename" : "avatar_by_shimoda7-d4",
+        "src"      : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
+    },
+    "title"          : "avatar",
+    "username"       : "shimoda7",
+},
+
+{
+    "#url"     : "https://deviantart.com/gdldev/banner",
+    "#category": ("", "deviantart", "background"),
+    "#class"   : deviantart.DeviantartBackgroundExtractor,
+    "#pattern"     : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
+    "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f",
+
+    "allows_comments"  : True,
+    "author"           : {
+        "type"    : "regular",
+        "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?2",
+        "userid"  : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E",
+        "username": "gdldev"
+    },
+    "category_path"    : "",
+    "content"          : {
+        "filename"    : "banner_by_gdldev_dgntyqc.png",
+        "filesize"    : 84510,
+        "height"      : 4000,
+        "src"         : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
+        "transparency": False,
+        "width"       : 6400
+    },
+    "da_category"      : "Uncategorized",
+    "date"             : "dt:2024-01-02 21:16:06",
+    "deviationid"      : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A",
+    "download_filesize": 84510,
+    "extension"        : "png",
+    "filename"         : "banner_by_gdldev-dgntyqc",
+    "index"            : 1007488020,
+    "index_base36"     : "gntyqc",
+    "is_blocked"       : False,
+    "is_deleted"       : False,
+    "is_downloadable"  : True,
+    "is_favourited"    : False,
+    "is_mature"        : False,
+    "is_original"      : True,
+    "is_published"     : False,
+    "preview"          : dict,
+    "printid"          : None,
+    "published_time"   : 1704230166,
+    "stats"            : {
+        "comments"  : 0,
+        "favourites": 0,
+    },
+    "target"           : dict,
+    "thumbs"           : list,
+    "title"            : "Banner",
+    "url"              : "https://sta.sh/0198jippkeys",
+    "username"         : "gdldev",
+},
+
 {
     "#url"     : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous",
     "#comment" : "user",