From 4f3671458efc2d4f91baf31d2a1cfc54055872c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 2 Jan 2024 23:45:59 +0100 Subject: [PATCH] [deviantart] add 'avatar' and 'background' extractors (#4995) --- docs/configuration.rst | 8 ++- docs/supportedsites.md | 2 +- gallery_dl/extractor/deviantart.py | 56 ++++++++++++++-- test/results/deviantart.py | 100 ++++++++++++++++++++++++++++- 4 files changed, 154 insertions(+), 12 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index e922f813f1..cbc54a7dfb 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1401,7 +1401,13 @@ Description when processing a user profile. Possible values are - ``"gallery"``, ``"scraps"``, ``"journal"``, ``"favorite"``, ``"status"``. + ``"avatar"``, + ``"background"``, + ``"gallery"``, + ``"scraps"``, + ``"journal"``, + ``"favorite"``, + ``"status"``. It is possible to use ``"all"`` instead of listing all values separately. diff --git a/docs/supportedsites.md b/docs/supportedsites.md index b538749bfe..dbdaac24ad 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -148,7 +148,7 @@ Consider all listed sites to potentially be NSFW. DeviantArt https://www.deviantart.com/ - Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches + Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches OAuth diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 2ba47e1e23..4b5f1d772a 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -38,7 +38,7 @@ class DeviantartExtractor(Extractor): def __init__(self, match): Extractor.__init__(self, match) - self.user = match.group(1) or match.group(2) + self.user = (match.group(1) or match.group(2)).lower() self.offset = 0 def _init(self): @@ -104,7 +104,6 @@ def items(self): raise exception.StopExtraction() else: self.subcategory = "group-" + self.subcategory - self.user = self.user.lower() self.group = True for deviation in self.deviations(): @@ -513,11 +512,13 @@ def initialize(self): def items(self): base = "{}/{}/".format(self.root, self.user) return self._dispatch_extractors(( - (DeviantartGalleryExtractor , base + "gallery"), - (DeviantartScrapsExtractor , base + "gallery/scraps"), - (DeviantartJournalExtractor , base + "posts"), - (DeviantartStatusExtractor , base + "posts/statuses"), - (DeviantartFavoriteExtractor, base + "favourites"), + (DeviantartAvatarExtractor , base + "avatar"), + (DeviantartBackgroundExtractor, base + "banner"), + (DeviantartGalleryExtractor , base + "gallery"), + (DeviantartScrapsExtractor , base + "gallery/scraps"), + (DeviantartJournalExtractor , base + "posts"), + (DeviantartStatusExtractor , base + "posts/statuses"), + (DeviantartFavoriteExtractor , base + "favourites"), ), ("gallery",)) @@ -538,6 +539,47 @@ def deviations(self): return self._folder_urls(folders, "gallery", DeviantartFolderExtractor) +class DeviantartAvatarExtractor(DeviantartExtractor): + """Extractor for an artist's avatar""" + subcategory = "avatar" + archive_fmt = "a_{_username}_{index}" + pattern = BASE_PATTERN + r"/avatar" + example = "https://www.deviantart.com/USER/avatar/" + + def deviations(self): + profile = self.api.user_profile(self.user.lower()) + if profile: + url = profile["user"]["usericon"] + return ({ + "author" : profile["user"], + "category" : "avatar", + "index" : text.parse_int(url.rpartition("?")[2]), + "is_deleted" : False, + "is_downloadable": False, + "published_time" : 0, + "title" : "avatar", + "content" : { + "src": url.replace("/avatars/", "/avatars-big/", 1), + }, + },) + return () + + +class DeviantartBackgroundExtractor(DeviantartExtractor): + """Extractor for an artist's banner""" + subcategory = "background" + archive_fmt = "b_{index}" + pattern = BASE_PATTERN + r"/ba(?:nner|ckground)" + example = "https://www.deviantart.com/USER/banner/" + + def deviations(self): + try: + return (self.api.user_profile(self.user.lower()) + ["cover_deviation"]["cover_deviation"],) + except Exception: + return () + + class DeviantartFolderExtractor(DeviantartExtractor): """Extractor for deviations inside an artist's gallery folder""" subcategory = "folder" diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 4196f32c6f..45ee6c18c6 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -14,7 +14,7 @@ "#url" : "https://www.deviantart.com/shimoda7", "#category": ("", "deviantart", "user"), "#class" : deviantart.DeviantartUserExtractor, - "#pattern" : "/shimoda7/gallery$", + "#urls" : "https://www.deviantart.com/shimoda7/gallery", }, { @@ -22,8 +22,15 @@ "#category": ("", "deviantart", "user"), "#class" : deviantart.DeviantartUserExtractor, "#options" : {"include": "all"}, - "#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$", - "#count" : 5, + "#urls" : ( + "https://www.deviantart.com/shimoda7/avatar", + "https://www.deviantart.com/shimoda7/banner", + "https://www.deviantart.com/shimoda7/gallery", + "https://www.deviantart.com/shimoda7/gallery/scraps", + "https://www.deviantart.com/shimoda7/posts", + "https://www.deviantart.com/shimoda7/posts/statuses", + "https://www.deviantart.com/shimoda7/favourites", + ), }, { @@ -195,6 +202,93 @@ "#class" : deviantart.DeviantartGalleryExtractor, }, +{ + "#url" : "https://deviantart.com/shimoda7/avatar", + "#category": ("", "deviantart", "avatar"), + "#class" : deviantart.DeviantartAvatarExtractor, + "#urls" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", + + "author" : { + "type" : "premium", + "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", + "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "content" : { + "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" + }, + "da_category" : "avatar", + "date" : "dt:1970-01-01 00:00:00", + "extension" : "jpg", + "filename" : "avatar_by_shimoda7-d4", + "index" : 4, + "index_base36" : "4", + "is_deleted" : False, + "is_downloadable": False, + "is_original" : True, + "published_time" : 0, + "target" : { + "extension": "jpg", + "filename" : "avatar_by_shimoda7-d4", + "src" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" + }, + "title" : "avatar", + "username" : "shimoda7", +}, + +{ + "#url" : "https://deviantart.com/gdldev/banner", + "#category": ("", "deviantart", "background"), + "#class" : deviantart.DeviantartBackgroundExtractor, + "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f", + + "allows_comments" : True, + "author" : { + "type" : "regular", + "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?2", + "userid" : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E", + "username": "gdldev" + }, + "category_path" : "", + "content" : { + "filename" : "banner_by_gdldev_dgntyqc.png", + "filesize" : 84510, + "height" : 4000, + "src" : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "transparency": False, + "width" : 6400 + }, + "da_category" : "Uncategorized", + "date" : "dt:2024-01-02 21:16:06", + "deviationid" : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A", + "download_filesize": 84510, + "extension" : "png", + "filename" : "banner_by_gdldev-dgntyqc", + "index" : 1007488020, + "index_base36" : "gntyqc", + "is_blocked" : False, + "is_deleted" : False, + "is_downloadable" : True, + "is_favourited" : False, + "is_mature" : False, + "is_original" : True, + "is_published" : False, + "preview" : dict, + "printid" : None, + "published_time" : 1704230166, + "stats" : { + "comments" : 0, + "favourites": 0, + }, + "target" : dict, + "thumbs" : list, + "title" : "Banner", + "url" : "https://sta.sh/0198jippkeys", + "username" : "gdldev", +}, + { "#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", "#comment" : "user",