From 9b1995dda3ca0e1151660f749027ddcbd98fe2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 30 Apr 2024 15:53:32 +0200 Subject: [PATCH] [mastodon] add 'favorite', 'list', and 'hashtag' extractors (#5529) --- docs/supportedsites.md | 6 ++-- gallery_dl/extractor/mastodon.py | 54 ++++++++++++++++++++++++++++++-- test/results/mastodonsocial.py | 27 ++++++++++++++++ 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index dc24a29edc..a90ce1593a 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1622,19 +1622,19 @@ Consider all listed sites to potentially be NSFW. mastodon.social https://mastodon.social/ - Bookmarks, Followed Users, Images from Statuses, User Profiles + Bookmarks, Favorites, Followed Users, Hashtags, Lists, Images from Statuses, User Profiles OAuth Pawoo https://pawoo.net/ - Bookmarks, Followed Users, Images from Statuses, User Profiles + Bookmarks, Favorites, Followed Users, Hashtags, Lists, Images from Statuses, User Profiles OAuth baraag https://baraag.net/ - Bookmarks, Followed Users, Images from Statuses, User Profiles + Bookmarks, Favorites, Followed Users, Hashtags, Lists, Images from Statuses, User Profiles OAuth diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 93d0f512d8..a021f00c9b 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -136,6 +136,36 @@ def statuses(self): return MastodonAPI(self).account_bookmarks() +class MastodonFavoriteExtractor(MastodonExtractor): + """Extractor for mastodon favorites""" + subcategory = "favorite" + pattern = BASE_PATTERN + r"/favourites" + example = "https://mastodon.social/favourites" + + def statuses(self): + return MastodonAPI(self).account_favorites() + + +class MastodonListExtractor(MastodonExtractor): + """Extractor for mastodon lists""" + subcategory = "list" + pattern = BASE_PATTERN + r"/lists/(\w+)" + example = "https://mastodon.social/lists/12345" + + def statuses(self): + return MastodonAPI(self).timelines_list(self.item) + + +class MastodonHashtagExtractor(MastodonExtractor): + """Extractor for mastodon hashtags""" + subcategory = "hashtag" + pattern = BASE_PATTERN + r"/tags/(\w+)" + example = "https://mastodon.social/tags/NAME" + + def statuses(self): + return MastodonAPI(self).timelines_tag(self.item) + + class MastodonFollowingExtractor(MastodonExtractor): """Extractor for followed mastodon users""" subcategory = "following" @@ -205,37 +235,55 @@ def account_id_by_username(self, username): raise exception.NotFoundError("account") def account_bookmarks(self): + """Statuses the user has bookmarked""" endpoint = "/v1/bookmarks" return self._pagination(endpoint, None) + def account_favorites(self): + """Statuses the user has favourited""" + endpoint = "/v1/favourites" + return self._pagination(endpoint, None) + def account_following(self, account_id): + """Accounts which the given account is following""" endpoint = "/v1/accounts/{}/following".format(account_id) return self._pagination(endpoint, None) def account_lookup(self, username): + """Quickly lookup a username to see if it is available""" endpoint = "/v1/accounts/lookup" params = {"acct": username} return self._call(endpoint, params).json() def account_search(self, query, limit=40): - """Search for accounts""" + """Search for matching accounts by username or display name""" endpoint = "/v1/accounts/search" params = {"q": query, "limit": limit} return self._call(endpoint, params).json() def account_statuses(self, account_id, only_media=True, exclude_replies=False): - """Fetch an account's statuses""" + """Statuses posted to the given account""" endpoint = "/v1/accounts/{}/statuses".format(account_id) params = {"only_media" : "true" if only_media else "false", "exclude_replies": "true" if exclude_replies else "false"} return self._pagination(endpoint, params) def status(self, status_id): - """Fetch a status""" + """Obtain information about a status""" endpoint = "/v1/statuses/" + status_id return self._call(endpoint).json() + def timelines_list(self, list_id): + """View statuses in the given list timeline""" + endpoint = "/v1/timelines/list/" + list_id + return self._pagination(endpoint, None) + + def timelines_tag(self, hashtag): + """View public statuses containing the given hashtag""" + endpoint = "/v1/timelines/tag/" + hashtag + return self._pagination(endpoint, None) + def _call(self, endpoint, params=None): if endpoint.startswith("http"): url = endpoint diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py index 8c22bcf374..aa4a7b8a52 100644 --- a/test/results/mastodonsocial.py +++ b/test/results/mastodonsocial.py @@ -74,6 +74,33 @@ "#url" : "https://mastodon.social/bookmarks", "#category": ("mastodon", "mastodon.social", "bookmark"), "#class" : mastodon.MastodonBookmarkExtractor, + "#auth" : True, + "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", +}, + +{ + "#url" : "https://mastodon.social/favourites", + "#category": ("mastodon", "mastodon.social", "favorite"), + "#class" : mastodon.MastodonFavoriteExtractor, + "#auth" : True, + "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", +}, + +{ + "#url" : "https://mastodon.social/lists/92653", + "#category": ("mastodon", "mastodon.social", "list"), + "#class" : mastodon.MastodonListExtractor, + "#auth" : True, + "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range" : "1-10", +}, + +{ + "#url" : "https://mastodon.social/tags/mastodon", + "#category": ("mastodon", "mastodon.social", "hashtag"), + "#class" : mastodon.MastodonHashtagExtractor, + "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range" : "1-10", }, {