[moebooru] add generalized extractors for moebooru sites

- add support for sakugabooru.com (closes #1136) - add support for lolibooru.moe (closes #1050) This allows users to dynamically add support for moebooru/myimouto based sites by adding an entry to their config file (like for foolslide, foolfuuka, etc) For example: { "extractor": { "moebooru": { "new-site-1": {"root": "https://site1.net"}, "new-site-2": {"root": "https://www.site2.moe"} } } }
mikf · Dec 1, 2020 · cc15fbe · cc15fbe
1 parent 4312040
commit cc15fbe
Show file tree

Hide file tree

Showing 7 changed files with 308 additions and 241 deletions.
diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst
@@ -69,13 +69,14 @@ Komikcast            https://komikcast.com/              Chapters, Manga
 Konachan             https://konachan.com/               Pools, Popular Images, Posts, Tag Searches
 LINE BLOG            https://www.lineblog.me/            Blogs, Posts
 livedoor Blog        http://blog.livedoor.jp/            Blogs, Posts
+Lolibooru            https://lolibooru.moe/              Pools, Popular Images, Posts, Tag Searches
 Luscious             https://members.luscious.net/       Albums, Search Results
 Manga Fox            https://fanfox.net/                 Chapters
 Manga Here           https://www.mangahere.cc/           Chapters, Manga
 Manga Stream         https://readms.net/                 Chapters
 MangaDex             https://mangadex.org/               Chapters, Manga
 MangaKakalot         https://mangakakalot.com/           Chapters, Manga
-Mangapanda           https://www.mangapanda.com/         Chapters, Manga
+Mangapanda           http://www.mangapanda.com/          Chapters, Manga
 MangaPark            https://mangapark.net/              Chapters, Manga
 Mangareader          https://www.mangareader.net/        Chapters, Manga
 Mangoxo              https://www.mangoxo.com/            Albums, Channels                                   Supported
@@ -110,6 +111,7 @@ RedGIFs              https://redgifs.com/                individual Images, Sear
 rule #34             https://rule34.paheal.net/          Posts, Tag Searches
 Rule 34              https://rule34.xxx/                 Pools, Posts, Tag Searches
 Safebooru            https://safebooru.org/              Pools, Posts, Tag Searches
+Sakugabooru          https://www.sakugabooru.com/        Pools, Popular Images, Posts, Tag Searches
 Sankaku Channel      https://chan.sankakucomplex.com/    Pools, Posts, Tag Searches                         Supported
 Sankaku Complex      https://www.sankakucomplex.com/     Articles, Tag Searches
 Sen Manga            https://raw.senmanga.com/           Chapters

diff --git a/gallery_dl/extractor/3dbooru.py b/gallery_dl/extractor/3dbooru.py
@@ -1,22 +1,21 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2019 Mike Fährmann
+# Copyright 2015-2020 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extract images from http://behoimi.org/"""
+"""Extractors for http://behoimi.org/"""
 
-from . import booru
+from . import moebooru
 
 
-class _3dbooruExtractor(booru.MoebooruPageMixin, booru.BooruExtractor):
+class _3dbooruBase():
     """Base class for 3dbooru extractors"""
     category = "3dbooru"
-    api_url = "http://behoimi.org/post/index.json"
-    post_url = "http://behoimi.org/post/show/{}"
-    page_limit = 1000
+    basecategory = "booru"
+    root = "http://behoimi.org"
 
     def __init__(self, match):
         super().__init__(match)
@@ -26,7 +25,7 @@ def __init__(self, match):
         })
 
 
-class _3dbooruTagExtractor(booru.TagMixin, _3dbooruExtractor):
+class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor):
     """Extractor for images from behoimi.org based on search-tags"""
     pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org/post"
                r"(?:/(?:index)?)?\?tags=(?P<tags>[^&#]+)")
@@ -35,17 +34,25 @@ class _3dbooruTagExtractor(booru.TagMixin, _3dbooruExtractor):
         "content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a",
     })
 
+    def posts(self):
+        params = {"tags": self.tags}
+        return self._pagination(self.root + "/post/index.json", params)
 
-class _3dbooruPoolExtractor(booru.PoolMixin, _3dbooruExtractor):
+
+class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor):
     """Extractor for image-pools from behoimi.org"""
     pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P<pool>\d+)"
     test = ("http://behoimi.org/pool/show/27", {
         "url": "da75d2d1475449d5ef0c266cb612683b110a30f2",
         "content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554",
     })
 
+    def posts(self):
+        params = {"tags": "pool:" + self.pool_id}
+        return self._pagination(self.root + "/post/index.json", params)
+
 
-class _3dbooruPostExtractor(booru.PostMixin, _3dbooruExtractor):
+class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor):
     """Extractor for single images from behoimi.org"""
     pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P<post>\d+)"
     test = ("http://behoimi.org/post/show/140852", {
@@ -60,8 +67,13 @@ class _3dbooruPostExtractor(booru.PostMixin, _3dbooruExtractor):
         },
     })
 
+    def posts(self):
+        params = {"tags": "id:" + self.post_id}
+        return self._pagination(self.root + "/post/index.json", params)
+
 
-class _3dbooruPopularExtractor(booru.MoebooruPopularMixin, _3dbooruExtractor):
+class _3dbooruPopularExtractor(
+        _3dbooruBase, moebooru.MoebooruPopularExtractor):
     """Extractor for popular images from behoimi.org"""
     pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org"
                r"/post/popular_(?P<scale>by_(?:day|week|month)|recent)"
@@ -70,8 +82,3 @@ class _3dbooruPopularExtractor(booru.MoebooruPopularMixin, _3dbooruExtractor):
         "pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg",
         "count": 20,
     })
-
-    def __init__(self, match):
-        super().__init__(match)
-        self.api_url = "http://behoimi.org/post/popular_{scale}.json".format(
-            scale=self.scale)
diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py
@@ -44,7 +44,6 @@
     "hentainexus",
     "hiperdex",
     "hitomi",
-    "hypnohub",
     "idolcomplex",
     "imagebam",
     "imagechest",
@@ -60,7 +59,6 @@
     "keenspot",
     "khinsider",
     "komikcast",
-    "konachan",
     "lineblog",
     "livedoor",
     "luscious",
@@ -123,8 +121,8 @@
     "wikiart",
     "xhamster",
     "xvideos",
-    "yandere",
     "yuki",
+    "moebooru",
     "foolfuuka",
     "foolslide",
     "mastodon",

diff --git a/gallery_dl/extractor/hypnohub.py b/gallery_dl/extractor/hypnohub.py
diff --git a/gallery_dl/extractor/konachan.py b/gallery_dl/extractor/konachan.py