Skip to content

Commit

Permalink
initial support for child extractor options
Browse files Browse the repository at this point in the history
Using "parent-category>child-category" as extractor category in a config
file allows to set options for a child extractor when it was spawned by
that parent.

For example "reddit>gfycat" to set gfycat options for when it was found
in a reddit post.

{
    "extractor": {
        "gfycat": {
            "filename": "regular filename"
        },
        "reddit>gfycat": {
            "filename": "reddit-specific filename"
        }
    }
}

Note: This does currently not work for most imgur links due to how its
extractor hierarchy is structured.
  • Loading branch information
mikf committed Jul 28, 2023
1 parent 255d08b commit ed21908
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 12 deletions.
26 changes: 14 additions & 12 deletions gallery_dl/extractor/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,6 @@ class Extractor():
def __init__(self, match):
self.log = logging.getLogger(self.category)
self.url = match.string

if self.basecategory:
self.config = self._config_shared
self.config_accumulate = self._config_shared_accumulate
self._cfgpath = ("extractor", self.category, self.subcategory)
self._parentdir = ""

Expand Down Expand Up @@ -98,16 +94,22 @@ def config_accumulate(self, key):
return config.accumulate(self._cfgpath, key)

def _config_shared(self, key, default=None):
return config.interpolate_common(("extractor",), (
(self.category, self.subcategory),
(self.basecategory, self.subcategory),
), key, default)
return config.interpolate_common(
("extractor",), self._cfgpath, key, default)

def _config_shared_accumulate(self, key):
values = config.accumulate(self._cfgpath, key)
conf = config.get(("extractor",), self.basecategory)
if conf:
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
first = True
extr = ("extractor",)

for path in self._cfgpath:
if first:
first = False
values = config.accumulate(extr + path, key)
else:
conf = config.get(extr, path[0])
if conf:
values[:0] = config.accumulate(
(self.subcategory,), key, conf=conf)
return values

def request(self, url, method="GET", session=None,
Expand Down
15 changes: 15 additions & 0 deletions gallery_dl/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,21 @@ def __init__(self, extr, parent=None):
self.kwdict = {}
self.status = 0

cfgpath = []
if parent and parent.extractor.category != extr.category:
cat = "{}>{}".format(
parent.extractor.category, extr.category)
cfgpath.append((cat, extr.subcategory))
cfgpath.append((extr.category, extr.subcategory))
if extr.basecategory:
if not cfgpath:
cfgpath.append((extr.category, extr.subcategory))
cfgpath.append((extr.basecategory, extr.subcategory))
if cfgpath:
extr._cfgpath = cfgpath
extr.config = extr._config_shared
extr.config_accumulate = extr._config_shared_accumulate

actions = extr.config("actions")
if actions:
from .actions import parse
Expand Down

0 comments on commit ed21908

Please sign in to comment.