Skip to content

Commit

Permalink
Fix serialization of 'datetime' objects in '--write-metadata'
Browse files Browse the repository at this point in the history
Simplified universal serialization support in json.dump() can be achieved
by passing 'default=str', which was already the case in DataJob.run()
for -j/--dump-json, but not for the 'metadata' post-processor.

This commit introduces util.dump_json() that (more or less) unifies the
JSON output procedure of both --write-metadata and --dump-json.

(#251, #252)
  • Loading branch information
mikf committed May 9, 2019
1 parent 8de5866 commit 523ebc9
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 14 deletions.
8 changes: 2 additions & 6 deletions gallery_dl/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

import sys
import time
import json
import logging
from . import extractor, downloader, postprocessor
from . import config, text, util, output, exception
Expand Down Expand Up @@ -469,16 +468,13 @@ def run(self):
except BaseException:
pass

# convert numbers to string
if config.get(("output", "num-to-str"), False):
for msg in self.data:
util.transform_dict(msg[-1], util.number_to_string)

# dump to 'file'
json.dump(
self.data, self.file,
sort_keys=True, indent=2, ensure_ascii=self.ascii, default=str,
)
self.file.write("\n")
util.dump_json(self.data, self.file, self.ascii, 2)

def handle_url(self, url, kwdict):
self.data.append((Message.Url, url, self._filter(kwdict)))
Expand Down
9 changes: 1 addition & 8 deletions gallery_dl/postprocessor/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

from .common import PostProcessor
from .. import util
import json


class MetadataPP(PostProcessor):
Expand Down Expand Up @@ -61,13 +60,7 @@ def _write_tags(self, file, pathfmt):
file.write("\n")

def _write_json(self, file, pathfmt):
json.dump(
pathfmt.keywords,
file,
sort_keys=True,
indent=self.indent,
ensure_ascii=self.ascii,
)
util.dump_json(pathfmt.keywords, file, self.ascii, self.indent)


__postprocessor__ = MetadataPP
13 changes: 13 additions & 0 deletions gallery_dl/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import re
import os
import sys
import json
import shutil
import string
import _string
Expand Down Expand Up @@ -92,6 +93,18 @@ def to_string(value):
return str(value)


def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4):
"""Serialize 'obj' as JSON and write it to 'fp'"""
json.dump(
obj, fp,
ensure_ascii=ensure_ascii,
indent=indent,
default=str,
sort_keys=True,
)
fp.write("\n")


def expand_path(path):
"""Expand environment variables and tildes (~)"""
if not path:
Expand Down

0 comments on commit 523ebc9

Please sign in to comment.