add 'output.num-to-str' option

... to convert any numeric values to string when outputting them as JSON
(during '--dump-json' or otherwise)
This commit is contained in:
Mike Fährmann
2018-10-08 20:28:54 +02:00
parent af3f81c7d9
commit 48a8717a7c
2 changed files with 24 additions and 3 deletions

View File

@@ -498,7 +498,6 @@ class DataJob(Job):
Job.__init__(self, url, parent) Job.__init__(self, url, parent)
self.file = file self.file = file
self.data = [] self.data = []
self.ensure_ascii = config.get(("output", "ascii"), True)
def run(self): def run(self):
# collect data # collect data
@@ -510,10 +509,15 @@ class DataJob(Job):
except BaseException: except BaseException:
pass pass
if config.get(("output", "num-to-str"), False):
for msg in self.data:
util.transform_dict(msg[-1], util.number_to_string)
# dump to 'file' # dump to 'file'
json.dump( json.dump(
self.data, self.file, self.data, self.file,
sort_keys=True, indent=2, ensure_ascii=self.ensure_ascii sort_keys=True, indent=2,
ensure_ascii=config.get(("output", "ascii"), True),
) )
self.file.write("\n") self.file.write("\n")
@@ -528,3 +532,6 @@ class DataJob(Job):
def handle_queue(self, url, keywords): def handle_queue(self, url, keywords):
self.data.append((Message.Queue, url, keywords.copy())) self.data.append((Message.Queue, url, keywords.copy()))
def handle_finalize(self):
self.file.close()

View File

@@ -57,7 +57,7 @@ def raises(obj):
def combine_dict(a, b): def combine_dict(a, b):
"""Recursively combine the contents of b into a""" """Recursively combine the contents of 'b' into 'a'"""
for key, value in b.items(): for key, value in b.items():
if key in a and isinstance(value, dict) and isinstance(a[key], dict): if key in a and isinstance(value, dict) and isinstance(a[key], dict):
combine_dict(a[key], value) combine_dict(a[key], value)
@@ -66,6 +66,20 @@ def combine_dict(a, b):
return a return a
def transform_dict(a, func):
"""Recursively apply 'func' to all values in 'a'"""
for key, value in a.items():
if isinstance(value, dict):
transform_dict(value, func)
else:
a[key] = func(value)
def number_to_string(value):
"""Convert numbers (int, float) to string; Return everything else as is."""
return str(value) if isinstance(value, (int, float)) else value
def expand_path(path): def expand_path(path):
"""Expand environment variables and tildes (~)""" """Expand environment variables and tildes (~)"""
if not path: if not path: