add 'output.num-to-str' option
... to convert any numeric values to string when outputting them as JSON (during '--dump-json' or otherwise)
This commit is contained in:
@@ -498,7 +498,6 @@ class DataJob(Job):
|
|||||||
Job.__init__(self, url, parent)
|
Job.__init__(self, url, parent)
|
||||||
self.file = file
|
self.file = file
|
||||||
self.data = []
|
self.data = []
|
||||||
self.ensure_ascii = config.get(("output", "ascii"), True)
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
# collect data
|
# collect data
|
||||||
@@ -510,10 +509,15 @@ class DataJob(Job):
|
|||||||
except BaseException:
|
except BaseException:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if config.get(("output", "num-to-str"), False):
|
||||||
|
for msg in self.data:
|
||||||
|
util.transform_dict(msg[-1], util.number_to_string)
|
||||||
|
|
||||||
# dump to 'file'
|
# dump to 'file'
|
||||||
json.dump(
|
json.dump(
|
||||||
self.data, self.file,
|
self.data, self.file,
|
||||||
sort_keys=True, indent=2, ensure_ascii=self.ensure_ascii
|
sort_keys=True, indent=2,
|
||||||
|
ensure_ascii=config.get(("output", "ascii"), True),
|
||||||
)
|
)
|
||||||
self.file.write("\n")
|
self.file.write("\n")
|
||||||
|
|
||||||
@@ -528,3 +532,6 @@ class DataJob(Job):
|
|||||||
|
|
||||||
def handle_queue(self, url, keywords):
|
def handle_queue(self, url, keywords):
|
||||||
self.data.append((Message.Queue, url, keywords.copy()))
|
self.data.append((Message.Queue, url, keywords.copy()))
|
||||||
|
|
||||||
|
def handle_finalize(self):
|
||||||
|
self.file.close()
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def raises(obj):
|
|||||||
|
|
||||||
|
|
||||||
def combine_dict(a, b):
|
def combine_dict(a, b):
|
||||||
"""Recursively combine the contents of b into a"""
|
"""Recursively combine the contents of 'b' into 'a'"""
|
||||||
for key, value in b.items():
|
for key, value in b.items():
|
||||||
if key in a and isinstance(value, dict) and isinstance(a[key], dict):
|
if key in a and isinstance(value, dict) and isinstance(a[key], dict):
|
||||||
combine_dict(a[key], value)
|
combine_dict(a[key], value)
|
||||||
@@ -66,6 +66,20 @@ def combine_dict(a, b):
|
|||||||
return a
|
return a
|
||||||
|
|
||||||
|
|
||||||
|
def transform_dict(a, func):
|
||||||
|
"""Recursively apply 'func' to all values in 'a'"""
|
||||||
|
for key, value in a.items():
|
||||||
|
if isinstance(value, dict):
|
||||||
|
transform_dict(value, func)
|
||||||
|
else:
|
||||||
|
a[key] = func(value)
|
||||||
|
|
||||||
|
|
||||||
|
def number_to_string(value):
|
||||||
|
"""Convert numbers (int, float) to string; Return everything else as is."""
|
||||||
|
return str(value) if isinstance(value, (int, float)) else value
|
||||||
|
|
||||||
|
|
||||||
def expand_path(path):
|
def expand_path(path):
|
||||||
"""Expand environment variables and tildes (~)"""
|
"""Expand environment variables and tildes (~)"""
|
||||||
if not path:
|
if not path:
|
||||||
|
|||||||
Reference in New Issue
Block a user