github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[joyreactor] improve error handling for faulty JSON (#148)

Browse Source 
- remove all ASCII escape codes, not just \n and \r
- ignore faulty posts instead of letting the exception propagate
This commit is contained in:
Mike Fährmann
2019-01-03 16:27:53 +01:00
parent a36f52a730
commit 8753627ef4
1 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
gallery_dl/extractor/joyreactor.py
View File

@@ -62,8 +62,7 @@ class JoyreactorExtractor(Extractor):
return return
url = self.root + path url = self.root + path
@staticmethod def _parse_post(self, post):
def _parse_post(post):
post, _, script = post.partition('<script type="application/ld+json">') post, _, script = post.partition('<script type="application/ld+json">')
images = text.extract_iter(post, '<div class="image">', '</div>') images = text.extract_iter(post, '<div class="image">', '</div>')
script = script[:script.index("</")].strip() script = script[:script.index("</")].strip()
@@ -71,10 +70,13 @@ class JoyreactorExtractor(Extractor):
try: try:
data = json.loads(script) data = json.loads(script)
except ValueError: except ValueError:
data = json.loads(script try:
.replace("\\", "\\\\") mapping = dict.fromkeys(range(32))
.replace("\n", "") script = script.translate(mapping).replace("\\", "\\\\")
.replace("\r", "")) data = json.loads(script)
except ValueError as exc:
self.log.warning("Unable to parse post: %s", exc)
return
num = 0 num = 0
date = data["datePublished"] date = data["datePublished"]