github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve handling of text:... URLs

Browse Source 
- don't require // after the colon
- open output files in text mode
This commit is contained in:
Mike Fährmann
2017-05-12 14:10:25 +02:00
parent 677c8ced11
commit 107d29ad8a
6 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
gallery_dl/downloader/text.py
View File

@@ -1,12 +1,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2016 Mike Fährmann # Copyright 2014-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation. # published by the Free Software Foundation.
"""Downloader module for text:// urls""" """Downloader module for text: urls"""
from .common import BasicDownloader from .common import BasicDownloader
@@ -26,7 +26,7 @@ class Downloader(BasicDownloader):
self.out.start(pathfmt.path) self.out.start(pathfmt.path)
self.downloading = True self.downloading = True
with pathfmt.open() as file: with pathfmt.open("w") as file:
file.write(bytes(url[7:], "utf-8")) file.write(url[5:])
self.downloading = False self.downloading = False
self.out.success(pathfmt.path, 0) self.out.success(pathfmt.path, 0)

8
gallery_dl/extractor/deviantart.py
View File

@@ -165,8 +165,8 @@ class DeviantartFavoriteExtractor(DeviantartExtractor):
r"(?:/((\d+)/([^/?]+)|\?catpath=/))?"] r"(?:/((\d+)/([^/?]+)|\?catpath=/))?"]
test = [ test = [
("http://rosuuri.deviantart.com/favourites/58951174/Useful", { ("http://rosuuri.deviantart.com/favourites/58951174/Useful", {
"url": "2545427f52012a8b9b07c95ca5c91002d5bf4f18", "url": "85d1dd231be0b880c69dc947b564595fe3dd8248",
"keyword": "7ba0e75aeeb0f51541c4a2411410f8e3b3717641", "keyword": "9ae736fd906d7b3a41d81dd5f2488fd8537a5858",
}), }),
("http://h3813067.deviantart.com/favourites/", { ("http://h3813067.deviantart.com/favourites/", {
"url": "71345ce3bef5b19bd2a56d7b96e6b5ddba747c2e", "url": "71345ce3bef5b19bd2a56d7b96e6b5ddba747c2e",
@@ -216,7 +216,7 @@ class DeviantartJournalExtractor(DeviantartExtractor):
subcategory = "journal" subcategory = "journal"
pattern = [r"(?:https?://)?([^.]+)\.deviantart\.com/journal/?$"] pattern = [r"(?:https?://)?([^.]+)\.deviantart\.com/journal/?$"]
test = [("http://shimoda7.deviantart.com/journal/", { test = [("http://shimoda7.deviantart.com/journal/", {
"url": "05204bddf5ebba330d73cec76bcd55b1249c6159", "url": "d046614f41849e4a2a8d8a20c5d00cbe814ca6f2",
"keyword": "8434f8bdd4b38634e206c8689a0906ac10c3fa77", "keyword": "8434f8bdd4b38634e206c8689a0906ac10c3fa77",
})] })]
@@ -343,7 +343,7 @@ class DeviantartAPI():
return return
JOURNAL_TEMPLATE = """text://<!DOCTYPE html> JOURNAL_TEMPLATE = """text:<!DOCTYPE html>
<html> <html>
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">

1
gallery_dl/extractor/imgchili.py
View File

@@ -52,7 +52,6 @@ class ImgchiliImageExtractor(ImgchiliExtractor):
"89427136_test___quot;___gt;.png"), { "89427136_test___quot;___gt;.png"), {
"url": "b93d92a6b58eb30a7ff6f9729cb748d25fea0c86", "url": "b93d92a6b58eb30a7ff6f9729cb748d25fea0c86",
"keyword": "376c4584dfae7d7d2e88687d4ee9618bbfd0a35c", "keyword": "376c4584dfae7d7d2e88687d4ee9618bbfd0a35c",
"content": "3b2a74be551b0b5549c831c9994760ef35138b22",
})] })]
def get_job_metadata(self, page): def get_job_metadata(self, page):

2
gallery_dl/extractor/pixiv.py
View File

@@ -62,7 +62,7 @@ class PixivUserExtractor(Extractor):
work["extension"] = "zip" work["extension"] = "zip"
yield Message.Url, url, work yield Message.Url, url, work
work["extension"] = "txt" work["extension"] = "txt"
yield Message.Url, "text://"+framelist, work yield Message.Url, "text:"+framelist, work
elif work["page_count"] == 1: elif work["page_count"] == 1:
yield Message.Url, work["url"], work yield Message.Url, work["url"], work

4
gallery_dl/util.py
View File

@@ -150,9 +150,9 @@ class PathFormat():
elif not skipmode: elif not skipmode:
self.exists = lambda: False self.exists = lambda: False
def open(self): def open(self, mode="wb"):
"""Open file to 'realpath' and return a corresponding file object""" """Open file to 'realpath' and return a corresponding file object"""
return open(self.realpath, "wb") return open(self.realpath, mode)
def exists(self): def exists(self):
"""Return True if 'path' is complete and refers to an existing path""" """Return True if 'path' is complete and refers to an existing path"""

2
test/test_extractors.py
View File

@@ -51,7 +51,7 @@ skip = [
# dont work on travis-ci # dont work on travis-ci
"exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie",
# temporary issues # temporary issues
"yomanga",
] ]
# enable selective testing for direct calls # enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1: if __name__ == '__main__' and len(sys.argv) > 1: