From 6d8b191ea7743cbac06b5525d34f399af12c5d2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 13 Apr 2018 19:21:32 +0200
Subject: [PATCH 01/21] improve 'parse_query()' and add tests

- another irrelevant micro-optimization !
- use urllib.parse.parse_qsl directly instead of parse_qs, which
  just packs the results of parse_qsl in a different data structure
- reduced memory requirements since no additional dict and lists are
  created
---
 gallery_dl/text.py |  6 +++++-
 test/test_text.py  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/gallery_dl/text.py b/gallery_dl/text.py
index 82c56a8b..7e5cb29f 100644
--- a/gallery_dl/text.py
+++ b/gallery_dl/text.py
@@ -130,7 +130,11 @@ def extract_iter(txt, begin, end, pos=0):
 
 def parse_query(qs):
     """Parse a query string into key-value pairs"""
-    return {key: vlist[0] for key, vlist in urllib.parse.parse_qs(qs).items()}
+    result = {}
+    for key, value in urllib.parse.parse_qsl(qs):
+        if key not in result:
+            result[key] = value
+    return result
 
 
 if os.name == "nt":
diff --git a/test/test_text.py b/test/test_text.py
index e26dde76..767952fd 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -123,6 +123,37 @@ class TestText(unittest.TestCase):
         result = ["c", "b", "a", "d"]
         self.assertEqual(list(text.extract_iter(txt, "[", "]")), result)
 
+    def test_parse_query(self):
+        # standard stuff
+        self.assertEqual(
+            text.parse_query(""), {})
+        self.assertEqual(
+            text.parse_query("foo=1"), {"foo": "1"})
+        self.assertEqual(
+            text.parse_query("foo=1&bar=2"), {"foo": "1", "bar": "2"})
+
+        # missing value
+        self.assertEqual(
+            text.parse_query("bar"), {})
+        self.assertEqual(
+            text.parse_query("foo=1&bar"), {"foo": "1"})
+        self.assertEqual(
+            text.parse_query("foo=1&bar&baz=3"), {"foo": "1", "baz": "3"})
+
+        # keys with identical names
+        self.assertEqual(
+            text.parse_query("foo=1&foo=2"), {"foo": "1"})
+        self.assertEqual(
+            text.parse_query("foo=1&bar=2&foo=3&bar=4"),
+            {"foo": "1", "bar": "2"},
+        )
+
+        # non-string arguments
+        self.assertEqual(text.parse_query(()), {})
+        self.assertEqual(text.parse_query([]), {})
+        self.assertEqual(text.parse_query({}), {})
+        self.assertEqual(text.parse_query(None), {})
+
 
 if __name__ == '__main__':
     unittest.main()

From e3f2bd4087b953a8df8e8e1c925f59225752c4d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sat, 14 Apr 2018 20:56:21 +0200
Subject: [PATCH 02/21] add tests for 'text.clean_xml()' and improve it

---
 gallery_dl/text.py | 16 ++++++++++++----
 test/test_text.py  | 28 ++++++++++++++++++++++++++--
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/gallery_dl/text.py b/gallery_dl/text.py
index 7e5cb29f..e439c2b8 100644
--- a/gallery_dl/text.py
+++ b/gallery_dl/text.py
@@ -15,14 +15,22 @@ import html
 import urllib.parse
 
 
-INVALID_XML_CHARS = (1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 14, 15, 16, 17, 18,
-                     19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31)
+INVALID_XML_CHARS = (
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f",
+)
 
 
 def clean_xml(xmldata, repl=""):
-    """Replace/Remove invalid control characters in XML data"""
+    """Replace/Remove invalid control characters in 'xmldata'"""
+    if not isinstance(xmldata, str):
+        try:
+            xmldata = "".join(xmldata)
+        except TypeError:
+            return ""
     for char in INVALID_XML_CHARS:
-        char = chr(char)
         if char in xmldata:
             xmldata = xmldata.replace(char, repl)
     return xmldata
diff --git a/test/test_text.py b/test/test_text.py
index 767952fd..c4b02969 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015 Mike Fährmann
+# Copyright 2015-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,11 +9,35 @@
 
 import unittest
 import sys
-import gallery_dl.text as text
+
+from gallery_dl import text
 
 
 class TestText(unittest.TestCase):
 
+    def test_clean_xml(self, f=text.clean_xml):
+        # standard usage
+        self.assertEqual(f(""), "")
+        self.assertEqual(f("foo"), "foo")
+        self.assertEqual(f("\tfoo\nbar\r"), "\tfoo\nbar\r")
+        self.assertEqual(f("<foo>\ab\ba\fr\v</foo>"), "<foo>bar</foo>")
+
+        # 'repl' argument
+        repl = "#"
+        self.assertEqual(f("", repl), "")
+        self.assertEqual(f("foo", repl), "foo")
+        self.assertEqual(f("\tfoo\nbar\r", repl), "\tfoo\nbar\r")
+        self.assertEqual(
+            f("<foo>\ab\ba\fr\v</foo>", repl), "<foo>#b#a#r#</foo>")
+
+        # removal of all illegal control characters
+        value = "".join(chr(x) for x in range(32))
+        self.assertEqual(f(value), "\t\n\r")
+
+        # 'invalid' arguments
+        for value in ((), [], {}, None, 1, 2.3):
+            self.assertEqual(f(value), "")
+
     def test_remove_html(self):
         cases = (
             "Hello World.",

From 27eab4e4672caf1a2266c85161ff6ef79d7005ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sat, 14 Apr 2018 22:09:42 +0200
Subject: [PATCH 03/21] rewrite text tests and improve functions

- test more edge cases
- consistently return an empty string for invalid arguments
- remove the ungreedy-flag in 'remove_html()'
---
 gallery_dl/text.py |  26 +++--
 test/test_text.py  | 249 ++++++++++++++++++++++++++++-----------------
 2 files changed, 173 insertions(+), 102 deletions(-)

diff --git a/gallery_dl/text.py b/gallery_dl/text.py
index e439c2b8..9ce7ef1f 100644
--- a/gallery_dl/text.py
+++ b/gallery_dl/text.py
@@ -36,17 +36,20 @@ def clean_xml(xmldata, repl=""):
     return xmldata
 
 
-def remove_html(text):
+def remove_html(txt):
     """Remove html-tags from a string"""
-    return " ".join(re.sub("<[^>]+?>", " ", text).split())
+    try:
+        return " ".join(re.sub("<[^>]+>", " ", txt).split())
+    except TypeError:
+        return ""
 
 
 def filename_from_url(url):
     """Extract the last part of an url to use as a filename"""
     try:
         return urllib.parse.urlsplit(url).path.rpartition("/")[2]
-    except ValueError:
-        return url
+    except (TypeError, AttributeError):
+        return ""
 
 
 def nameext_from_url(url, data=None):
@@ -64,7 +67,7 @@ def clean_path_windows(path):
     try:
         return re.sub(r'[<>:"\\/|?*]', "_", path)
     except TypeError:
-        return path
+        return ""
 
 
 def clean_path_posix(path):
@@ -72,7 +75,7 @@ def clean_path_posix(path):
     try:
         return path.replace("/", "_")
     except AttributeError:
-        return path
+        return ""
 
 
 def shorten_path(path, limit=255, encoding=sys.getfilesystemencoding()):
@@ -112,7 +115,7 @@ def extract(txt, begin, end, pos=0):
         first = txt.index(begin, pos) + len(begin)
         last = txt.index(end, first)
         return txt[first:last], last+len(end)
-    except ValueError:
+    except (ValueError, TypeError, AttributeError):
         return None, pos
 
 
@@ -139,9 +142,12 @@ def extract_iter(txt, begin, end, pos=0):
 def parse_query(qs):
     """Parse a query string into key-value pairs"""
     result = {}
-    for key, value in urllib.parse.parse_qsl(qs):
-        if key not in result:
-            result[key] = value
+    try:
+        for key, value in urllib.parse.parse_qsl(qs):
+            if key not in result:
+                result[key] = value
+    except AttributeError:
+        pass
     return result
 
 
diff --git a/test/test_text.py b/test/test_text.py
index c4b02969..4afa058d 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -13,6 +13,9 @@ import sys
 from gallery_dl import text
 
 
+INVALID = ((), [], {}, None, 1, 2.3)
+
+
 class TestText(unittest.TestCase):
 
     def test_clean_xml(self, f=text.clean_xml):
@@ -35,59 +38,85 @@ class TestText(unittest.TestCase):
         self.assertEqual(f(value), "\t\n\r")
 
         # 'invalid' arguments
-        for value in ((), [], {}, None, 1, 2.3):
+        for value in INVALID:
             self.assertEqual(f(value), "")
 
-    def test_remove_html(self):
-        cases = (
-            "Hello World.",
-            " Hello  World. ",
-            "Hello<br/>World.",
-            "<div><span class='a'>Hello</span><strong>World.</strong></div>"
-        )
+    def test_remove_html(self, f=text.remove_html):
         result = "Hello World."
-        for case in cases:
-            self.assertEqual(text.remove_html(case), result)
 
-    def test_filename_from_url(self):
-        cases = (
-            "http://example.org/v2/filename.ext",
-            "http://example.org/v2/filename.ext?param=value#fragment",
-            "example.org/filename.ext",
-            "/filename.ext",
-            "filename.ext",
-        )
+        # standard usage
+        self.assertEqual(f(""), "")
+        self.assertEqual(f("Hello World."), result)
+        self.assertEqual(f(" Hello  World.  "), result)
+        self.assertEqual(f("Hello<br/>World."), result)
+        self.assertEqual(
+            f("<div><b class='a'>Hello</b><i>World.</i></div>"), result)
+
+        # empty HTML
+        self.assertEqual(f("<div></div>"), "")
+        self.assertEqual(f(" <div>   </div> "), "")
+
+        # malformed HTML
+        self.assertEqual(f("<div</div>"), "")
+        self.assertEqual(f("<div<Hello World.</div>"), "")
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), "")
+
+    def test_filename_from_url(self, f=text.filename_from_url):
         result = "filename.ext"
-        for case in cases:
-            self.assertEqual(text.filename_from_url(case), result)
 
-    def test_nameext_from_url(self):
-        cases = (
-            "http://example.org/v2/filename.ext",
-            "http://example.org/v2/filename.ext?param=value#fragment",
-            "example.org/filename.ext",
-            "/filename.ext",
-            "filename.ext",
-        )
-        result = {
-            "filename" : "filename.ext",
-            "name"     : "filename",
-            "extension": "ext",
-        }
-        for case in cases:
-            self.assertEqual(text.nameext_from_url(case), result)
+        # standard usage
+        self.assertEqual(f(""), "")
+        self.assertEqual(f("filename.ext"), result)
+        self.assertEqual(f("/filename.ext"), result)
+        self.assertEqual(f("example.org/filename.ext"), result)
+        self.assertEqual(f("http://example.org/v2/filename.ext"), result)
+        self.assertEqual(
+            f("http://example.org/v2/filename.ext?param=value#frag"), result)
 
-    def test_clean_path(self):
-        cases = {
-            "Hello World." : ("Hello World.", "Hello World."),
-            "Hello/World/.": ("Hello_World_.", "Hello_World_."),
-            r'<Hello>:|"World\*?': (
-                '_Hello____World___', r'<Hello>:|"World\*?'
-            ),
-        }
-        for case, result in cases.items():
-            self.assertEqual(text.clean_path_windows(case), result[0])
-            self.assertEqual(text.clean_path_posix(case), result[1])
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), "")
+
+    def test_nameext_from_url(self, f=text.nameext_from_url):
+        empty = {"filename": "", "name": "", "extension": ""}
+        result = {"filename": "filename.ext",
+                  "name": "filename", "extension": "ext"}
+
+        # standard usage
+        self.assertEqual(f(""), empty)
+        self.assertEqual(f("filename.ext"), result)
+        self.assertEqual(f("/filename.ext"), result)
+        self.assertEqual(f("example.org/filename.ext"), result)
+        self.assertEqual(f("http://example.org/v2/filename.ext"), result)
+        self.assertEqual(
+            f("http://example.org/v2/filename.ext?param=value#frag"), result)
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), empty)
+
+    def test_clean_path_windows(self, f=text.clean_path_windows):
+        self.assertEqual(f(""), "")
+        self.assertEqual(f("foo"), "foo")
+        self.assertEqual(f("foo/bar"), "foo_bar")
+        self.assertEqual(f("foo<>:\"\\/|?*bar"), "foo_________bar")
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), "")
+
+    def test_clean_path_posix(self, f=text.clean_path_posix):
+        self.assertEqual(f(""), "")
+        self.assertEqual(f("foo"), "foo")
+        self.assertEqual(f("foo/bar"), "foo_bar")
+        self.assertEqual(f("foo<>:\"\\/|?*bar"), "foo<>:\"\\_|?*bar")
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), "")
 
     def test_shorten_path(self):
         cases = {
@@ -115,68 +144,104 @@ class TestText(unittest.TestCase):
             self.assertEqual(fname, result)
             self.assertTrue(len(fname.encode(enc)) <= 255)
 
-    def test_extract(self):
-        cases = {
-            ("<a><b>", "<", ">")   : ("a", 3),
-            ("<a><b>", "X", ">")   : (None, 0),
-            ("<a><b>", "<", "X")   : (None, 0),
-            ("<a><b>", "<", ">", 3): ("b", 6),
-            ("<a><b>", "X", ">", 3): (None, 3),
-            ("<a><b>", "<", "X", 3): (None, 3),
-        }
-        for case, result in cases.items():
-            self.assertEqual(text.extract(*case), result)
+    def test_extract(self, f=text.extract):
+        txt = "<a><b>"
+        self.assertEqual(f(txt, "<", ">"), ("a", 3))
+        self.assertEqual(f(txt, "X", ">"), (None, 0))
+        self.assertEqual(f(txt, "<", "X"), (None, 0))
 
-    def test_extract_all(self):
+        # 'pos' argument
+        for i in range(1, 4):
+            self.assertEqual(f(txt, "<", ">", i), ("b", 6))
+        for i in range(4, 10):
+            self.assertEqual(f(txt, "<", ">", i), (None, i))
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value   , "<"  , ">")  , (None, 0))
+            self.assertEqual(f(txt, value, ">")  , (None, 0))
+            self.assertEqual(f(txt, "<"  , value), (None, 0))
+
+    def test_extract_all(self, f=text.extract_all):
         txt = "[c][b][a]: xyz! [d][e"
-        result = ({
-            "A": "a",
-            "B": "b",
-            "X": "xyz",
-            "E": "xtra",
-        }, 15)
-        self.assertEqual(text.extract_all(txt, (
-            (None, "[", "]"),
-            ("B" , "[", "]"),
-            ("A" , "[", "]"),
-            ("X" , ": ", "!"),
-        ), values={"E": "xtra"}), result)
 
-    def test_extract_iter(self):
+        self.assertEqual(
+            f(txt, ()), ({}, 0))
+        self.assertEqual(
+            f(txt, (("C", "[", "]"), ("B", "[", "]"), ("A", "[", "]"))),
+            ({"A": "a", "B": "b", "C": "c"}, 9),
+        )
+
+        # 'None' as field name
+        self.assertEqual(
+            f(txt, ((None, "[", "]"), (None, "[", "]"), ("A", "[", "]"))),
+            ({"A": "a"}, 9),
+        )
+        self.assertEqual(
+            f(txt, ((None, "[", "]"), (None, "[", "]"), (None, "[", "]"))),
+            ({}, 9),
+        )
+
+        # failed matches
+        self.assertEqual(
+            f(txt, (("C", "[", "]"), ("X", "X", "X"), ("B", "[", "]"))),
+            ({"B": "b", "C": "c", "X": None}, 6),
+        )
+
+        # 'pos' argument
+        self.assertEqual(
+            f(txt, (("B", "[", "]"), ("A", "[", "]")), pos=1),
+            ({"A": "a", "B": "b"}, 9),
+        )
+
+        # 'values' argument
+        self.assertEqual(
+            f(txt, (("C", "[", "]"),), values={"A": "a", "B": "b"}),
+            ({"A": "a", "B": "b", "C": "c"}, 3),
+        )
+
+        vdict = {}
+        rdict, pos = f(txt, (), values=vdict)
+        self.assertIs(vdict, rdict)
+
+    def test_extract_iter(self, f=text.extract_iter):
         txt = "[c][b][a]: xyz! [d][e"
-        result = ["c", "b", "a", "d"]
-        self.assertEqual(list(text.extract_iter(txt, "[", "]")), result)
 
-    def test_parse_query(self):
-        # standard stuff
+        def g(*args):
+            return list(f(*args))
+
         self.assertEqual(
-            text.parse_query(""), {})
+            g("", "[", "]"), [])
         self.assertEqual(
-            text.parse_query("foo=1"), {"foo": "1"})
+            g("[a]", "[", "]"), ["a"])
         self.assertEqual(
-            text.parse_query("foo=1&bar=2"), {"foo": "1", "bar": "2"})
+            g(txt, "[", "]"), ["c", "b", "a", "d"])
+        self.assertEqual(
+            g(txt, "X", "X"), [])
+        self.assertEqual(
+            g(txt, "[", "]", 6), ["a", "d"])
+
+    def test_parse_query(self, f=text.parse_query):
+        # standard usage
+        self.assertEqual(f(""), {})
+        self.assertEqual(f("foo=1"), {"foo": "1"})
+        self.assertEqual(f("foo=1&bar=2"), {"foo": "1", "bar": "2"})
 
         # missing value
-        self.assertEqual(
-            text.parse_query("bar"), {})
-        self.assertEqual(
-            text.parse_query("foo=1&bar"), {"foo": "1"})
-        self.assertEqual(
-            text.parse_query("foo=1&bar&baz=3"), {"foo": "1", "baz": "3"})
+        self.assertEqual(f("bar"), {})
+        self.assertEqual(f("foo=1&bar"), {"foo": "1"})
+        self.assertEqual(f("foo=1&bar&baz=3"), {"foo": "1", "baz": "3"})
 
         # keys with identical names
+        self.assertEqual(f("foo=1&foo=2"), {"foo": "1"})
         self.assertEqual(
-            text.parse_query("foo=1&foo=2"), {"foo": "1"})
-        self.assertEqual(
-            text.parse_query("foo=1&bar=2&foo=3&bar=4"),
+            f("foo=1&bar=2&foo=3&bar=4"),
             {"foo": "1", "bar": "2"},
         )
 
-        # non-string arguments
-        self.assertEqual(text.parse_query(()), {})
-        self.assertEqual(text.parse_query([]), {})
-        self.assertEqual(text.parse_query({}), {})
-        self.assertEqual(text.parse_query(None), {})
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value), {})
 
 
 if __name__ == '__main__':

From 4ffa94f634cbdd6d566defb2bcaf97b418e08c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 15 Apr 2018 18:44:13 +0200
Subject: [PATCH 04/21] remove 'shorten_path()' and 'shorten_filename()'

---
 gallery_dl/text.py | 14 --------------
 test/test_text.py  | 27 ---------------------------
 2 files changed, 41 deletions(-)

diff --git a/gallery_dl/text.py b/gallery_dl/text.py
index 9ce7ef1f..4bed863a 100644
--- a/gallery_dl/text.py
+++ b/gallery_dl/text.py
@@ -8,7 +8,6 @@
 
 """Collection of functions that work in strings/text"""
 
-import sys
 import re
 import os.path
 import html
@@ -78,19 +77,6 @@ def clean_path_posix(path):
         return ""
 
 
-def shorten_path(path, limit=255, encoding=sys.getfilesystemencoding()):
-    """Shorten a path segment to at most 'limit' bytes"""
-    return (path.encode(encoding)[:limit]).decode(encoding, "ignore")
-
-
-def shorten_filename(fname, limit=255, encoding=sys.getfilesystemencoding()):
-    """Shorten filename to at most 'limit' bytes while preserving extension"""
-    name, extension = os.path.splitext(fname)
-    bext = extension.encode(encoding)
-    bname = name.encode(encoding)[:limit-len(bext)]
-    return bname.decode(encoding, "ignore") + extension
-
-
 def extract(txt, begin, end, pos=0):
     """Extract the text between 'begin' and 'end' from 'txt'
 
diff --git a/test/test_text.py b/test/test_text.py
index 4afa058d..b07dff10 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -8,7 +8,6 @@
 # published by the Free Software Foundation.
 
 import unittest
-import sys
 
 from gallery_dl import text
 
@@ -118,32 +117,6 @@ class TestText(unittest.TestCase):
         for value in INVALID:
             self.assertEqual(f(value), "")
 
-    def test_shorten_path(self):
-        cases = {
-            "dirname": "dirname",
-            "X"*255: "X"*255,
-            "X"*256: "X"*255,
-            "Ä"*255: "Ä"*127,
-        }
-        enc = sys.getfilesystemencoding()
-        for case, result in cases.items():
-            self.assertEqual(text.shorten_path(case), result)
-            self.assertTrue(len(text.shorten_path(case).encode(enc)) <= 255)
-
-    def test_shorten_filename(self):
-        self.maxDiff = None
-        cases = {
-            "filename.ext": "filename.ext",
-            "X"*251 + ".ext": "X"*251 + ".ext",
-            "X"*255 + ".ext": "X"*251 + ".ext",
-            "Ä"*251 + ".ext": "Ä"*125 + ".ext",
-        }
-        enc = sys.getfilesystemencoding()
-        for case, result in cases.items():
-            fname = text.shorten_filename(case)
-            self.assertEqual(fname, result)
-            self.assertTrue(len(fname.encode(enc)) <= 255)
-
     def test_extract(self, f=text.extract):
         txt = "<a><b>"
         self.assertEqual(f(txt, "<", ">"), ("a", 3))

From 728c64a3fb2fce8e4c75bbc5d39d255852ddf587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 15 Apr 2018 18:58:32 +0200
Subject: [PATCH 05/21] [tumblr] rename 'offset' to 'num and adjust formats

Trying to somehow emulate Tumblr filenames is a bad idea ...
---
 gallery_dl/extractor/tumblr.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py
index f853d964..bb951856 100644
--- a/gallery_dl/extractor/tumblr.py
+++ b/gallery_dl/extractor/tumblr.py
@@ -52,8 +52,8 @@ class TumblrExtractor(Extractor):
     """Base class for tumblr extractors"""
     category = "tumblr"
     directory_fmt = ["{category}", "{name}"]
-    filename_fmt = "{category}_{blog_name}_{id}o{offset}.{extension}"
-    archive_fmt = "{id}_{offset}"
+    filename_fmt = "{category}_{blog_name}_{id}_{num:>02}.{extension}"
+    archive_fmt = "{id}_{num}"
 
     def __init__(self, match):
         Extractor.__init__(self)
@@ -87,7 +87,7 @@ class TumblrExtractor(Extractor):
             post["reblogged"] = reblog
 
             post["blog"] = blog
-            post["offset"] = 0
+            post["num"] = 0
 
             if "trail" in post:
                 del post["trail"]
@@ -148,14 +148,14 @@ class TumblrExtractor(Extractor):
     @staticmethod
     def _prepare(url, post):
         text.nameext_from_url(url, post)
-        post["offset"] += 1
+        post["num"] += 1
         post["hash"] = post["name"].partition("_")[2]
         return Message.Url, url, post
 
     @staticmethod
     def _prepare_image(url, post):
         text.nameext_from_url(url, post)
-        post["offset"] += 1
+        post["num"] += 1
 
         parts = post["name"].split("_")
         post["hash"] = parts[1] if parts[1] != "inline" else parts[2]
@@ -237,7 +237,7 @@ class TumblrLikesExtractor(TumblrExtractor):
     """Extractor for images from a tumblr-user by tag"""
     subcategory = "likes"
     directory_fmt = ["{category}", "{name}", "likes"]
-    archive_fmt = "f_{blog[name]}_{id}_{offset}"
+    archive_fmt = "f_{blog[name]}_{id}_{num}"
     pattern = [BASE_PATTERN + r"/likes"]
     test = [("http://mikf123.tumblr.com/likes", {
         "count": 1,

From ff643793bd7a279035689e3a9fecc5ac497ab088 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 19 Apr 2018 21:32:10 +0200
Subject: [PATCH 06/21] improve and document cloudflare bypass code

---
 gallery_dl/cloudflare.py | 60 ++++++++++++++++++++++++++--------------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/gallery_dl/cloudflare.py b/gallery_dl/cloudflare.py
index 53cc58e8..676e723f 100644
--- a/gallery_dl/cloudflare.py
+++ b/gallery_dl/cloudflare.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2017 Mike Fährmann
+# Copyright 2015-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -8,6 +8,7 @@
 
 """Methods to access sites behind Cloudflare protection"""
 
+import re
 import time
 import operator
 import urllib.parse
@@ -30,6 +31,7 @@ def request_func(self, *args, **kwargs):
 
 
 def solve_challenge(session, response):
+
     session.headers["Referer"] = response.url
     page = response.text
     params = text.extract_all(page, (
@@ -37,6 +39,7 @@ def solve_challenge(session, response):
         ('pass'    , 'name="pass" value="', '"'),
     ))[0]
     params["jschl_answer"] = solve_jschl(response.url, page)
+
     time.sleep(4)
     url = urllib.parse.urljoin(response.url, "/cdn-cgi/l/chk_jschl")
     return session.get(url, params=params)
@@ -44,51 +47,66 @@ def solve_challenge(session, response):
 
 def solve_jschl(url, page):
     """Solve challenge to get 'jschl_answer' value"""
+
+    # build variable name
+    # e.g. '...f, wqnVscP={"DERKbJk":+(...' --> wqnVscP.DERKbJk
     data, pos = text.extract_all(page, (
         ('var' , ',f, ', '='),
         ('key' , '"', '"'),
         ('expr', ':', '}'),
     ))
-    solution = evaluate_expression(data["expr"])
     variable = "{}.{}".format(data["var"], data["key"])
     vlength = len(variable)
+
+    # evaluate the initial expression
+    solution = evaluate_expression(data["expr"])
+
+    # iterator over all remaining expressions
+    # and combine their values in 'solution'
     expressions = text.extract(
-        page, "'challenge-form');", "f.submit();", pos
-    )[0]
+        page, "'challenge-form');", "f.submit();", pos)[0]
     for expr in expressions.split(";")[1:]:
+
         if expr.startswith(variable):
+            # select arithmetc function based on operator (+, -, *)
             func = operator_functions[expr[vlength]]
+            # evaluate the rest of the expression
             value = evaluate_expression(expr[vlength+2:])
+            # combine the expression value with our current solution
             solution = func(solution, value)
+
         elif expr.startswith("a.value"):
+            # add length of the hostname, i.e. add 11 for 'example.org'
             solution += len(urllib.parse.urlsplit(url).netloc)
+
             if ".toFixed(" in expr:
+                # trim the solution to 10 decimal places
+                # and strip trailing zeros
                 solution = "{:.10f}".format(solution).rstrip("0")
+
             return solution
 
 
-def evaluate_expression(expr):
+def evaluate_expression(expr, split_re=re.compile(r"\(+([^)]*)\)")):
     """Evaluate a Javascript expression for the challenge"""
+
     if "/" in expr:
+        # split the expression in numerator and denominator subexpressions,
+        # evaluate them separately,
+        # and return their fraction-result
         num, _, denom = expr.partition("/")
         return evaluate_expression(num) / evaluate_expression(denom)
 
-    stack = []
-    ranges = []
-    value = ""
-    for index, char in enumerate(expr):
-        if char == "(":
-            stack.append(index+1)
-        elif char == ")":
-            begin = stack.pop()
-            if stack:
-                ranges.append((begin, index))
-    for subexpr in [expr[begin:end] for begin, end in ranges] or (expr,):
-        num = 0
-        for part in subexpr.split("[]"):
-            num += expression_values[part]
-        value += str(num)
-    return int(value)
+    # iterate over all subexpressions,
+    # evaluate them,
+    # and accumulate their values in 'result'
+    result = ""
+    for subexpr in split_re.findall(expr):
+        result += str(sum(
+            expression_values[part]
+            for part in subexpr.split("[]")
+        ))
+    return int(result)
 
 
 operator_functions = {

From cc36f8858689bec4e00eee0979136fce6b929e81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 20 Apr 2018 14:53:21 +0200
Subject: [PATCH 07/21] rename safe_int to parse_int; move parse_* to text
 module

---
 gallery_dl/downloader/http.py           |  6 ++--
 gallery_dl/extractor/artstation.py      |  4 +--
 gallery_dl/extractor/deviantart.py      |  4 +--
 gallery_dl/extractor/dynastyscans.py    |  4 +--
 gallery_dl/extractor/exhentai.py        | 18 +++++------
 gallery_dl/extractor/fallenangels.py    |  4 +--
 gallery_dl/extractor/foolslide.py       |  8 ++---
 gallery_dl/extractor/gelbooru.py        |  4 +--
 gallery_dl/extractor/hbrowse.py         | 12 +++----
 gallery_dl/extractor/hentai2read.py     | 15 ++++-----
 gallery_dl/extractor/hentaifoundry.py   |  8 ++---
 gallery_dl/extractor/hentaihere.py      | 14 ++++-----
 gallery_dl/extractor/hitomi.py          |  2 +-
 gallery_dl/extractor/imagefap.py        |  4 +--
 gallery_dl/extractor/kissmanga.py       | 10 +++---
 gallery_dl/extractor/komikcast.py       |  8 ++---
 gallery_dl/extractor/mangadex.py        | 20 ++++++------
 gallery_dl/extractor/mangafox.py        |  4 +--
 gallery_dl/extractor/mangahere.py       | 14 ++++-----
 gallery_dl/extractor/mangapark.py       | 12 +++----
 gallery_dl/extractor/mangareader.py     | 12 +++----
 gallery_dl/extractor/mangastream.py     |  6 ++--
 gallery_dl/extractor/nhentai.py         |  4 +--
 gallery_dl/extractor/nijie.py           |  8 ++---
 gallery_dl/extractor/paheal.py          |  2 +-
 gallery_dl/extractor/pinterest.py       | 10 +++---
 gallery_dl/extractor/readcomiconline.py |  6 ++--
 gallery_dl/extractor/sankaku.py         | 14 ++++-----
 gallery_dl/extractor/seiga.py           |  8 ++---
 gallery_dl/extractor/senmanga.py        |  4 +--
 gallery_dl/extractor/slideshare.py      |  4 +--
 gallery_dl/extractor/spectrumnexus.py   |  8 ++---
 gallery_dl/extractor/xvideos.py         | 10 +++---
 gallery_dl/text.py                      | 32 ++++++++++++++++++-
 gallery_dl/util.py                      | 28 +----------------
 test/test_text.py                       | 42 +++++++++++++++++++++++++
 test/test_util.py                       | 26 ---------------
 37 files changed, 210 insertions(+), 189 deletions(-)

diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index bf461ae2..b590485f 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -12,7 +12,7 @@ import time
 import mimetypes
 from requests.exceptions import ConnectionError, Timeout
 from .common import DownloaderBase
-from .. import util, exception
+from .. import text, exception
 
 
 class Downloader(DownloaderBase):
@@ -28,7 +28,7 @@ class Downloader(DownloaderBase):
         self.chunk_size = 16384
 
         if self.rate:
-            self.rate = util.parse_bytes(self.rate)
+            self.rate = text.parse_bytes(self.rate)
             if not self.rate:
                 self.log.warning("Invalid rate limit specified")
             elif self.rate < self.chunk_size:
@@ -61,7 +61,7 @@ class Downloader(DownloaderBase):
         else:
             self.response.raise_for_status()
 
-        return offset, util.safe_int(size)
+        return offset, text.parse_int(size)
 
     def receive(self, file):
         if self.rate:
diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py
index fbea9595..6f8dbd53 100644
--- a/gallery_dl/extractor/artstation.py
+++ b/gallery_dl/extractor/artstation.py
@@ -158,7 +158,7 @@ class ArtstationAlbumExtractor(ArtstationExtractor):
 
     def __init__(self, match):
         ArtstationExtractor.__init__(self, match)
-        self.album_id = util.safe_int(match.group(2))
+        self.album_id = text.parse_int(match.group(2))
 
     def metadata(self):
         userinfo = self.get_user_info(self.user)
@@ -256,7 +256,7 @@ class ArtstationChallengeExtractor(ArtstationExtractor):
     def _id_from_url(url):
         """Get an image's submission ID from its URL"""
         parts = url.split("/")
-        return util.safe_int("".join(parts[7:10]))
+        return text.parse_int("".join(parts[7:10]))
 
 
 class ArtstationSearchExtractor(ArtstationExtractor):
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 19f7a58d..ee156342 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.deviantart.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, exception
 from ..cache import cache, memcache
 import itertools
 import datetime
@@ -62,7 +62,7 @@ class DeviantartExtractor(Extractor):
 
             if "videos" in deviation:
                 video = max(deviation["videos"],
-                            key=lambda x: util.safe_int(x["quality"][:-1]))
+                            key=lambda x: text.parse_int(x["quality"][:-1]))
                 yield self.commit(deviation, video)
 
             if "flash" in deviation:
diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py
index bd9107ac..d63ddc0a 100644
--- a/gallery_dl/extractor/dynastyscans.py
+++ b/gallery_dl/extractor/dynastyscans.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters from https://dynasty-scans.com/"""
 
 from .common import ChapterExtractor
-from .. import text, util
+from .. import text
 import re
 import json
 
@@ -53,7 +53,7 @@ class DynastyscansChapterExtractor(ChapterExtractor):
 
         return {
             "manga": text.unescape(match.group(1)),
-            "chapter": util.safe_int(match.group(2)),
+            "chapter": text.parse_int(match.group(2)),
             "chapter_minor": match.group(3) or "",
             "title": text.unescape(match.group(4) or ""),
             "author": text.remove_html(author),
diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py
index 41eaeca1..2af58110 100644
--- a/gallery_dl/extractor/exhentai.py
+++ b/gallery_dl/extractor/exhentai.py
@@ -120,7 +120,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
         self.key = {}
         self.count = 0
         self.version, self.gid, self.token = match.groups()
-        self.gid = util.safe_int(self.gid)
+        self.gid = text.parse_int(self.gid)
 
     def items(self):
         self.login()
@@ -163,7 +163,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
         data["lang"] = util.language_to_code(data["language"])
         data["title"] = text.unescape(data["title"])
         data["title_jp"] = text.unescape(data["title_jp"])
-        data["count"] = util.safe_int(data["count"])
+        data["count"] = text.parse_int(data["count"])
         data["gallery_size"] = util.parse_bytes(
             data["gallery_size"].rstrip("Bb"))
         return data
@@ -245,17 +245,17 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
     def _parse_image_info(url):
         parts = url.split("/")[4].split("-")
         return {
-            "width": util.safe_int(parts[2]),
-            "height": util.safe_int(parts[3]),
-            "size": util.safe_int(parts[1]),
+            "width": text.parse_int(parts[2]),
+            "height": text.parse_int(parts[3]),
+            "size": text.parse_int(parts[1]),
         }
 
     @staticmethod
     def _parse_original_info(info):
         parts = info.lstrip().split(" ")
         return {
-            "width": util.safe_int(parts[0]),
-            "height": util.safe_int(parts[2]),
+            "width": text.parse_int(parts[0]),
+            "height": text.parse_int(parts[2]),
             "size": util.parse_bytes(parts[3] + parts[4][0]),
         }
 
@@ -274,7 +274,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor):
     def __init__(self, match):
         ExhentaiExtractor.__init__(self)
         self.params = text.parse_query(match.group(1) or "")
-        self.params["page"] = util.safe_int(self.params.get("page"))
+        self.params["page"] = text.parse_int(self.params.get("page"))
         self.url = self.root
 
     def items(self):
@@ -308,7 +308,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor):
         return Message.Queue, url, {
             "type": gtype,
             "date": date,
-            "gallery_id": util.safe_int(parts[1]),
+            "gallery_id": text.parse_int(parts[1]),
             "gallery_token": parts[2],
             "title": text.unescape(title),
             key: last,
diff --git a/gallery_dl/extractor/fallenangels.py b/gallery_dl/extractor/fallenangels.py
index a4ea6f58..3cd3f7a2 100644
--- a/gallery_dl/extractor/fallenangels.py
+++ b/gallery_dl/extractor/fallenangels.py
@@ -98,8 +98,8 @@ class FallenangelsMangaExtractor(MangaExtractor):
             chapter, dot, minor = chapter.partition(".")
             results.append((url, {
                 "manga": manga, "title": title,
-                "volume": util.safe_int(volume),
-                "chapter": util.safe_int(chapter),
+                "volume": text.parse_int(volume),
+                "chapter": text.parse_int(chapter),
                 "chapter_minor": dot + minor,
                 "lang": self.lang, "language": language,
             }))
diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py
index 117c3bdc..cf92b3cf 100644
--- a/gallery_dl/extractor/foolslide.py
+++ b/gallery_dl/extractor/foolslide.py
@@ -50,8 +50,8 @@ class FoolslideExtractor(SharedConfigExtractor):
         lang = info[1].partition("-")[0]
         data["lang"] = lang
         data["language"] = util.code_to_language(lang)
-        data["volume"] = util.safe_int(info[2])
-        data["chapter"] = util.safe_int(info[3])
+        data["volume"] = text.parse_int(info[2])
+        data["chapter"] = text.parse_int(info[3])
         data["chapter_minor"] = "." + info[4] if len(info) >= 5 else ""
         return data
 
@@ -75,7 +75,7 @@ class FoolslideChapterExtractor(FoolslideExtractor):
         imgs = self.get_images(page)
 
         data["count"] = len(imgs)
-        data["chapter_id"] = util.safe_int(imgs[0]["chapter_id"])
+        data["chapter_id"] = text.parse_int(imgs[0]["chapter_id"])
 
         yield Message.Version, 1
         yield Message.Directory, data
@@ -88,7 +88,7 @@ class FoolslideChapterExtractor(FoolslideExtractor):
             except KeyError:
                 pass
             for key in ("height", "id", "size", "width"):
-                image[key] = util.safe_int(image[key])
+                image[key] = text.parse_int(image[key])
             data.update(image)
             text.nameext_from_url(data["filename"], data)
             yield Message.Url, url, data
diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py
index 33abdbd4..110160a6 100644
--- a/gallery_dl/extractor/gelbooru.py
+++ b/gallery_dl/extractor/gelbooru.py
@@ -37,7 +37,7 @@ class GelbooruExtractor(SharedConfigExtractor):
             if isinstance(post, str):
                 post = self.get_post_data(post)
             for key in ("id", "width", "height", "score", "change"):
-                post[key] = util.safe_int(post[key])
+                post[key] = text.parse_int(post[key])
             url = post["file_url"]
             post.update(data)
             yield Message.Url, url, text.nameext_from_url(url, post)
@@ -174,7 +174,7 @@ class GelbooruPoolExtractor(GelbooruExtractor):
             raise exception.NotFoundError("pool")
 
         return {
-            "pool": util.safe_int(self.pool_id),
+            "pool": text.parse_int(self.pool_id),
             "pool_name": text.unescape(name),
             "count": len(self.posts),
         }
diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py
index 03232799..fde517ac 100644
--- a/gallery_dl/extractor/hbrowse.py
+++ b/gallery_dl/extractor/hbrowse.py
@@ -9,7 +9,7 @@
 """Extract images from http://www.hbrowse.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 from urllib.parse import urljoin
 import json
 
@@ -30,7 +30,7 @@ class HbrowseExtractor():
         ), values=data)
 
         data["manga"] = text.unescape(data["manga"])
-        data["total"] = util.safe_int(data["total"])
+        data["total"] = text.parse_int(data["total"])
         data["artist"] = text.remove_html(data["artist"])
         data["origin"] = text.remove_html(data["origin"])
         return data
@@ -48,7 +48,7 @@ class HbrowseMangaExtractor(HbrowseExtractor, MangaExtractor):
     def chapters(self, page):
         results = []
         data = self.parse_page(page, {
-            "manga_id": util.safe_int(
+            "manga_id": text.parse_int(
                 self.url.rstrip("/").rpartition("/")[2])
         })
 
@@ -59,7 +59,7 @@ class HbrowseMangaExtractor(HbrowseExtractor, MangaExtractor):
             if not url:
                 return results
             title, pos = text.extract(page, '>View ', '<', pos)
-            data["chapter"] = util.safe_int(url.rpartition("/")[2][1:])
+            data["chapter"] = text.parse_int(url.rpartition("/")[2][1:])
             data["title"] = title
             results.append((urljoin(self.root, url), data.copy()))
 
@@ -84,8 +84,8 @@ class HbrowseChapterExtractor(HbrowseExtractor, ChapterExtractor):
 
     def get_metadata(self, page):
         return self.parse_page(page, {
-            "manga_id": util.safe_int(self.gid),
-            "chapter": util.safe_int(self.chapter)
+            "manga_id": text.parse_int(self.gid),
+            "chapter": text.parse_int(self.chapter)
         })
 
     def get_images(self, page):
diff --git a/gallery_dl/extractor/hentai2read.py b/gallery_dl/extractor/hentai2read.py
index 6d2cd75f..34a7749c 100644
--- a/gallery_dl/extractor/hentai2read.py
+++ b/gallery_dl/extractor/hentai2read.py
@@ -9,7 +9,7 @@
 """Extract hentai-manga from https://hentai2read.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 import re
 import json
 
@@ -36,7 +36,8 @@ class Hentai2readMangaExtractor(MangaExtractor):
             page, '<span itemprop="name">', '</span>')
         mtype, pos = text.extract(
             page, '<small class="text-danger">[', ']</small>', pos)
-        manga_id = util.safe_int(text.extract(page, 'data-mid="', '"', pos)[0])
+        manga_id = text.parse_int(text.extract(
+            page, 'data-mid="', '"', pos)[0])
 
         while True:
             chapter_id, pos = text.extract(page, ' data-cid="', '"', pos)
@@ -49,8 +50,8 @@ class Hentai2readMangaExtractor(MangaExtractor):
             chapter, _, title = text.unescape(chapter).strip().partition(" - ")
             results.append((url, {
                 "manga_id": manga_id, "manga": manga, "type": mtype,
-                "chapter_id": util.safe_int(chapter_id),
-                "chapter": util.safe_int(chapter),
+                "chapter_id": text.parse_int(chapter_id),
+                "chapter": text.parse_int(chapter),
                 "title": title, "lang": "en", "language": "English",
             }))
 
@@ -78,9 +79,9 @@ class Hentai2readChapterExtractor(ChapterExtractor):
                          r"(\d+): (.+) . Page 1 ", title)
         return {
             "manga": match.group(1),
-            "manga_id": util.safe_int(manga_id),
-            "chapter": util.safe_int(self.chapter),
-            "chapter_id": util.safe_int(chapter_id),
+            "manga_id": text.parse_int(manga_id),
+            "chapter": text.parse_int(self.chapter),
+            "chapter_id": text.parse_int(chapter_id),
             "type": match.group(2),
             "author": match.group(3),
             "title": match.group(5),
diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py
index 2fe4daa7..d3d2e8a4 100644
--- a/gallery_dl/extractor/hentaifoundry.py
+++ b/gallery_dl/extractor/hentaifoundry.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.hentai-foundry.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, exception
 
 
 class HentaifoundryUserExtractor(Extractor):
@@ -69,7 +69,7 @@ class HentaifoundryUserExtractor(Extractor):
         page = response.text
         token, pos = text.extract(page, 'hidden" value="', '"')
         count, pos = text.extract(page, 'class="active" >Pictures (', ')', pos)
-        return {"artist": self.artist, "count": util.safe_int(count)}, token
+        return {"artist": self.artist, "count": text.parse_int(count)}, token
 
     def get_image_metadata(self, url):
         """Collect metadata for an image"""
@@ -79,7 +79,7 @@ class HentaifoundryUserExtractor(Extractor):
             page, 'Pictures</a> &raquo; <span>', '<')
         part, pos = text.extract(
             page, '//pictures.hentai-foundry.com', '"', pos)
-        data = {"index": util.safe_int(index), "title": text.unescape(title)}
+        data = {"index": text.parse_int(index), "title": text.unescape(title)}
         text.nameext_from_url(part, data)
         return "https://pictures.hentai-foundry.com" + part, data
 
@@ -161,7 +161,7 @@ class HentaifoundryImageExtractor(Extractor):
         url   , pos = extr(page, '//pictures.hentai-foundry.com', '"', pos)
         data = {
             "artist": artist,
-            "index": util.safe_int(self.index),
+            "index": text.parse_int(self.index),
             "title": text.unescape(title),
         }
         text.nameext_from_url(url, data)
diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py
index d1c5c932..50150cb8 100644
--- a/gallery_dl/extractor/hentaihere.py
+++ b/gallery_dl/extractor/hentaihere.py
@@ -9,7 +9,7 @@
 """Extract hentai-manga from https://hentaihere.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 import re
 import json
 
@@ -32,7 +32,7 @@ class HentaihereMangaExtractor(MangaExtractor):
 
     def chapters(self, page):
         results = []
-        manga_id = util.safe_int(
+        manga_id = text.parse_int(
             self.url.rstrip("/").rpartition("/")[2][1:])
         manga, pos = text.extract(
             page, '<span itemprop="name">', '</span>')
@@ -50,8 +50,8 @@ class HentaihereMangaExtractor(MangaExtractor):
             chapter, _, title = text.unescape(chapter).strip().partition(" - ")
             results.append((url, {
                 "manga_id": manga_id, "manga": manga, "type": mtype,
-                "chapter_id": util.safe_int(chapter_id),
-                "chapter": util.safe_int(chapter),
+                "chapter_id": text.parse_int(chapter_id),
+                "chapter": text.parse_int(chapter),
                 "title": title, "lang": "en", "language": "English",
             }))
 
@@ -79,9 +79,9 @@ class HentaihereChapterExtractor(ChapterExtractor):
         match = re.match(pattern, title)
         return {
             "manga": match.group(1),
-            "manga_id": util.safe_int(self.manga_id),
-            "chapter": util.safe_int(self.chapter),
-            "chapter_id": util.safe_int(chapter_id),
+            "manga_id": text.parse_int(self.manga_id),
+            "chapter": text.parse_int(self.chapter),
+            "chapter_id": text.parse_int(chapter_id),
             "type": match.group(2),
             "title": match.group(3),
             "author": match.group(4),
diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py
index 64ae1e12..60d91a3f 100644
--- a/gallery_dl/extractor/hitomi.py
+++ b/gallery_dl/extractor/hitomi.py
@@ -30,7 +30,7 @@ class HitomiGalleryExtractor(ChapterExtractor):
     ]
 
     def __init__(self, match):
-        self.gid = util.safe_int(match.group(1))
+        self.gid = text.parse_int(match.group(1))
         url = "https://hitomi.la/galleries/{}.html".format(self.gid)
         ChapterExtractor.__init__(self, url)
 
diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py
index 97d8cb68..75f2e623 100644
--- a/gallery_dl/extractor/imagefap.py
+++ b/gallery_dl/extractor/imagefap.py
@@ -9,7 +9,7 @@
 """Extract images from http://imagefap.com/"""
 
 from .common import Extractor, Message
-from .. import text, util
+from .. import text
 import json
 
 
@@ -159,7 +159,7 @@ class ImagefapUserExtractor(ImagefapExtractor):
         yield Message.Version, 1
         for gid, name in self.get_gallery_data():
             url = "http://www.imagefap.com/gallery/" + gid
-            data = {"gallery_id": util.safe_int(gid), "title": name}
+            data = {"gallery_id": text.parse_int(gid), "title": name}
             yield Message.Queue, url, data
 
     def get_gallery_data(self):
diff --git a/gallery_dl/extractor/kissmanga.py b/gallery_dl/extractor/kissmanga.py
index 36e1f02f..534c36bf 100644
--- a/gallery_dl/extractor/kissmanga.py
+++ b/gallery_dl/extractor/kissmanga.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from http://kissmanga.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util, cloudflare, aes, exception
+from .. import text, cloudflare, aes, exception
 from ..cache import cache
 import re
 import hashlib
@@ -56,8 +56,8 @@ class KissmangaBase():
             ), data["chapter_string"])
 
         volume, chapter, minor, title = match.groups()
-        data["volume"] = util.safe_int(volume)
-        data["chapter"] = util.safe_int(chapter)
+        data["volume"] = text.parse_int(volume)
+        data["chapter"] = text.parse_int(chapter)
         data["chapter_minor"] = "." + minor if minor else ""
         data["title"] = title if title and title != "Read Online" else ""
         return data
@@ -89,7 +89,7 @@ class KissmangaMangaExtractor(KissmangaBase, MangaExtractor):
             url, _, chapter = item.partition(needle)
             data = {
                 "manga": manga, "chapter_string": chapter,
-                "chapter_id": util.safe_int(url.rpartition("=")[2]),
+                "chapter_id": text.parse_int(url.rpartition("=")[2]),
                 "lang": "en", "language": "English",
             }
             self.parse_chapter_string(data)
@@ -128,7 +128,7 @@ class KissmangaChapterExtractor(KissmangaBase, ChapterExtractor):
         data = {
             "manga": manga.strip(),
             "chapter_string": cinfo.strip(),
-            "chapter_id": util.safe_int(self.chapter_id),
+            "chapter_id": text.parse_int(self.chapter_id),
             "lang": "en",
             "language": "English",
         }
diff --git a/gallery_dl/extractor/komikcast.py b/gallery_dl/extractor/komikcast.py
index 04805001..9270fdef 100644
--- a/gallery_dl/extractor/komikcast.py
+++ b/gallery_dl/extractor/komikcast.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from https://komikcast.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util, cloudflare
+from .. import text, cloudflare
 import re
 
 
@@ -39,7 +39,7 @@ class KomikcastBase():
             data["title"] = title.strip()
         else:
             data["title"] = ""
-        data["chapter"] = util.safe_int(chapter)
+        data["chapter"] = text.parse_int(chapter)
         data["lang"] = "id"
         data["language"] = "Indonesian"
 
@@ -75,8 +75,8 @@ class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor):
             page, '<div id="readerarea">', '<div class="navig">')[0]
         return [
             (url, {
-                "width": util.safe_int(width),
-                "height": util.safe_int(height),
+                "width": text.parse_int(width),
+                "height": text.parse_int(height),
             })
             for url, width, height in re.findall(
                 r"<img[^>]*? src=[\"']([^\"']+)[\"']"
diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py
index a53183d7..4529a735 100644
--- a/gallery_dl/extractor/mangadex.py
+++ b/gallery_dl/extractor/mangadex.py
@@ -65,11 +65,11 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
 
         return {
             "manga": match.group(5),
-            "manga_id": util.safe_int(manga_id),
-            "volume": util.safe_int(match.group(1)),
-            "chapter": util.safe_int(match.group(2)),
+            "manga_id": text.parse_int(manga_id),
+            "volume": text.parse_int(match.group(1)),
+            "chapter": text.parse_int(match.group(2)),
             "chapter_minor": match.group(3) or "",
-            "chapter_id": util.safe_int(self.chapter_id),
+            "chapter_id": text.parse_int(self.chapter_id),
             "chapter_string": info.replace(" - MangaDex", ""),
             "group": text.unescape(group),
             "lang": util.language_to_code(language),
@@ -124,7 +124,7 @@ class MangadexMangaExtractor(MangadexExtractor, MangaExtractor):
 
         manga = text.unescape(extr(
             page, '"og:title" content="', '"')[0].rpartition(" (")[0])
-        manga_id = util.safe_int(extr(
+        manga_id = text.parse_int(extr(
             page, '/images/manga/', '.')[0])
 
         while True:
@@ -145,15 +145,15 @@ class MangadexMangaExtractor(MangadexExtractor, MangaExtractor):
 
                 results.append((self.root + "/chapter/" + chid, {
                     "manga": manga,
-                    "manga_id": util.safe_int(manga_id),
+                    "manga_id": text.parse_int(manga_id),
                     "title": text.unescape(title),
-                    "volume": util.safe_int(volume),
-                    "chapter": util.safe_int(chapter),
+                    "volume": text.parse_int(volume),
+                    "chapter": text.parse_int(chapter),
                     "chapter_minor": sep + minor,
-                    "chapter_id": util.safe_int(chid),
+                    "chapter_id": text.parse_int(chid),
                     "group": text.unescape(text.remove_html(group)),
                     "contributor": text.remove_html(user),
-                    "views": util.safe_int(views),
+                    "views": text.parse_int(views),
                     "date": date,
                     "lang": util.language_to_code(language),
                     "language": language,
diff --git a/gallery_dl/extractor/mangafox.py b/gallery_dl/extractor/mangafox.py
index 906c2372..38eefa12 100644
--- a/gallery_dl/extractor/mangafox.py
+++ b/gallery_dl/extractor/mangafox.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from http://fanfox.net/"""
 
 from .common import ChapterExtractor
-from .. import text, util, exception
+from .. import text, exception
 import re
 
 
@@ -47,7 +47,7 @@ class MangafoxChapterExtractor(ChapterExtractor):
         data["chapter_minor"] = match.group(4) or ""
         data["manga"] = data["manga"].rpartition(" ")[0]
         for key in ("sid", "cid", "count", "volume", "chapter"):
-            data[key] = util.safe_int(data[key])
+            data[key] = text.parse_int(data[key])
         return data
 
     def get_images(self, page):
diff --git a/gallery_dl/extractor/mangahere.py b/gallery_dl/extractor/mangahere.py
index 31e758f5..b4ade702 100644
--- a/gallery_dl/extractor/mangahere.py
+++ b/gallery_dl/extractor/mangahere.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from http://www.mangahere.co/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 from urllib.parse import urljoin
 import re
 
@@ -53,8 +53,8 @@ class MangahereMangaExtractor(MangaExtractor):
             date, pos = text.extract(page, 'class="right">', '</span>', pos)
             results.append((urljoin("http:", url), {
                 "manga": manga, "title": title, "date": date,
-                "volume": util.safe_int(volume.rpartition(" ")[2]),
-                "chapter": util.safe_int(chapter),
+                "volume": text.parse_int(volume.rpartition(" ")[2]),
+                "chapter": text.parse_int(chapter),
                 "chapter_minor": dot + minor,
                 "lang": "en", "language": "English",
             }))
@@ -93,11 +93,11 @@ class MangahereChapterExtractor(ChapterExtractor):
         return {
             "manga": text.unescape(manga),
             # "title": TODO,
-            "volume": util.safe_int(self.volume),
-            "chapter": util.safe_int(self.chapter),
+            "volume": text.parse_int(self.volume),
+            "chapter": text.parse_int(self.chapter),
             "chapter_minor": self.chminor or "",
-            "chapter_id": util.safe_int(chid),
-            "count": util.safe_int(count),
+            "chapter_id": text.parse_int(chid),
+            "count": text.parse_int(count),
             "lang": "en",
             "language": "English",
         }
diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py
index 4752d1f5..b12b7d45 100644
--- a/gallery_dl/extractor/mangapark.py
+++ b/gallery_dl/extractor/mangapark.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from https://mangapark.me/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 from urllib.parse import urljoin
 
 
@@ -25,12 +25,12 @@ class MangaparkExtractor():
         for part in path.split("/")[3:]:
             key, value = part[0], part[1:]
             if key == "s":
-                data["version"] = util.safe_int(value)
+                data["version"] = text.parse_int(value)
             elif key == "v":
-                data["volume"] = util.safe_int(value)
+                data["volume"] = text.parse_int(value)
             elif key == "c":
                 chapter, dot, minor = value.partition(".")
-                data["chapter"] = util.safe_int(chapter)
+                data["chapter"] = text.parse_int(chapter)
                 data["chapter_minor"] = dot + minor
             elif key == "e":
                 data["chapter_minor"] = "v" + value
@@ -64,7 +64,7 @@ class MangaparkMangaExtractor(MangaparkExtractor, MangaExtractor):
             self.parse_chapter_path(path, data)
             data["title"] = title[3:].strip()
             data["date"] = date
-            data["count"] = util.safe_int(count)
+            data["count"] = text.parse_int(count)
             results.append((self.root + path, data.copy()))
 
 
@@ -107,7 +107,7 @@ class MangaparkChapterExtractor(MangaparkExtractor, ChapterExtractor):
         data["manga"], _, data["type"] = data["manga"].rpartition(" ")
         data["manga"] = text.unescape(data["manga"])
         data["title"] = data["title"].partition(": ")[2]
-        data["count"] = util.safe_int(data["count"])
+        data["count"] = text.parse_int(data["count"])
         return data
 
     def get_images(self, page):
diff --git a/gallery_dl/extractor/mangareader.py b/gallery_dl/extractor/mangareader.py
index 11714a6d..3aabb153 100644
--- a/gallery_dl/extractor/mangareader.py
+++ b/gallery_dl/extractor/mangareader.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters and entire manga from https://www.mangareader.net/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 
 
 class MangareaderBase():
@@ -53,7 +53,7 @@ class MangareaderMangaExtractor(MangareaderBase, MangaExtractor):
                 return results
             data["title"], pos = text.extract(page, '</a> : ', '</td>', pos)
             data["date"] , pos = text.extract(page, '<td>', '</td>', pos)
-            data["chapter"] = util.safe_int(url.rpartition("/")[2])
+            data["chapter"] = text.parse_int(url.rpartition("/")[2])
             results.append((self.root + url, data.copy()))
 
 
@@ -79,7 +79,7 @@ class MangareaderChapterExtractor(MangareaderBase, ChapterExtractor):
         """Collect metadata for extractor-job"""
         page = self.request(self.root + self.url_title).text
         data = self.parse_page(page, {
-            "chapter": util.safe_int(self.chapter),
+            "chapter": text.parse_int(self.chapter),
             "lang": "en",
             "language": "English",
         })
@@ -87,7 +87,7 @@ class MangareaderChapterExtractor(MangareaderBase, ChapterExtractor):
             ('title', ' ' + self.chapter + '</a> : ', '</td>'),
             ('date', '<td>', '</td>'),
         ), page.index('<div id="chapterlist">'), data)
-        data["count"] = util.safe_int(text.extract(
+        data["count"] = text.parse_int(text.extract(
             chapter_page, '</select> of ', '<')[0]
         )
         return data
@@ -118,6 +118,6 @@ class MangareaderChapterExtractor(MangareaderBase, ChapterExtractor):
             height, pos = extr(page, ' height="', '"', pos)
         image, pos = extr(page, ' src="', '"', pos)
         return self.root + url, image, {
-            "width": util.safe_int(width),
-            "height": util.safe_int(height),
+            "width": text.parse_int(width),
+            "height": text.parse_int(height),
         }
diff --git a/gallery_dl/extractor/mangastream.py b/gallery_dl/extractor/mangastream.py
index 51a469a3..70c18f1a 100644
--- a/gallery_dl/extractor/mangastream.py
+++ b/gallery_dl/extractor/mangastream.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters from https://mangastream.com/"""
 
 from .common import ChapterExtractor
-from .. import text, util
+from .. import text
 from urllib.parse import urljoin
 
 
@@ -35,9 +35,9 @@ class MangastreamChapterExtractor(ChapterExtractor):
         return {
             "manga": manga,
             "chapter": text.unquote(self.chapter),
-            "chapter_id": util.safe_int(self.ch_id),
+            "chapter_id": text.parse_int(self.ch_id),
             "title": title,
-            "count": util.safe_int(count, 1),
+            "count": text.parse_int(count, 1),
             "lang": "en",
             "language": "English",
         }
diff --git a/gallery_dl/extractor/nhentai.py b/gallery_dl/extractor/nhentai.py
index 35816e60..a0c8abc1 100644
--- a/gallery_dl/extractor/nhentai.py
+++ b/gallery_dl/extractor/nhentai.py
@@ -9,7 +9,7 @@
 """Extract images from https://nhentai.net/"""
 
 from .common import Extractor, Message
-from .. import text, util
+from .. import text
 
 
 class NHentaiExtractor(Extractor):
@@ -95,7 +95,7 @@ class NhentaiSearchExtractor(NHentaiExtractor):
     def _pagination(self, endpoint, params):
         """Pagination over API responses"""
         url = "{}/api/{}".format(self.root, endpoint)
-        params["page"] = util.safe_int(params.get("page"), 1)
+        params["page"] = text.parse_int(params.get("page"), 1)
 
         while True:
             data = self.request(url, params=params, fatal=False).json()
diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py
index 56e1e521..79b7a988 100644
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@@ -9,7 +9,7 @@
 """Extract images from https://nijie.info/"""
 
 from .common import AsynchronousExtractor, Message
-from .. import text, util, exception
+from .. import text, exception
 from ..cache import cache
 
 
@@ -44,7 +44,7 @@ class NijieExtractor(AsynchronousExtractor):
 
     def get_job_metadata(self):
         """Collect metadata for extractor-job"""
-        return {"user_id": util.safe_int(self.user_id)}
+        return {"user_id": text.parse_int(self.user_id)}
 
     def get_image_ids(self):
         """Collect all relevant image-ids"""
@@ -63,8 +63,8 @@ class NijieExtractor(AsynchronousExtractor):
         images = list(text.extract_iter(page, '<img src="//pic', '"', pos))
 
         title = title.rpartition("|")[0].strip()
-        image_id = util.safe_int(image_id)
-        artist_id = util.safe_int(self._userid_from_popup(page))
+        image_id = text.parse_int(image_id)
+        artist_id = text.parse_int(self._userid_from_popup(page))
 
         for index, url in enumerate(images):
             yield "https://pic" + url, text.nameext_from_url(url, {
diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py
index 4f27f598..8cdb3569 100644
--- a/gallery_dl/extractor/paheal.py
+++ b/gallery_dl/extractor/paheal.py
@@ -27,7 +27,7 @@ class PahealExtractor(SharedConfigExtractor):
         for data in self.get_posts():
             url = data["file_url"]
             for key in ("id", "width", "height"):
-                data[key] = util.safe_int(data[key])
+                data[key] = text.parse_int(data[key])
             data["tags"] = text.unquote(data["tags"])
             yield Message.Url, url, text.nameext_from_url(url, data)
 
diff --git a/gallery_dl/extractor/pinterest.py b/gallery_dl/extractor/pinterest.py
index e4484102..6739a18b 100644
--- a/gallery_dl/extractor/pinterest.py
+++ b/gallery_dl/extractor/pinterest.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.pinterest.com"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, exception
 
 
 class PinterestExtractor(Extractor):
@@ -27,10 +27,10 @@ class PinterestExtractor(Extractor):
         img = pin["image"]["original"]
         url = img["url"]
         data = {
-            "pin_id": util.safe_int(pin["id"]),
+            "pin_id": text.parse_int(pin["id"]),
             "note": pin["note"],
-            "width": util.safe_int(img["width"]),
-            "height": util.safe_int(img["height"]),
+            "width": text.parse_int(img["width"]),
+            "height": text.parse_int(img["height"]),
         }
         return url, text.nameext_from_url(url, data)
 
@@ -99,7 +99,7 @@ class PinterestBoardExtractor(PinterestExtractor):
         """Get metadata from a board-object"""
         data = {
             "user": self.user,
-            "board_id": util.safe_int(board["id"]),
+            "board_id": text.parse_int(board["id"]),
             "board": board["name"],
             "count": board["counts"]["pins"],
         }
diff --git a/gallery_dl/extractor/readcomiconline.py b/gallery_dl/extractor/readcomiconline.py
index 2d8b6222..ca26b5e7 100644
--- a/gallery_dl/extractor/readcomiconline.py
+++ b/gallery_dl/extractor/readcomiconline.py
@@ -9,7 +9,7 @@
 """Extract comic-issues and entire comics from http://readcomiconline.to/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util, cloudflare
+from .. import text, cloudflare
 import re
 
 
@@ -56,7 +56,7 @@ class ReadcomiconlineComicExtractor(ReadcomiconlineBase, MangaExtractor):
                 issue = issue[7:]
             results.append((self.root + url, {
                 "comic": comic, "issue": issue,
-                "issue_id": util.safe_int(url.rpartition("=")[2]),
+                "issue_id": text.parse_int(url.rpartition("=")[2]),
                 "lang": "en", "language": "English",
             }))
         return results
@@ -84,7 +84,7 @@ class ReadcomiconlineIssueExtractor(ReadcomiconlineBase, ChapterExtractor):
         return {
             "comic": comic,
             "issue": match.group(1) or match.group(2),
-            "issue_id": util.safe_int(self.issue_id),
+            "issue_id": text.parse_int(self.issue_id),
             "lang": "en",
             "language": "English",
         }
diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py
index 81de60c2..001cbb2c 100644
--- a/gallery_dl/extractor/sankaku.py
+++ b/gallery_dl/extractor/sankaku.py
@@ -82,16 +82,16 @@ class SankakuExtractor(SharedConfigExtractor):
             file_url = extr(page, '<embed src="', '"', pos)[0]
 
         return {
-            "id": util.safe_int(post_id),
+            "id": text.parse_int(post_id),
             "md5": file_url.rpartition("/")[2].partition(".")[0],
             "tags": tags,
             "vote_average": float(vavg or 0),
-            "vote_count": util.safe_int(vcnt),
+            "vote_count": text.parse_int(vcnt),
             "created_at": created,
             "rating": (rating or "?")[0].lower(),
             "file_url": "https:" + text.unescape(file_url),
-            "width": util.safe_int(width),
-            "height": util.safe_int(height),
+            "width": text.parse_int(width),
+            "height": text.parse_int(height),
         }
 
     def wait(self):
@@ -165,8 +165,8 @@ class SankakuTagExtractor(SankakuExtractor):
         SankakuExtractor.__init__(self)
         query = text.parse_query(match.group(1))
         self.tags = text.unquote(query.get("tags", "").replace("+", " "))
-        self.start_page = util.safe_int(query.get("page"), 1)
-        self.next = util.safe_int(query.get("next"), 0)
+        self.start_page = text.parse_int(query.get("page"), 1)
+        self.next = text.parse_int(query.get("next"), 0)
 
     def skip(self, num):
         if self.next:
@@ -212,7 +212,7 @@ class SankakuTagExtractor(SankakuExtractor):
             yield from ids
 
             params["page"] = 2
-            params["next"] = util.safe_int(ids[-1]) - 1
+            params["next"] = text.parse_int(ids[-1]) - 1
 
 
 class SankakuPoolExtractor(SankakuExtractor):
diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py
index 93249007..05470b05 100644
--- a/gallery_dl/extractor/seiga.py
+++ b/gallery_dl/extractor/seiga.py
@@ -123,11 +123,11 @@ class SeigaUserExtractor(SeigaExtractor):
 
         return {
             "user": {
-                "id": util.safe_int(self.user_id),
+                "id": text.parse_int(self.user_id),
                 "name": data["name"],
                 "message": (data["msg"] or "").strip(),
             },
-            "count": util.safe_int(data["count"]),
+            "count": text.parse_int(data["count"]),
         }
 
     def get_images(self):
@@ -152,7 +152,7 @@ class SeigaUserExtractor(SeigaExtractor):
                     ("clips"   , '</span>', '</li>'),
                 ))[0]
                 for key in ("image_id", "views", "comments", "clips"):
-                    data[key] = util.safe_int(data[key])
+                    data[key] = text.parse_int(data[key])
                 yield data
                 cnt += 1
 
@@ -188,4 +188,4 @@ class SeigaImageExtractor(SeigaExtractor):
         return num
 
     def get_images(self):
-        return ({}, {"image_id": util.safe_int(self.image_id)})
+        return ({}, {"image_id": text.parse_int(self.image_id)})
diff --git a/gallery_dl/extractor/senmanga.py b/gallery_dl/extractor/senmanga.py
index d46d5e7c..98f096c0 100644
--- a/gallery_dl/extractor/senmanga.py
+++ b/gallery_dl/extractor/senmanga.py
@@ -9,7 +9,7 @@
 """Extract manga-chapters from from http://raw.senmanga.com/"""
 
 from .common import Extractor, Message
-from .. import text, util
+from .. import text
 
 
 class SenmangaChapterExtractor(Extractor):
@@ -59,7 +59,7 @@ class SenmangaChapterExtractor(Extractor):
         return {
             "manga": text.unescape(manga),
             "chapter_string": chapter.partition(" - Page ")[0],
-            "count": util.safe_int(count),
+            "count": text.parse_int(count),
             "lang": "jp",
             "language": "Japanese",
         }
diff --git a/gallery_dl/extractor/slideshare.py b/gallery_dl/extractor/slideshare.py
index 1a680d34..3ab15809 100644
--- a/gallery_dl/extractor/slideshare.py
+++ b/gallery_dl/extractor/slideshare.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.slideshare.net/"""
 
 from .common import Extractor, Message
-from .. import text, util
+from .. import text
 
 
 class SlideshareExtractor(Extractor):
@@ -78,7 +78,7 @@ class SlideshareExtractor(Extractor):
             "presentation": self.presentation,
             "title": text.unescape(title.strip()),
             "description": text.unescape(descr),
-            "views": util.safe_int(views.replace(",", "")),
+            "views": text.parse_int(views.replace(",", "")),
             "published": published,
         }
 
diff --git a/gallery_dl/extractor/spectrumnexus.py b/gallery_dl/extractor/spectrumnexus.py
index 8bc2aa1c..12207d7c 100644
--- a/gallery_dl/extractor/spectrumnexus.py
+++ b/gallery_dl/extractor/spectrumnexus.py
@@ -9,7 +9,7 @@
 """Extract manga pages from http://www.thespectrum.net/manga_scans/"""
 
 from .common import ChapterExtractor, MangaExtractor
-from .. import text, util
+from .. import text
 
 
 class SpectrumnexusMangaExtractor(MangaExtractor):
@@ -55,15 +55,15 @@ class SpectrumnexusChapterExtractor(ChapterExtractor):
 
     def get_metadata(self, page):
         data = {
-            "chapter": util.safe_int(self.chapter),
+            "chapter": text.parse_int(self.chapter),
             "chapter_string": self.chapter_string.replace("+", " "),
-            "volume": util.safe_int(self.volume),
+            "volume": text.parse_int(self.volume),
         }
         data = text.extract_all(page, (
             ('manga', '<title>', ' &#183; SPECTRUM NEXUS </title>'),
             ('count', '<div class="viewerLabel"> of ', '<'),
         ), values=data)[0]
-        data["count"] = util.safe_int(data["count"])
+        data["count"] = text.parse_int(data["count"])
         return data
 
     def get_images(self, page):
diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py
index d3575389..99d936a6 100644
--- a/gallery_dl/extractor/xvideos.py
+++ b/gallery_dl/extractor/xvideos.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.xvideos.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, exception
 import json
 
 
@@ -57,7 +57,7 @@ class XvideosGalleryExtractor(XvideosExtractor):
         yield Message.Version, 1
         yield Message.Directory, data
         for url in imgs:
-            data["num"] = util.safe_int(url.rsplit("_", 2)[1])
+            data["num"] = text.parse_int(url.rsplit("_", 2)[1])
             data["extension"] = url.rpartition(".")[2]
             yield Message.Url, url, data
 
@@ -73,14 +73,14 @@ class XvideosGalleryExtractor(XvideosExtractor):
 
         return {
             "user": {
-                "id": util.safe_int(data["userid"]),
+                "id": text.parse_int(data["userid"]),
                 "name": self.user,
                 "display": data["display"],
                 "description": text.remove_html(data["descr"]).strip(),
             },
             "tags": text.unescape(data["tags"] or "").strip().split(", "),
             "title": text.unescape(data["title"]),
-            "gallery_id": util.safe_int(self.gid),
+            "gallery_id": text.parse_int(self.gid),
         }
 
     @staticmethod
@@ -123,7 +123,7 @@ class XvideosUserExtractor(XvideosExtractor):
             del data["galleries"]["0"]
 
         galleries = [
-            {"gallery_id": util.safe_int(gid),
+            {"gallery_id": text.parse_int(gid),
              "title": text.unescape(gdata["title"]),
              "count": gdata["nb_pics"]}
             for gid, gdata in data["galleries"].items()
diff --git a/gallery_dl/text.py b/gallery_dl/text.py
index 4bed863a..033cf5f3 100644
--- a/gallery_dl/text.py
+++ b/gallery_dl/text.py
@@ -9,8 +9,8 @@
 """Collection of functions that work in strings/text"""
 
 import re
-import os.path
 import html
+import os.path
 import urllib.parse
 
 
@@ -125,6 +125,35 @@ def extract_iter(txt, begin, end, pos=0):
         yield value
 
 
+def parse_bytes(value, default=0, suffixes="bkmgtp"):
+    """Convert a bytes-amount ("500k", "2.5M", ...) to int"""
+    try:
+        last = value[-1].lower()
+    except (TypeError, KeyError, IndexError):
+        return default
+
+    if last in suffixes:
+        mul = 1024 ** suffixes.index(last)
+        value = value[:-1]
+    else:
+        mul = 1
+
+    try:
+        return round(float(value) * mul)
+    except ValueError:
+        return default
+
+
+def parse_int(value, default=0):
+    """Convert 'value' to int"""
+    if not value:
+        return default
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return default
+
+
 def parse_query(qs):
     """Parse a query string into key-value pairs"""
     result = {}
@@ -142,6 +171,7 @@ if os.name == "nt":
 else:
     clean_path = clean_path_posix
 
+urljoin = urllib.parse.urljoin
 unquote = urllib.parse.unquote
 escape = html.escape
 
diff --git a/gallery_dl/util.py b/gallery_dl/util.py
index 8341e084..af754f07 100644
--- a/gallery_dl/util.py
+++ b/gallery_dl/util.py
@@ -95,22 +95,6 @@ def bdecode(data, alphabet="0123456789"):
     return num
 
 
-def parse_bytes(value, suffixes="bkmgtp"):
-    """Convert a bytes-amount ("500k", "2.5M", ...) to int"""
-    last = value[-1].lower()
-
-    if last in suffixes:
-        mul = 1024 ** suffixes.index(last)
-        value = value[:-1]
-    else:
-        mul = 1
-
-    try:
-        return round(float(value) * mul)
-    except ValueError:
-        return 0
-
-
 def advance(iterable, num):
     """"Advance the iterable by 'num' steps"""
     iterator = iter(iterable)
@@ -135,16 +119,6 @@ def combine_dict(a, b):
     return a
 
 
-def safe_int(value, default=0):
-    """Safely convert value to integer"""
-    if value is None or value == "":
-        return default
-    try:
-        return int(value)
-    except (ValueError, TypeError):
-        return default
-
-
 def expand_path(path):
     """Expand environment variables and tildes (~)"""
     if not path:
@@ -253,7 +227,7 @@ class UniquePredicate():
 class FilterPredicate():
     """Predicate; True if evaluating the given expression returns True"""
     globalsdict = {
-        "safe_int": safe_int,
+        "parse_int": text.parse_int,
         "urlsplit": urllib.parse.urlsplit,
         "datetime": datetime.datetime,
         "abort": raises(exception.StopExtraction()),
diff --git a/test/test_text.py b/test/test_text.py
index b07dff10..f76fc2a2 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -13,6 +13,7 @@ from gallery_dl import text
 
 
 INVALID = ((), [], {}, None, 1, 2.3)
+INVALID_ALT = ((), [], {}, None, "")
 
 
 class TestText(unittest.TestCase):
@@ -194,6 +195,47 @@ class TestText(unittest.TestCase):
         self.assertEqual(
             g(txt, "[", "]", 6), ["a", "d"])
 
+    def test_parse_bytes(self, f=text.parse_bytes):
+        self.assertEqual(f("0"), 0)
+        self.assertEqual(f("50"), 50)
+        self.assertEqual(f("50k"), 50 * 1024**1)
+        self.assertEqual(f("50m"), 50 * 1024**2)
+        self.assertEqual(f("50g"), 50 * 1024**3)
+        self.assertEqual(f("50t"), 50 * 1024**4)
+        self.assertEqual(f("50p"), 50 * 1024**5)
+
+        # fractions
+        self.assertEqual(f("123.456"), 123)
+        self.assertEqual(f("123.567"), 124)
+        self.assertEqual(f("0.5M"), round(0.5 * 1024**2))
+
+        # invalid arguments
+        for value in INVALID_ALT:
+            self.assertEqual(f(value), 0)
+        self.assertEqual(f("NaN"), 0)
+        self.assertEqual(f("invalid"), 0)
+        self.assertEqual(f(" 123 kb "), 0)
+
+    def test_parse_int(self, f=text.parse_int):
+        self.assertEqual(f(0), 0)
+        self.assertEqual(f("0"), 0)
+        self.assertEqual(f(123), 123)
+        self.assertEqual(f("123"), 123)
+
+        # invalid arguments
+        for value in INVALID_ALT:
+            self.assertEqual(f(value), 0)
+        self.assertEqual(f("123.456"), 0)
+        self.assertEqual(f("zzz"), 0)
+        self.assertEqual(f([1, 2, 3]), 0)
+        self.assertEqual(f({1: 2, 3: 4}), 0)
+
+        # 'default' argument
+        default = "default"
+        for value in INVALID_ALT:
+            self.assertEqual(f(value, default), default)
+        self.assertEqual(f("zzz", default), default)
+
     def test_parse_query(self, f=text.parse_query):
         # standard usage
         self.assertEqual(f(""), {})
diff --git a/test/test_util.py b/test/test_util.py
index 7c684d89..8333828e 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -227,22 +227,6 @@ class TestOther(unittest.TestCase):
                 result = util.bdecode(util.bencode(value, alphabet), alphabet)
                 self.assertEqual(result, value)
 
-    def test_parse_bytes(self):
-        self.assertEqual(util.parse_bytes("50"), 50)
-        self.assertEqual(util.parse_bytes("50k"), 50 * 1024**1)
-        self.assertEqual(util.parse_bytes("50m"), 50 * 1024**2)
-        self.assertEqual(util.parse_bytes("50g"), 50 * 1024**3)
-        self.assertEqual(util.parse_bytes("50t"), 50 * 1024**4)
-        self.assertEqual(util.parse_bytes("50p"), 50 * 1024**5)
-
-        self.assertEqual(util.parse_bytes("123.456"), 123)
-        self.assertEqual(util.parse_bytes("123.567"), 124)
-        self.assertEqual(util.parse_bytes("0.5M"), round(0.5 * 1024**2))
-
-        self.assertEqual(util.parse_bytes("NaN"), 0)
-        self.assertEqual(util.parse_bytes("invalid"), 0)
-        self.assertEqual(util.parse_bytes(" 123 kb "), 0)
-
     def test_advance(self):
         items = range(5)
 
@@ -281,16 +265,6 @@ class TestOther(unittest.TestCase):
                 {1: {2: {3: {4: {"1": "A", "3": "C"}}}}}),
             {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}})
 
-    def test_safe_int(self):
-        self.assertEqual(util.safe_int(123), 123)
-        self.assertEqual(util.safe_int("123"), 123)
-        self.assertEqual(util.safe_int("zzz"), 0)
-        self.assertEqual(util.safe_int(""), 0)
-        self.assertEqual(util.safe_int(None), 0)
-        self.assertEqual(util.safe_int("zzz", "default"), "default")
-        self.assertEqual(util.safe_int("", "default"), "default")
-        self.assertEqual(util.safe_int(None, "default"), "default")
-
 
 if __name__ == '__main__':
     unittest.main()

From 2d17a9e07f8dde18f2b635641fc707c5136ad6d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Mon, 23 Apr 2018 18:45:59 +0200
Subject: [PATCH 08/21] improve extractor.request()

- better retry behavior
- exponential back-off
- removed 'allow_empty' argument
---
 gallery_dl/extractor/artstation.py    |  2 +-
 gallery_dl/extractor/common.py        | 42 +++++++++++++--------------
 gallery_dl/extractor/hentaifoundry.py |  2 +-
 gallery_dl/extractor/nijie.py         |  4 +--
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py
index 2d582957..80ed0295 100644
--- a/gallery_dl/extractor/artstation.py
+++ b/gallery_dl/extractor/artstation.py
@@ -73,7 +73,7 @@ class ArtstationExtractor(Extractor):
     def get_user_info(self, username):
         """Return metadata for a specific user"""
         url = "{}/users/{}/quick.json".format(self.root, username.lower())
-        response = self.request(url, fatal=False, allow_empty=True)
+        response = self.request(url, fatal=False)
         if response.status_code == 404:
             raise exception.NotFoundError("user")
         return response.json()
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 8e42c32c..72a53369 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -52,34 +52,34 @@ class Extractor():
             ("extractor", self.category, self.subcategory, key), default)
 
     def request(self, url, method="GET", encoding=None, fatal=True, retries=3,
-                allow_empty=False, *args, **kwargs):
-        max_retries = retries
+                *args, **kwargs):
+        max_tries = retries
         while True:
             try:
-                response = None
                 response = self.session.request(method, url, *args, **kwargs)
-                if fatal:
-                    response.raise_for_status()
-                if encoding:
-                    response.encoding = encoding
-                if response.content or allow_empty:
-                    return response
-                msg = "empty response body"
-            except requests.exceptions.HTTPError as exc:
+            except (requests.ConnectionError, requests.Timeout) as exc:
                 msg = exc
-                code = response.status_code
-                if 400 <= code < 500 and code != 429:  # Client Error
-                    retries = 0
             except requests.exceptions.RequestException as exc:
-                msg = exc
-            if not retries:
-                raise exception.HttpError(msg)
-            if response and response.status_code == 429:  # Too Many Requests
-                waittime = float(response.headers.get("Retry-After", 10.0))
+                raise exception.HttpError(exc)
             else:
-                waittime = 1
+                if 200 <= response.status_code < 400 or not fatal:
+                    if encoding:
+                        response.encoding = encoding
+                    return response
+
+                msg = "{} HTTP Error: {} for url: {}".format(
+                    response.status_code, response.reason, url)
+                if response.status_code < 500 and response.status_code != 429:
+                    break
+
+            if not retries:
+                break
+            tries = max_tries - retries
             retries -= 1
-            time.sleep(waittime * (max_retries - retries))
+            self.log.debug("%s (%d/%d)", msg, tries, max_tries)
+            time.sleep(2 ** tries)
+
+        raise exception.HttpError(msg)
 
     def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""
diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py
index 21a23499..e697aaab 100644
--- a/gallery_dl/extractor/hentaifoundry.py
+++ b/gallery_dl/extractor/hentaifoundry.py
@@ -142,7 +142,7 @@ class HentaifoundryUserExtractor(HentaifoundryExtractor):
             "filter_type": 0,
         }
         self.request("https://www.hentai-foundry.com/site/filters",
-                     method="post", data=formdata, allow_empty=True)
+                     method="post", data=formdata)
 
 
 class HentaifoundryImageExtractor(HentaifoundryExtractor):
diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py
index 79b7a988..db2ce317 100644
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@@ -193,8 +193,8 @@ class NijieImageExtractor(NijieExtractor):
         self.page = ""
 
     def get_job_metadata(self):
-        response = self.request(self.popup_url + self.image_id,
-                                allow_redirects=False, allow_empty=True)
+        response = self.request(
+            self.popup_url + self.image_id, allow_redirects=False)
         if 300 <= response.status_code < 400:
             raise exception.NotFoundError("image")
         self.page = response.text

From 95392554ee2034f5818c2c2b4e739a7af4d393ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 26 Apr 2018 17:00:26 +0200
Subject: [PATCH 09/21] use text.urljoin()

---
 gallery_dl/cloudflare.py              | 2 +-
 gallery_dl/extractor/booru.py         | 3 +--
 gallery_dl/extractor/hbrowse.py       | 3 +--
 gallery_dl/extractor/hentaifoundry.py | 3 +--
 gallery_dl/extractor/imagehosts.py    | 4 +---
 gallery_dl/extractor/khinsider.py     | 4 ++--
 gallery_dl/extractor/mangadex.py      | 3 +--
 gallery_dl/extractor/mangahere.py     | 3 +--
 gallery_dl/extractor/mangapark.py     | 3 +--
 gallery_dl/extractor/mangastream.py   | 5 ++---
 10 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/gallery_dl/cloudflare.py b/gallery_dl/cloudflare.py
index 676e723f..414a461d 100644
--- a/gallery_dl/cloudflare.py
+++ b/gallery_dl/cloudflare.py
@@ -41,7 +41,7 @@ def solve_challenge(session, response):
     params["jschl_answer"] = solve_jschl(response.url, page)
 
     time.sleep(4)
-    url = urllib.parse.urljoin(response.url, "/cdn-cgi/l/chk_jschl")
+    url = text.urljoin(response.url, "/cdn-cgi/l/chk_jschl")
     return session.get(url, params=params)
 
 
diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py
index aaddbaac..0113f62c 100644
--- a/gallery_dl/extractor/booru.py
+++ b/gallery_dl/extractor/booru.py
@@ -10,7 +10,6 @@
 
 from .common import SharedConfigExtractor, Message
 from .. import text
-from urllib.parse import urljoin
 from xml.etree import ElementTree
 import datetime
 import operator
@@ -52,7 +51,7 @@ class BooruExtractor(SharedConfigExtractor):
                 try:
                     url = image["file_url"]
                     if url.startswith("/"):
-                        url = urljoin(self.api_url, url)
+                        url = text.urljoin(self.api_url, url)
                     image.update(data)
                     yield Message.Url, url, text.nameext_from_url(url, image)
                 except KeyError:
diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py
index fde517ac..f1f04ed2 100644
--- a/gallery_dl/extractor/hbrowse.py
+++ b/gallery_dl/extractor/hbrowse.py
@@ -10,7 +10,6 @@
 
 from .common import ChapterExtractor, MangaExtractor
 from .. import text
-from urllib.parse import urljoin
 import json
 
 
@@ -61,7 +60,7 @@ class HbrowseMangaExtractor(HbrowseExtractor, MangaExtractor):
             title, pos = text.extract(page, '>View ', '<', pos)
             data["chapter"] = text.parse_int(url.rpartition("/")[2][1:])
             data["title"] = title
-            results.append((urljoin(self.root, url), data.copy()))
+            results.append((text.urljoin(self.root, url), data.copy()))
 
 
 class HbrowseChapterExtractor(HbrowseExtractor, ChapterExtractor):
diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py
index e697aaab..23713fc0 100644
--- a/gallery_dl/extractor/hentaifoundry.py
+++ b/gallery_dl/extractor/hentaifoundry.py
@@ -10,7 +10,6 @@
 
 from .common import Extractor, Message
 from .. import text, util, exception
-from urllib.parse import urljoin
 
 
 class HentaifoundryExtractor(Extractor):
@@ -47,7 +46,7 @@ class HentaifoundryExtractor(Extractor):
 
     def get_image_metadata(self, url):
         """Collect metadata for an image"""
-        page = self.request(urljoin(self.root, url)).text
+        page = self.request(text.urljoin(self.root, url)).text
         index = url.rsplit("/", 2)[1]
         title, pos = text.extract(
             page, 'Pictures</a> &raquo; <span>', '<')
diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py
index d3d07826..1b029d33 100644
--- a/gallery_dl/extractor/imagehosts.py
+++ b/gallery_dl/extractor/imagehosts.py
@@ -12,7 +12,6 @@ from .common import Extractor, Message
 from .. import text, exception
 from ..cache import memcache
 from os.path import splitext
-from urllib.parse import urljoin
 
 
 class ImagehostImageExtractor(Extractor):
@@ -142,8 +141,7 @@ class ImagevenueImageExtractor(ImagehostImageExtractor):
 
     def get_info(self, page):
         url = text.extract(page, 'SRC="', '"')[0]
-        url = urljoin(self.url, url)
-        return url, url
+        return text.urljoin(self.url, url), url
 
 
 class ImagetwistImageExtractor(ImagehostImageExtractor):
diff --git a/gallery_dl/extractor/khinsider.py b/gallery_dl/extractor/khinsider.py
index ddbe56c3..cde65827 100644
--- a/gallery_dl/extractor/khinsider.py
+++ b/gallery_dl/extractor/khinsider.py
@@ -10,7 +10,6 @@
 
 from .common import AsynchronousExtractor, Message
 from .. import text, exception
-from urllib.parse import urljoin
 
 
 class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
@@ -63,7 +62,8 @@ class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
         page = text.extract(page, '<table id="songlist">', '</table>')[0]
         for num, url in enumerate(text.extract_iter(
                 page, '<td class="clickable-row"><a href="', '"'), 1):
-            page = self.request(urljoin(self.root, url), encoding="utf-8").text
+            url = text.urljoin(self.root, url)
+            page = self.request(url, encoding="utf-8").text
             url = text.extract(
                 page, '<p><a style="color: #21363f;" href="', '"')[0]
             yield url, text.nameext_from_url(url, {"num": num})
diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py
index 677d4049..c8776199 100644
--- a/gallery_dl/extractor/mangadex.py
+++ b/gallery_dl/extractor/mangadex.py
@@ -10,7 +10,6 @@
 
 from .common import ChapterExtractor, MangaExtractor
 from .. import text, util, exception
-from urllib.parse import urljoin
 import json
 import re
 
@@ -84,7 +83,7 @@ class MangadexChapterExtractor(MangadexExtractor, ChapterExtractor):
         pagelist, pos = text.extract(page, "var page_array = [", "]", pos)
         server  , pos = text.extract(page, "var server = '", "'", pos)
 
-        base = urljoin(self.root, server + dataurl + "/")
+        base = text.urljoin(self.root, server + dataurl + "/")
 
         return [
             (base + page, None)
diff --git a/gallery_dl/extractor/mangahere.py b/gallery_dl/extractor/mangahere.py
index b4ade702..968aaaae 100644
--- a/gallery_dl/extractor/mangahere.py
+++ b/gallery_dl/extractor/mangahere.py
@@ -10,7 +10,6 @@
 
 from .common import ChapterExtractor, MangaExtractor
 from .. import text
-from urllib.parse import urljoin
 import re
 
 
@@ -51,7 +50,7 @@ class MangahereMangaExtractor(MangaExtractor):
             volume, pos = text.extract(page, 'span class="mr6">', '<', pos)
             title, pos = text.extract(page, '/span>', '<', pos)
             date, pos = text.extract(page, 'class="right">', '</span>', pos)
-            results.append((urljoin("http:", url), {
+            results.append((text.urljoin("http:", url), {
                 "manga": manga, "title": title, "date": date,
                 "volume": text.parse_int(volume.rpartition(" ")[2]),
                 "chapter": text.parse_int(chapter),
diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py
index b12b7d45..9f62cbaa 100644
--- a/gallery_dl/extractor/mangapark.py
+++ b/gallery_dl/extractor/mangapark.py
@@ -10,7 +10,6 @@
 
 from .common import ChapterExtractor, MangaExtractor
 from .. import text
-from urllib.parse import urljoin
 
 
 class MangaparkExtractor():
@@ -120,7 +119,7 @@ class MangaparkChapterExtractor(MangaparkExtractor, ChapterExtractor):
             num += 1
             width , pos = text.extract(page, ' width="', '"', pos)
             height, pos = text.extract(page, ' _heighth="', '"', pos)
-            yield urljoin(self.root, url), {
+            yield text.urljoin(self.root, url), {
                 "page": num,
                 "width": width,
                 "height": height,
diff --git a/gallery_dl/extractor/mangastream.py b/gallery_dl/extractor/mangastream.py
index 70c18f1a..12ae717a 100644
--- a/gallery_dl/extractor/mangastream.py
+++ b/gallery_dl/extractor/mangastream.py
@@ -10,7 +10,6 @@
 
 from .common import ChapterExtractor
 from .. import text
-from urllib.parse import urljoin
 
 
 class MangastreamChapterExtractor(ChapterExtractor):
@@ -47,5 +46,5 @@ class MangastreamChapterExtractor(ChapterExtractor):
             pos = page.index(' class="page"')
             next_url = text.extract(page, ' href="', '"', pos)[0]
             image_url = text.extract(page, ' src="', '"', pos)[0]
-            yield urljoin(self.base_url, image_url), None
-            page = self.request(urljoin(self.base_url, next_url)).text
+            yield text.urljoin(self.base_url, image_url), None
+            page = self.request(text.urljoin(self.base_url, next_url)).text

From 8bf3cdd82bb5822688b00c3f54285e884b742f20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Tue, 1 May 2018 17:54:52 +0200
Subject: [PATCH 10/21] implement logging options

Standard logging to stderr, logfiles, and unsupported URL files (which
are now handled through the logging module) can now be configured by
setting their respective option keys (log, logfile, unsupportedfile)
to a dict and specifying the following options;

- format:
    format string for logging messages
    available keys: see [1]
    default: "[{name}][{levelname}] {message}"
- format-date:
    format string for {asctime} fields in logging messages
    available keys: see [2]
    default: "%Y-%m-%d %H:%M:%S"
- level:
    the lowercase levelname until which the logger should activate;
    available levels are debug, info, warning, error, exception
    default: "info"
- path:
    path of the file to be written to
- mode:
    'mode' argument when opening the specified file
    can be either "w" to truncate the file or "a" to append to it (see [3])

If 'output.log', '.logfile', or '.unsupportedfile' is a string, it will
be interpreted, as it has been, as the filepath
(or as format string for .log)

[1] https://docs.python.org/3/library/logging.html#logrecord-attributes
[2] https://docs.python.org/3/library/time.html#time.strftime
[3] https://docs.python.org/3/library/functions.html#open
---
 docs/gallery-dl-example.conf | 16 ++++++-
 gallery_dl/__init__.py       | 91 +++++++++++++++++++++++++++---------
 gallery_dl/job.py            |  6 +--
 3 files changed, 87 insertions(+), 26 deletions(-)

diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf
index 16445e2e..a3ca7f63 100644
--- a/docs/gallery-dl-example.conf
+++ b/docs/gallery-dl-example.conf
@@ -100,7 +100,21 @@
     "output":
     {
         "mode": "terminal",
-        "logfile": "~/gallery-dl/log.txt"
+        "log": {
+            "format": "{name}: {message}",
+            "level": "info"
+        },
+        "logfile": {
+            "path": "~/gallery-dl/log.txt",
+            "mode": "w",
+            "level": "debug"
+        },
+        "unsupportedfile": {
+            "path": "~/gallery-dl/unsupported.txt",
+            "mode": "a",
+            "format": "{asctime} {message}",
+            "format-date": "%Y-%m-%d-%H-%M-%S"
+        }
     },
 
     "cache": {
diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py
index 8e62f9c7..e698de62 100644
--- a/gallery_dl/__init__.py
+++ b/gallery_dl/__init__.py
@@ -27,20 +27,72 @@ from . import version, config, option, extractor, job, util, exception
 __version__ = version.__version__
 log = logging.getLogger("gallery-dl")
 
-def initialize_logging(loglevel, formatter):
+LOG_FORMAT = "[{name}][{levelname}] {message}"
+LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S"
+LOG_LEVEL = logging.INFO
+
+def initialize_logging(loglevel):
     """Setup basic logging functionality before configfiles have been loaded"""
     # convert levelnames to lowercase
     for level in (10, 20, 30, 40, 50):
         name = logging.getLevelName(level)
         logging.addLevelName(level, name.lower())
     # setup basic logging to stderr
+    formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{")
     handler = logging.StreamHandler()
     handler.setFormatter(formatter)
+    handler.setLevel(loglevel)
     root = logging.getLogger()
-    root.setLevel(loglevel)
+    root.setLevel(logging.NOTSET)
     root.addHandler(handler)
 
 
+def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
+    """Setup a new logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return None
+    if isinstance(opts, str):
+        opts = {"path": opts}
+
+    path = opts.get("path")
+    mode = opts.get("mode", "w")
+    try:
+        path = util.expand_path(path)
+        handler = logging.FileHandler(path, mode)
+    except (OSError, ValueError) as exc:
+        log.warning("%s: %s", key, exc)
+        return None
+    except TypeError as exc:
+        log.warning("%s: missing or invalid path (%s)", key, exc)
+        return None
+
+    level = opts.get("level", lvl)
+    logfmt = opts.get("format", fmt)
+    datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+    formatter = logging.Formatter(logfmt, datefmt, "{")
+    handler.setFormatter(formatter)
+    handler.setLevel(level)
+
+    return handler
+
+
+def configure_logging_handler(key, handler):
+    """Configure a logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return
+    if isinstance(opts, str):
+        opts = {"format": opts}
+    if handler.level == LOG_LEVEL and "level" in opts:
+        handler.setLevel(opts["level"])
+    if "format" in opts or "format-date" in opts:
+        logfmt = opts.get("format", LOG_FORMAT)
+        datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+        formatter = logging.Formatter(logfmt, datefmt, "{")
+        handler.setFormatter(formatter)
+
+
 def replace_std_streams(errors="replace"):
     """Replace standard streams and set their error handlers to 'errors'"""
     for name in ("stdout", "stdin", "stderr"):
@@ -159,8 +211,7 @@ def main():
         parser = option.build_parser()
         args = parser.parse_args()
 
-        formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s")
-        initialize_logging(args.loglevel, formatter)
+        initialize_logging(args.loglevel)
 
         # configuration
         if args.load_config:
@@ -173,17 +224,13 @@ def main():
             config.set(key, value)
         config.set(("_",), {})
 
-        # logfile
-        logfile = config.interpolate(("output", "logfile"))
-        if logfile:
-            try:
-                path = util.expand_path(logfile)
-                handler = logging.FileHandler(path, "w")
-            except OSError as exc:
-                log.warning("log file: %s", exc)
-            else:
-                handler.setFormatter(formatter)
-                logging.getLogger().addHandler(handler)
+        # stream logging handler
+        configure_logging_handler("log", logging.getLogger().handlers[0])
+
+        # file logging handler
+        handler = setup_logging_handler("logfile", lvl=args.loglevel)
+        if handler:
+            logging.getLogger().addHandler(handler)
 
         # loglevels
         if args.loglevel >= logging.ERROR:
@@ -243,13 +290,13 @@ def main():
                 except OSError as exc:
                     log.warning("input file: %s", exc)
 
-            unsupportedfile = config.interpolate(("output", "unsupportedfile"))
-            if unsupportedfile:
-                try:
-                    path = util.expand_path(unsupportedfile)
-                    job.Job.ufile = open(path, "w")
-                except OSError as exc:
-                    log.warning("unsupported-URL file: %s", exc)
+            # unsupported file logging handler
+            handler = setup_logging_handler("unsupportedfile", fmt="{message}")
+            if handler:
+                ulog = logging.getLogger("unsupported")
+                ulog.addHandler(handler)
+                ulog.propagate = False
+                job.Job.ulog = ulog
 
             prepare_range(args.image_range, "image")
             prepare_range(args.chapter_range, "chapter")
diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 3e31e4cd..61755f3a 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -17,7 +17,7 @@ from .extractor.message import Message
 
 class Job():
     """Base class for Job-types"""
-    ufile = None
+    ulog = None
 
     def __init__(self, url, parent=None):
         self.url = url
@@ -144,8 +144,8 @@ class Job():
             kwdict.update(self.userkwds)
 
     def _write_unsupported(self, url):
-        if self.ufile:
-            print(url, file=self.ufile, flush=True)
+        if self.ulog:
+            self.ulog.info(url)
 
 
 class DownloadJob(Job):

From 0e3883303fbbc546307cd28b1a1613a6c9751c3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Mon, 7 May 2018 20:05:44 +0200
Subject: [PATCH 11/21] [pixiv] implement AppAPI wrapper

---
 gallery_dl/extractor/pixiv.py | 98 +++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index e7289e26..e94f4105 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -429,3 +429,101 @@ class PixivAPI():
             if pinfo["current"] == pinfo["pages"]:
                 return
             params["page"] = pinfo["next"]
+
+
+class PixivAppAPI():
+    """Minimal interface for the Pixiv App-API for mobile devices
+
+    For a more complete implementation, see
+    - https://github.com/upbit/pixivpy
+    """
+    CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
+    CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
+
+    def __init__(self, extractor):
+        self.session = extractor.session
+        self.log = extractor.log
+        self.username, self.password = extractor._get_auth_info()
+        self.user_info = None
+
+        self.client_id = extractor.config(
+            "client-id", self.CLIENT_ID)
+        self.client_secret = extractor.config(
+            "client-secret", self.CLIENT_SECRET)
+
+        self.session.headers.update({
+            "App-OS": "ios",
+            "App-OS-Version": "10.3.1",
+            "App-Version": "6.7.1",
+            "User-Agent": "PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)",
+        })
+
+    def illust_detail(self, illust_id):
+        params = {"illust_id": illust_id}
+        return self._call("v1/illust/detail", params)
+
+    def illust_ranking(self, mode="day", date=None):
+        params = {"mode": mode, "date": date}
+        return self._pagination("v1/illust/ranking", params)
+
+    def user_detail(self, user_id):
+        params = {"user_id": user_id}
+        return self._call("v1/user/detail", params)
+
+    def user_illusts(self, user_id, illust_type=None):
+        params = {"user_id": user_id, "type": illust_type}
+        return self._pagination("v1/user/illusts", params)
+
+    def ugoira_metadata(self, illust_id):
+        params = {"illust_id": illust_id}
+        return self._call("v1/ugoira/metadata", params)
+
+    def authenticate(self):
+        """Authenticate the application by requesting an access token"""
+        self.user_info, auth = self._authenticate_impl(
+            self.username, self.password)
+        self.session.headers["Authorization"] = auth
+
+    @cache(maxage=3590, keyarg=1)
+    def _authenticate_impl(self, username, password):
+        """Actual authenticate implementation"""
+        self.log.info("Logging in as %s", username)
+
+        url = "https://oauth.secure.pixiv.net/auth/token"
+        data = {
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "grant_type": "password",
+            "username": username,
+            "password": password,
+            "get_secure_url": 1,
+        }
+
+        response = self.session.post(url, data=data)
+        if response.status_code >= 400:
+            raise exception.AuthenticationError()
+
+        data = response.json()["response"]
+        return data["user"], "Bearer " + data["access_token"]
+
+    def _call(self, endpoint, params=None):
+        url = "https://app-api.pixiv.net/" + endpoint
+
+        self.authenticate()
+        response = self.session.get(url, params=params)
+
+        if 200 <= response.status_code < 400:
+            return response.json()
+        if response.status_code == 404:
+            raise exception.NotFoundError()
+        self.log.error("API request failed: %s", response.text)
+        raise exception.StopExtraction()
+
+    def _pagination(self, endpoint, params):
+        while True:
+            data = self._call(endpoint, params)
+            yield from data["illusts"]
+
+            if not data["next_url"]:
+                return
+            params["offset"] = data["next_url"].rpartition("=")[2]

From 6a31ada9e3733c48a58ea03baece22aefd2eea0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 10 May 2018 18:26:10 +0200
Subject: [PATCH 12/21] re-implement OAuth1.0 code

OAuth support for SmugMug needs some additional features
(auth-rebuild on redirect, query parameters in URL, ...)
and fixing this in the old code wouldn't work all that well.
---
 gallery_dl/extractor/flickr.py  |  17 +++---
 gallery_dl/extractor/oauth.py   |  18 +++---
 gallery_dl/extractor/smugmug.py |   5 +-
 gallery_dl/extractor/tumblr.py  |   5 +-
 gallery_dl/oauth.py             | 101 ++++++++++++++++++++++++++++++++
 gallery_dl/util.py              |  53 -----------------
 test/test_oauth.py              |  52 ++++------------
 7 files changed, 132 insertions(+), 119 deletions(-)
 create mode 100644 gallery_dl/oauth.py

diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py
index 150b2a28..967ec114 100644
--- a/gallery_dl/extractor/flickr.py
+++ b/gallery_dl/extractor/flickr.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.flickr.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, oauth, util, exception
 
 
 class FlickrExtractor(Extractor):
@@ -264,17 +264,20 @@ class FlickrAPI():
     ]
 
     def __init__(self, extractor):
-        self.api_key = extractor.config("api-key", self.API_KEY)
-        self.api_secret = extractor.config("api-secret", self.API_SECRET)
+        api_key = extractor.config("api-key", self.API_KEY)
+        api_secret = extractor.config("api-secret", self.API_SECRET)
         token = extractor.config("access-token")
         token_secret = extractor.config("access-token-secret")
-        if token and token_secret:
-            self.session = util.OAuthSession(
-                extractor.session,
-                self.api_key, self.api_secret, token, token_secret)
+
+        if api_key and api_secret and token and token_secret:
+            self.session = oauth.OAuth1Session(
+                api_key, api_secret,
+                token, token_secret,
+            )
             self.api_key = None
         else:
             self.session = extractor.session
+            self.api_key = api_key
 
         self.maxsize = extractor.config("size-max")
         if isinstance(self.maxsize, str):
diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py
index f161126e..6d8a1c7a 100644
--- a/gallery_dl/extractor/oauth.py
+++ b/gallery_dl/extractor/oauth.py
@@ -10,7 +10,7 @@
 
 from .common import Extractor, Message
 from . import deviantart, flickr, reddit, tumblr
-from .. import text, util, config
+from .. import text, oauth, config
 import os
 import urllib.parse
 
@@ -70,21 +70,19 @@ class OAuthBase(Extractor):
     def _oauth1_authorization_flow(
             self, request_token_url, authorize_url, access_token_url):
         """Perform the OAuth 1.0a authorization flow"""
-        del self.session.params["oauth_token"]
-
         # get a request token
         params = {"oauth_callback": self.redirect_uri}
         data = self.session.get(request_token_url, params=params).text
 
         data = text.parse_query(data)
-        self.session.params["oauth_token"] = token = data["oauth_token"]
-        self.session.token_secret = data["oauth_token_secret"]
+        self.session.auth.token_secret = data["oauth_token_secret"]
 
         # get the user's authorization
-        params = {"oauth_token": token, "perms": "read"}
+        params = {"oauth_token": data["oauth_token"], "perms": "read"}
         data = self.open(authorize_url, params)
 
         # exchange the request token for an access token
+        # self.session.token = data["oauth_token"]
         data = self.session.get(access_token_url, params=data).text
 
         data = text.parse_query(data)
@@ -101,7 +99,7 @@ class OAuthBase(Extractor):
 
         state = "gallery-dl_{}_{}".format(
             self.subcategory,
-            util.OAuthSession.nonce(8)
+            oauth.nonce(8),
         )
 
         auth_params = {
@@ -182,8 +180,7 @@ class OAuthFlickr(OAuthBase):
 
     def __init__(self, match):
         OAuthBase.__init__(self, match)
-        self.session = util.OAuthSession(
-            self.session,
+        self.session = oauth.OAuth1Session(
             self.oauth_config("api-key", flickr.FlickrAPI.API_KEY),
             self.oauth_config("api-secret", flickr.FlickrAPI.API_SECRET),
         )
@@ -221,8 +218,7 @@ class OAuthTumblr(OAuthBase):
 
     def __init__(self, match):
         OAuthBase.__init__(self, match)
-        self.session = util.OAuthSession(
-            self.session,
+        self.session = oauth.OAuth1Session(
             self.oauth_config("api-key", tumblr.TumblrAPI.API_KEY),
             self.oauth_config("api-secret", tumblr.TumblrAPI.API_SECRET),
         )
diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py
index 3306bc58..fa5fa751 100644
--- a/gallery_dl/extractor/smugmug.py
+++ b/gallery_dl/extractor/smugmug.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.smugmug.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, oauth, exception
 
 BASE_PATTERN = (
     r"(?:smugmug:(?!album:)(?:https?://)?([^/]+)|"
@@ -186,8 +186,7 @@ class SmugmugAPI():
         token_secret = extractor.config("access-token-secret")
 
         if api_key and api_secret and token and token_secret:
-            self.session = util.OAuthSession(
-                extractor.session,
+            self.session = oauth.OAuth1Session(
                 api_key, api_secret,
                 token, token_secret,
             )
diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py
index 770ca03f..8a87d8a5 100644
--- a/gallery_dl/extractor/tumblr.py
+++ b/gallery_dl/extractor/tumblr.py
@@ -9,7 +9,7 @@
 """Extract images from https://www.tumblr.com/"""
 
 from .common import Extractor, Message
-from .. import text, util, exception
+from .. import text, oauth, exception
 from datetime import datetime, timedelta
 import re
 import time
@@ -261,8 +261,7 @@ class TumblrAPI():
         token_secret = extractor.config("access-token-secret")
 
         if api_key and api_secret and token and token_secret:
-            self.session = util.OAuthSession(
-                extractor.session,
+            self.session = oauth.OAuth1Session(
                 api_key, api_secret,
                 token, token_secret,
             )
diff --git a/gallery_dl/oauth.py b/gallery_dl/oauth.py
new file mode 100644
index 00000000..459b0d43
--- /dev/null
+++ b/gallery_dl/oauth.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""OAuth helper functions and classes"""
+
+import hmac
+import time
+import base64
+import random
+import string
+import hashlib
+import urllib.parse
+
+import requests
+import requests.auth
+
+from . import text
+
+
+class OAuth1Session(requests.Session):
+    """Extension to requests.Session objects to support OAuth 1.0"""
+
+    def __init__(self, consumer_key, consumer_secret,
+                 token=None, token_secret=None):
+
+        requests.Session.__init__(self)
+        self.auth = OAuth1Client(
+            consumer_key, consumer_secret,
+            token, token_secret,
+        )
+
+    def rebuild_auth(self, prepared_request, response):
+        if "Authorization" in prepared_request.headers:
+            del prepared_request.headers["Authorization"]
+            prepared_request.prepare_auth(self.auth)
+
+
+class OAuth1Client(requests.auth.AuthBase):
+    """OAuth1.0a authentication"""
+    def __init__(self, consumer_key, consumer_secret,
+                 token=None, token_secret=None):
+
+        self.consumer_key = consumer_key
+        self.consumer_secret = consumer_secret
+        self.token = token
+        self.token_secret = token_secret
+
+    def __call__(self, request):
+        oauth_params = [
+            ("oauth_consumer_key", self.consumer_key),
+            ("oauth_nonce", nonce(16)),
+            ("oauth_signature_method", "HMAC-SHA1"),
+            ("oauth_timestamp", str(int(time.time()))),
+            ("oauth_version", "1.0"),
+        ]
+        if self.token:
+            oauth_params.append(("oauth_token", self.token))
+
+        signature = self.generate_signature(request, oauth_params)
+        oauth_params.append(("oauth_signature", signature))
+
+        request.headers["Authorization"] = "OAuth " + ",".join(
+            key + '="' + value + '"' for key, value in oauth_params)
+
+        return request
+
+    def generate_signature(self, request, params):
+        """Generate 'oauth_signature' value"""
+        url, _, query = request.url.partition("?")
+
+        params = params.copy()
+        for key, value in text.parse_query(query).items():
+            params.append((quote(key), quote(value)))
+        params.sort()
+        query = "&".join("=".join(item) for item in params)
+
+        message = concat(request.method, url, query).encode()
+        key = concat(self.consumer_secret, self.token_secret or "").encode()
+        signature = hmac.new(key, message, hashlib.sha1).digest()
+
+        return quote(base64.b64encode(signature).decode())
+
+
+def concat(*args):
+    """Concatenate 'args'"""
+    return "&".join(quote(item) for item in args)
+
+
+def nonce(size, alphabet=string.ascii_letters):
+    """Generate a nonce value with 'size' characters"""
+    return "".join(random.choice(alphabet) for _ in range(size))
+
+
+def quote(value, quote=urllib.parse.quote):
+    """Quote 'value' according to the OAuth1.0 standard"""
+    return quote(value, "~")
diff --git a/gallery_dl/util.py b/gallery_dl/util.py
index 642162e4..d488d039 100644
--- a/gallery_dl/util.py
+++ b/gallery_dl/util.py
@@ -11,14 +11,9 @@
 import re
 import os
 import sys
-import hmac
-import time
-import base64
-import random
 import shutil
 import string
 import _string
-import hashlib
 import sqlite3
 import datetime
 import itertools
@@ -497,54 +492,6 @@ class PathFormat():
         return "\\\\?\\" + os.path.abspath(path) if os.name == "nt" else path
 
 
-class OAuthSession():
-    """Minimal wrapper for requests.session objects to support OAuth 1.0"""
-    def __init__(self, session, consumer_key, consumer_secret,
-                 token=None, token_secret=None):
-        self.session = session
-        self.consumer_secret = consumer_secret
-        self.token_secret = token_secret or ""
-        self.params = {}
-        self.params["oauth_consumer_key"] = consumer_key
-        self.params["oauth_token"] = token
-        self.params["oauth_signature_method"] = "HMAC-SHA1"
-        self.params["oauth_version"] = "1.0"
-
-    def get(self, url, params, **kwargs):
-        params.update(self.params)
-        params["oauth_nonce"] = self.nonce(16)
-        params["oauth_timestamp"] = int(time.time())
-        return self.session.get(url + self.sign(url, params), **kwargs)
-
-    def sign(self, url, params):
-        """Generate 'oauth_signature' value and return query string"""
-        query = self.urlencode(params)
-        message = self.concat("GET", url, query).encode()
-        key = self.concat(self.consumer_secret, self.token_secret).encode()
-        signature = hmac.new(key, message, hashlib.sha1).digest()
-        return "?{}&oauth_signature={}".format(
-            query, self.quote(base64.b64encode(signature).decode()))
-
-    @staticmethod
-    def concat(*args):
-        return "&".join(OAuthSession.quote(item) for item in args)
-
-    @staticmethod
-    def nonce(N, alphabet=string.ascii_letters):
-        return "".join(random.choice(alphabet) for _ in range(N))
-
-    @staticmethod
-    def quote(value, quote=urllib.parse.quote):
-        return quote(value, "~")
-
-    @staticmethod
-    def urlencode(params):
-        return "&".join(
-            OAuthSession.quote(str(key)) + "=" + OAuthSession.quote(str(value))
-            for key, value in sorted(params.items()) if value
-        )
-
-
 class DownloadArchive():
 
     def __init__(self, path, extractor):
diff --git a/test/test_oauth.py b/test/test_oauth.py
index 36f1bd96..5176cc35 100644
--- a/test/test_oauth.py
+++ b/test/test_oauth.py
@@ -8,10 +8,8 @@
 # published by the Free Software Foundation.
 
 import unittest
-import requests
 
-from gallery_dl import text
-from gallery_dl.util import OAuthSession
+from gallery_dl import oauth, text
 
 TESTSERVER = "http://oauthbin.com"
 CONSUMER_KEY = "key"
@@ -25,7 +23,7 @@ ACCESS_TOKEN_SECRET = "accesssecret"
 class TestOAuthSession(unittest.TestCase):
 
     def test_concat(self):
-        concat = OAuthSession.concat
+        concat = oauth.concat
 
         self.assertEqual(concat(), "")
         self.assertEqual(concat("str"), "str")
@@ -37,18 +35,18 @@ class TestOAuthSession(unittest.TestCase):
             "GET&http%3A%2F%2Fexample.org%2F&foo%3Dbar%26baz%3Da"
         )
 
-    def test_nonce(self, N=16):
-        nonce_values = set(OAuthSession.nonce(N) for _ in range(N))
+    def test_nonce(self, size=16):
+        nonce_values = set(oauth.nonce(size) for _ in range(size))
 
         # uniqueness
-        self.assertEqual(len(nonce_values), N)
+        self.assertEqual(len(nonce_values), size)
 
         # length
         for nonce in nonce_values:
-            self.assertEqual(len(nonce), N)
+            self.assertEqual(len(nonce), size)
 
     def test_quote(self):
-        quote = OAuthSession.quote
+        quote = oauth.quote
 
         reserved = ",;:!\"§$%&/(){}[]=?`´+*'äöü"
         unreserved = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -65,33 +63,6 @@ class TestOAuthSession(unittest.TestCase):
             self.assertTrue(len(quoted) >= 3)
             self.assertEqual(quoted_hex.upper(), quoted_hex)
 
-    def test_urlencode(self):
-        urlencode = OAuthSession.urlencode
-
-        self.assertEqual(urlencode({}), "")
-        self.assertEqual(urlencode({"foo": "bar"}), "foo=bar")
-
-        self.assertEqual(
-            urlencode({"foo": "bar", "baz": "a", "a": "baz"}),
-            "a=baz&baz=a&foo=bar"
-        )
-        self.assertEqual(
-            urlencode({
-                "oauth_consumer_key": "0685bd9184jfhq22",
-                "oauth_token": "ad180jjd733klru7",
-                "oauth_signature_method": "HMAC-SHA1",
-                "oauth_timestamp": 137131200,
-                "oauth_nonce": "4572616e48616d6d65724c61686176",
-                "oauth_version": "1.0"
-            }),
-            "oauth_consumer_key=0685bd9184jfhq22&"
-            "oauth_nonce=4572616e48616d6d65724c61686176&"
-            "oauth_signature_method=HMAC-SHA1&"
-            "oauth_timestamp=137131200&"
-            "oauth_token=ad180jjd733klru7&"
-            "oauth_version=1.0"
-        )
-
     def test_request_token(self):
         response = self._oauth_request(
             "/v1/request-token", {})
@@ -113,23 +84,20 @@ class TestOAuthSession(unittest.TestCase):
         self.assertTrue(data["oauth_token_secret"], ACCESS_TOKEN_SECRET)
 
     def test_authenticated_call(self):
-        params = {"method": "foo", "bar": "baz", "a": "äöüß/?&#"}
+        params = {"method": "foo", "a": "äöüß/?&#", "äöüß/?&#": "a"}
         response = self._oauth_request(
             "/v1/echo", params, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
-        expected = OAuthSession.urlencode(params)
 
-        self.assertEqual(response, expected, msg=response)
         self.assertEqual(text.parse_query(response), params)
 
     def _oauth_request(self, endpoint, params=None,
                        oauth_token=None, oauth_token_secret=None):
-        session = OAuthSession(
-            requests.session(),
+        session = oauth.OAuth1Session(
             CONSUMER_KEY, CONSUMER_SECRET,
             oauth_token, oauth_token_secret,
         )
         url = TESTSERVER + endpoint
-        return session.get(url, params.copy()).text
+        return session.get(url, params=params).text
 
 
 if __name__ == "__main__":

From f3483a2b7c6fb02395d5d22c1ef9cdb3e4edce1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 10 May 2018 18:58:05 +0200
Subject: [PATCH 13/21] [smugmug] add OAuth support

---
 README.rst                      |  2 +-
 docs/supportedsites.rst         |  2 +-
 gallery_dl/extractor/oauth.py   | 23 ++++++++++++++++++++++-
 scripts/build_supportedsites.py |  1 +
 4 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index cbdfdfc6..4302e457 100644
--- a/README.rst
+++ b/README.rst
@@ -194,7 +194,7 @@ OAuth
 -----
 
 *gallery-dl* supports user authentication via OAuth_ for
-``deviantart``, ``flickr``, ``reddit`` and ``tumblr``.
+``deviantart``, ``flickr``, ``reddit``, ``smugmug`` and ``tumblr``.
 This is entirely optional, but grants *gallery-dl* the ability
 to issue requests on your account's behalf and enables it to access resources
 which would otherwise be unavailable to a public user.
diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst
index 3a7a8fcf..da6b774a 100644
--- a/docs/supportedsites.rst
+++ b/docs/supportedsites.rst
@@ -68,7 +68,7 @@ Sea Otter Scans      https://reader.seaotterscans.com/   Chapters, Manga
 Sen Manga            http://raw.senmanga.com/            Chapters
 Sense-Scans          http://sensescans.com/              Chapters, Manga
 SlideShare           https://www.slideshare.net/         Presentations
-SmugMug              https://www.smugmug.com/            |Albums, individ-5|
+SmugMug              https://www.smugmug.com/            |Albums, individ-5|                                Optional (OAuth)
 Subapics             https://subapics.com/               Chapters, Manga
 The /b/ Archive      https://thebarchive.com/            Threads
 Tumblr               https://www.tumblr.com/             Images from Users, Likes, Posts, Tag-Searches      Optional (OAuth)
diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py
index 6d8a1c7a..7c682541 100644
--- a/gallery_dl/extractor/oauth.py
+++ b/gallery_dl/extractor/oauth.py
@@ -9,7 +9,7 @@
 """Utility classes to setup OAuth and link a users account to gallery-dl"""
 
 from .common import Extractor, Message
-from . import deviantart, flickr, reddit, tumblr
+from . import deviantart, flickr, reddit, smugmug, tumblr
 from .. import text, oauth, config
 import os
 import urllib.parse
@@ -212,6 +212,27 @@ class OAuthReddit(OAuthBase):
         )
 
 
+class OAuthSmugmug(OAuthBase):
+    subcategory = "smugmug"
+    pattern = ["oauth:smugmug$"]
+
+    def __init__(self, match):
+        OAuthBase.__init__(self, match)
+        self.session = oauth.OAuth1Session(
+            self.oauth_config("api-key", smugmug.SmugmugAPI.API_KEY),
+            self.oauth_config("api-secret", smugmug.SmugmugAPI.API_SECRET),
+        )
+
+    def items(self):
+        yield Message.Version, 1
+
+        self._oauth1_authorization_flow(
+            "https://api.smugmug.com/services/oauth/1.0a/getRequestToken",
+            "https://api.smugmug.com/services/oauth/1.0a/authorize",
+            "https://api.smugmug.com/services/oauth/1.0a/getAccessToken",
+        )
+
+
 class OAuthTumblr(OAuthBase):
     subcategory = "tumblr"
     pattern = ["oauth:tumblr$"]
diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py
index db12075d..16567b63 100755
--- a/scripts/build_supportedsites.py
+++ b/scripts/build_supportedsites.py
@@ -87,6 +87,7 @@ AUTH_MAP = {
     "reddit"     : "Optional (OAuth)",
     "sankaku"    : "Optional",
     "seiga"      : "Required",
+    "smugmug"    : "Optional (OAuth)",
     "tumblr"     : "Optional (OAuth)",
 }
 

From 1c1e086d01ffd13b33751daa139c4a481baaa8c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 10 May 2018 21:57:45 +0200
Subject: [PATCH 14/21] use common base class for OAuth1.0 based API interfaces

---
 gallery_dl/extractor/flickr.py  | 17 ++----------
 gallery_dl/extractor/smugmug.py | 20 +-------------
 gallery_dl/extractor/tumblr.py  | 19 ++-----------
 gallery_dl/oauth.py             | 47 +++++++++++++++++++++++++--------
 4 files changed, 41 insertions(+), 62 deletions(-)

diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py
index 967ec114..cea46b1d 100644
--- a/gallery_dl/extractor/flickr.py
+++ b/gallery_dl/extractor/flickr.py
@@ -243,7 +243,7 @@ class FlickrSearchExtractor(FlickrExtractor):
         return self.api.photos_search(self.search)
 
 
-class FlickrAPI():
+class FlickrAPI(oauth.OAuth1API):
     """Minimal interface for the flickr API"""
     API_URL = "https://api.flickr.com/services/rest/"
     API_KEY = "ac4fd7aa98585b9eee1ba761c209de68"
@@ -264,20 +264,7 @@ class FlickrAPI():
     ]
 
     def __init__(self, extractor):
-        api_key = extractor.config("api-key", self.API_KEY)
-        api_secret = extractor.config("api-secret", self.API_SECRET)
-        token = extractor.config("access-token")
-        token_secret = extractor.config("access-token-secret")
-
-        if api_key and api_secret and token and token_secret:
-            self.session = oauth.OAuth1Session(
-                api_key, api_secret,
-                token, token_secret,
-            )
-            self.api_key = None
-        else:
-            self.session = extractor.session
-            self.api_key = api_key
+        oauth.OAuth1API.__init__(self, extractor)
 
         self.maxsize = extractor.config("size-max")
         if isinstance(self.maxsize, str):
diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py
index fa5fa751..90bbd94e 100644
--- a/gallery_dl/extractor/smugmug.py
+++ b/gallery_dl/extractor/smugmug.py
@@ -171,7 +171,7 @@ class SmugmugPathExtractor(SmugmugExtractor):
                 yield from self.album_nodes(node)
 
 
-class SmugmugAPI():
+class SmugmugAPI(oauth.OAuth1API):
     """Minimal interface for the smugmug API v2"""
     API_DOMAIN = "api.smugmug.com"
     API_KEY = "DFqxg4jf7GrtsQ5PnbNB8899zKfnDrdK"
@@ -179,24 +179,6 @@ class SmugmugAPI():
                   "9nMFQm25ndGBzNPnwRDbRnnVBvqt4xTq")
     HEADERS = {"Accept": "application/json"}
 
-    def __init__(self, extractor):
-        api_key = extractor.config("api-key", self.API_KEY)
-        api_secret = extractor.config("api-secret", self.API_SECRET)
-        token = extractor.config("access-token")
-        token_secret = extractor.config("access-token-secret")
-
-        if api_key and api_secret and token and token_secret:
-            self.session = oauth.OAuth1Session(
-                api_key, api_secret,
-                token, token_secret,
-            )
-            self.api_key = None
-        else:
-            self.session = extractor.session
-            self.api_key = api_key
-
-        self.log = extractor.log
-
     def album(self, album_id, expands=None):
         return self._expansion("album/" + album_id, expands)
 
diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py
index 8a87d8a5..d0b5e64d 100644
--- a/gallery_dl/extractor/tumblr.py
+++ b/gallery_dl/extractor/tumblr.py
@@ -248,30 +248,15 @@ class TumblrLikesExtractor(TumblrExtractor):
         return self.api.likes(self.blog)
 
 
-class TumblrAPI():
+class TumblrAPI(oauth.OAuth1API):
     """Minimal interface for the Tumblr API v2"""
     API_KEY = "O3hU2tMi5e4Qs5t3vezEi6L0qRORJ5y9oUpSGsrWu8iA3UCc3B"
     API_SECRET = "sFdsK3PDdP2QpYMRAoq0oDnw0sFS24XigXmdfnaeNZpJpqAn03"
     BLOG_CACHE = {}
 
     def __init__(self, extractor):
-        api_key = extractor.config("api-key", self.API_KEY)
-        api_secret = extractor.config("api-secret", self.API_SECRET)
-        token = extractor.config("access-token")
-        token_secret = extractor.config("access-token-secret")
-
-        if api_key and api_secret and token and token_secret:
-            self.session = oauth.OAuth1Session(
-                api_key, api_secret,
-                token, token_secret,
-            )
-            self.api_key = None
-        else:
-            self.session = extractor.session
-            self.api_key = api_key
-
+        oauth.OAuth1API.__init__(self, extractor)
         self.posts_type = None
-        self.log = extractor.log
 
     def info(self, blog):
         """Return general information about a blog"""
diff --git a/gallery_dl/oauth.py b/gallery_dl/oauth.py
index 459b0d43..ee66504d 100644
--- a/gallery_dl/oauth.py
+++ b/gallery_dl/oauth.py
@@ -22,8 +22,23 @@ import requests.auth
 from . import text
 
 
+def nonce(size, alphabet=string.ascii_letters):
+    """Generate a nonce value with 'size' characters"""
+    return "".join(random.choice(alphabet) for _ in range(size))
+
+
+def quote(value, quote=urllib.parse.quote):
+    """Quote 'value' according to the OAuth1.0 standard"""
+    return quote(value, "~")
+
+
+def concat(*args):
+    """Concatenate 'args' as expected by OAuth1.0"""
+    return "&".join(quote(item) for item in args)
+
+
 class OAuth1Session(requests.Session):
-    """Extension to requests.Session objects to support OAuth 1.0"""
+    """Extension to requests.Session to support OAuth 1.0"""
 
     def __init__(self, consumer_key, consumer_secret,
                  token=None, token_secret=None):
@@ -42,6 +57,7 @@ class OAuth1Session(requests.Session):
 
 class OAuth1Client(requests.auth.AuthBase):
     """OAuth1.0a authentication"""
+
     def __init__(self, consumer_key, consumer_secret,
                  token=None, token_secret=None):
 
@@ -86,16 +102,25 @@ class OAuth1Client(requests.auth.AuthBase):
         return quote(base64.b64encode(signature).decode())
 
 
-def concat(*args):
-    """Concatenate 'args'"""
-    return "&".join(quote(item) for item in args)
+class OAuth1API():
+    """Base class for OAuth1.0 based API interfaces"""
+    API_KEY = None
+    API_SECRET = None
 
+    def __init__(self, extractor):
+        self.log = extractor.log
 
-def nonce(size, alphabet=string.ascii_letters):
-    """Generate a nonce value with 'size' characters"""
-    return "".join(random.choice(alphabet) for _ in range(size))
+        api_key = extractor.config("api-key", self.API_KEY)
+        api_secret = extractor.config("api-secret", self.API_SECRET)
+        token = extractor.config("access-token")
+        token_secret = extractor.config("access-token-secret")
 
-
-def quote(value, quote=urllib.parse.quote):
-    """Quote 'value' according to the OAuth1.0 standard"""
-    return quote(value, "~")
+        if api_key and api_secret and token and token_secret:
+            self.log.debug("Using OAuth1.0 authentication")
+            self.session = OAuth1Session(
+                api_key, api_secret, token, token_secret)
+            self.api_key = None
+        else:
+            self.log.debug("Using api_key authentication")
+            self.session = extractor.session
+            self.api_key = api_key

From 7a58151566519f6ec0d3ed4fedcc9c621a2969f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 10 May 2018 22:07:55 +0200
Subject: [PATCH 15/21] fix util.parse_bytes invocations

(should be text.parse_bytes)
---
 gallery_dl/extractor/exhentai.py | 4 ++--
 gallery_dl/extractor/paheal.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py
index 2af58110..4b07abcb 100644
--- a/gallery_dl/extractor/exhentai.py
+++ b/gallery_dl/extractor/exhentai.py
@@ -164,7 +164,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
         data["title"] = text.unescape(data["title"])
         data["title_jp"] = text.unescape(data["title_jp"])
         data["count"] = text.parse_int(data["count"])
-        data["gallery_size"] = util.parse_bytes(
+        data["gallery_size"] = text.parse_bytes(
             data["gallery_size"].rstrip("Bb"))
         return data
 
@@ -256,7 +256,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
         return {
             "width": text.parse_int(parts[0]),
             "height": text.parse_int(parts[2]),
-            "size": util.parse_bytes(parts[3] + parts[4][0]),
+            "size": text.parse_bytes(parts[3] + parts[4][0]),
         }
 
 
diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py
index 8cdb3569..998a1767 100644
--- a/gallery_dl/extractor/paheal.py
+++ b/gallery_dl/extractor/paheal.py
@@ -9,7 +9,7 @@
 """Extract images from http://rule34.paheal.net/"""
 
 from .common import SharedConfigExtractor, Message
-from .. import text, util
+from .. import text
 
 
 class PahealExtractor(SharedConfigExtractor):
@@ -85,7 +85,7 @@ class PahealTagExtractor(PahealExtractor):
         return {
             "id": pid, "md5": md5, "tags": tags, "file_url": url,
             "width": width, "height": height,
-            "size": util.parse_bytes(size[:-1]),
+            "size": text.parse_bytes(size[:-1]),
         }
 
 

From 247f785af18bb69f49f679452753946d3055981e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 13 May 2018 22:24:21 +0200
Subject: [PATCH 16/21] [pixiv] use App API

Transitioning to the App API breaks favorites archive IDs (there is
no longer any bookmark ID information), but the favorites API endpoint
of the public API was gone anyways ...
---
 gallery_dl/extractor/pixiv.py | 356 +++++++++++-----------------------
 1 file changed, 118 insertions(+), 238 deletions(-)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index e94f4105..c1f8f011 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -11,7 +11,7 @@
 from .common import Extractor, Message
 from .. import text, exception
 from ..cache import cache
-import re
+from datetime import datetime, timedelta
 
 
 class PixivExtractor(Extractor):
@@ -20,11 +20,10 @@ class PixivExtractor(Extractor):
     directory_fmt = ["{category}", "{user[id]} {user[account]}"]
     filename_fmt = "{category}_{user[id]}_{id}{num}.{extension}"
     archive_fmt = "{id}{num}.{extension}"
-    illust_url = "https://www.pixiv.net/member_illust.php?mode=medium"
 
     def __init__(self):
         Extractor.__init__(self)
-        self.api = PixivAPI(self)
+        self.api = PixivAppAPI(self)
         self.user_id = -1
         self.load_ugoira = self.config("ugoira", True)
 
@@ -35,70 +34,54 @@ class PixivExtractor(Extractor):
         yield Message.Directory, metadata
 
         for work in self.works():
-            work = self.prepare_work(work)
+            if not work["user"]["id"]:
+                continue
+
+            meta_single_page = work["meta_single_page"]
+            meta_pages = work["meta_pages"]
+            del work["meta_single_page"]
+            del work["image_urls"]
+            del work["meta_pages"]
+            work["num"] = ""
+            work["tags"] = [tag["name"] for tag in work["tags"]]
+            work.update(metadata)
 
             if work["type"] == "ugoira":
                 if not self.load_ugoira:
                     continue
-                url, framelist = self.parse_ugoira(work)
+                ugoira = self.api.ugoira_metadata(work["id"])
+
+                url = ugoira["zip_urls"]["medium"].replace(
+                    "_ugoira600x600", "_ugoira1920x1080")
                 work["extension"] = "zip"
                 yield Message.Url, url, work
+
+                framelist = "".join(
+                    "{file} {delay}\n".format_map(frame)
+                    for frame in ugoira["frames"]
+                )
                 work["extension"] = "txt"
-                yield Message.Url, "text:"+framelist, work
+                yield Message.Url, "text:" + framelist, work
 
             elif work["page_count"] == 1:
-                yield Message.Url, work["url"], work
+                url = meta_single_page["original_image_url"]
+                work["extension"] = url.rpartition(".")[2]
+                yield Message.Url, url, work
 
             else:
-                url, _, ext = work["url"].rpartition("_p0")
-                for i in range(work["page_count"]):
-                    work["num"] = "_p{:02}".format(i)
-                    yield Message.Url, "{}_p{}{}".format(url, i, ext), work
+                for num, img in enumerate(meta_pages):
+                    url = img["image_urls"]["original"]
+                    work["num"] = "_p{:02}".format(num)
+                    work["extension"] = url.rpartition(".")[2]
+                    yield Message.Url, url, work
 
     def works(self):
         """Return an iterable containing all relevant 'work'-objects"""
-        return []
-
-    def prepare_work(self, work):
-        """Prepare a work-dictionary with additional keywords"""
-        url = work["image_urls"]["large"]
-        del work["image_urls"]
-        work["num"] = ""
-        work["url"] = url
-        work["extension"] = url.rpartition(".")[2]
-        return work
-
-    def parse_ugoira(self, data):
-        """Parse ugoira data"""
-        # get illust page
-        page = self.request(
-            self.illust_url,
-            params={"illust_id": data["id"]},
-            headers={"User-Agent": "Mozilla/5.0"},
-        ).text
-
-        # parse page
-        frames = text.extract(page, ',"frames":[', ']')[0]
-
-        # build url
-        url = re.sub(
-            r"/img-original/(.+/\d+)[^/]+",
-            r"/img-zip-ugoira/\g<1>_ugoira1920x1080.zip",
-            data["url"]
-        )
-
-        # build framelist
-        framelist = re.sub(
-            r'\{"file":"([^"]+)","delay":(\d+)\},?',
-            r'\1 \2\n', frames
-        )
-
-        return url, framelist
 
     def get_metadata(self, user=None):
         """Collect metadata for extractor-job"""
         if not user:
-            user = self.api.user(self.user_id)[0]
+            user = self.api.user_detail(self.user_id)
         return {"user": user}
 
 
@@ -135,11 +118,11 @@ class PixivUserExtractor(PixivExtractor):
             self.works = self._tagged_works
 
     def works(self):
-        return self.api.user_works(self.user_id)
+        return self.api.user_illusts(self.user_id)
 
     def _tagged_works(self):
-        for work in self.api.user_works(self.user_id):
-            if self.tag in [tag.lower() for tag in work["tags"]]:
+        for work in self.api.user_illusts(self.user_id):
+            if self.tag in [tag["name"].lower() for tag in work["tags"]]:
                 yield work
 
 
@@ -188,14 +171,6 @@ class PixivWorkExtractor(PixivExtractor):
           "?mode=medium&illust_id=966411"), {
             "exception": exception.NotFoundError,
         }),
-        (("http://i1.pixiv.net/c/600x600/img-master/"
-          "img/2008/06/13/00/29/13/966412_p0_master1200.jpg"), {
-            "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba",
-        }),
-        (("https://i.pximg.net/img-original/"
-          "img/2017/04/25/07/33/29/62568267_p0.png"), {
-            "url": "71b8bbd070d6b03a75ca4afb89f64d1445b2278d",
-        }),
         # ugoira
         (("https://www.pixiv.net/member_illust.php"
           "?mode=medium&illust_id=66806629"), {
@@ -203,6 +178,10 @@ class PixivWorkExtractor(PixivExtractor):
                         r"66806629_ugoira1920x1080\.zip|text:.+"),
             "count": 2,
         }),
+        (("http://i1.pixiv.net/c/600x600/img-master/"
+          "img/2008/06/13/00/29/13/966412_p0_master1200.jpg"), None),
+        (("https://i.pximg.net/img-original/"
+          "img/2017/04/25/07/33/29/62568267_p0.png"), None),
         ("https://www.pixiv.net/i/966412", None),
         ("http://img.pixiv.net/img/soundcross/42626136.jpg", None),
         ("http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", None),
@@ -218,15 +197,16 @@ class PixivWorkExtractor(PixivExtractor):
         return (self.work,)
 
     def get_metadata(self, user=None):
-        self.work = self.api.work(self.illust_id)[0]
+        self.work = self.api.illust_detail(self.illust_id)
         return PixivExtractor.get_metadata(self, self.work["user"])
 
 
 class PixivFavoriteExtractor(PixivExtractor):
     """Extractor for all favorites/bookmarks of a pixiv-user"""
     subcategory = "favorite"
-    directory_fmt = ["{category}", "bookmarks", "{user[id]} {user[account]}"]
-    archive_fmt = "f_{bookmark[id]}{num}.{extension}"
+    directory_fmt = ["{category}", "bookmarks",
+                     "{user_bookmark[id]} {user_bookmark[account]}"]
+    archive_fmt = "f_{user_bookmark[id]}_{id}{num}.{extension}"
     pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
                r"/bookmark\.php\?id=(\d+)"]
     test = [
@@ -239,16 +219,14 @@ class PixivFavoriteExtractor(PixivExtractor):
     def __init__(self, match):
         PixivExtractor.__init__(self)
         self.user_id = match.group(1)
+        self.user = None
 
     def works(self):
-        return self.api.user_favorite_works(self.user_id)
+        return self.api.user_bookmarks_illust(self.user_id)
 
-    def prepare_work(self, work):
-        work["work"]["bookmark"] = {
-            key: work[key]
-            for key in ("id", "comment", "tags", "publicity")
-        }
-        return PixivExtractor.prepare_work(self, work["work"])
+    def get_metadata(self, user=None):
+        self.user = user or self.api.user_detail(self.user_id)
+        return {"user_bookmark": self.user}
 
 
 class PixivBookmarkExtractor(PixivFavoriteExtractor):
@@ -264,178 +242,76 @@ class PixivBookmarkExtractor(PixivFavoriteExtractor):
         self.api.login()
         user = self.api.user_info
         self.user_id = user["id"]
-        return PixivExtractor.get_metadata(self, user)
+        return PixivFavoriteExtractor.get_metadata(self, user)
 
 
 class PixivRankingExtractor(PixivExtractor):
     """Extractor for pixiv ranking pages"""
     subcategory = "ranking"
     archive_fmt = "r_{ranking[mode]}_{ranking[date]}_{id}{num}.{extension}"
-    directory_fmt = ["{category}", "rankings", "{mode}", "{date}"]
+    directory_fmt = ["{category}", "rankings",
+                     "{ranking[mode]}", "{ranking[date]}"]
     pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
                r"/ranking\.php(?:\?([^#]*))?"]
     test = [
         (("https://www.pixiv.net/ranking.php"
-          "?mode=daily&content=illust&date=20170818"), None),
+          "?mode=daily&date=20170818"), None),
         ("https://www.pixiv.net/ranking.php", None),
         ("https://touch.pixiv.net/ranking.php", None),
     ]
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
-        self.ranking_info = None
-        self._iter = None
-        self._first = None
+
+        modes = {
+            "daily": "day",
+            "daily_r18": "day_r18",
+            "weekly": "week",
+            "weekly_r18": "week_r18",
+            "monthly": "month",
+            "male": "day_male",
+            "male_r18": "day_male_r18",
+            "female": "day_female",
+            "female_r18": "day_female_r18",
+            "original": "week_original",
+            "rookie": "week_rookie",
+            "r18g": "week_r18g",
+        }
 
         query = text.parse_query(match.group(1))
-        self.mode = query.get("mode", "daily")
-        self.content = query.get("content", "all")
-        self.date = query.get("date")
 
-        if self.date:
-            if len(self.date) == 8 and self.date.isdecimal():
-                self.date = (self.date[0:4] + "-" +
-                             self.date[4:6] + "-" +
-                             self.date[6:8])
+        mode = query.get("mode", "daily").lower()
+        if mode not in modes:
+            self.log.warning("invalid mode '%s'", mode)
+            mode = "daily"
+        self.mode = modes[mode]
+
+        date = query.get("date")
+        if date:
+            if len(date) == 8 and date.isdecimal():
+                date = "{}-{}-{}".format(date[0:4], date[4:6], date[6:8])
             else:
-                self.log.warning("invalid date '%s'", self.date)
-                self.date = None
+                self.log.warning("invalid date '%s'", date)
+                date = None
+        if not date:
+            date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%d")
+        self.date = date
 
-        if self.content not in ("all", "illust", "manga", "ugoira"):
-            self.log.warning("unrecognized content value '%s' - "
-                             "falling back to 'all'", self.content)
-            self.content = "all"
+        self.ranking_info = {"mode": mode, "date": self.date}
 
     def works(self):
-        yield from self._first["works"]
-        for page in self._iter:
-            yield from page["works"]
+        return self.api.illust_ranking(self.mode, self.date)
 
     def get_metadata(self, user=None):
-        self._iter = self.api.ranking(self.mode, self.content, self.date)
-        self._first = next(self._iter)
-        self.ranking_info = {
-            key: self._first[key]
-            for key in ("mode", "content", "date")
-        }
-        return self.ranking_info.copy()
-
-    def prepare_work(self, work):
-        work["work"]["rank"] = work["rank"]
-        work["work"]["ranking"] = self.ranking_info
-        return PixivExtractor.prepare_work(self, work["work"])
-
-
-class PixivAPI():
-    """Minimal interface for the Pixiv Public-API for mobile devices
-
-    For a better and more complete implementation, see
-    - https://github.com/upbit/pixivpy
-    For in-depth information regarding the Pixiv Public-API, see
-    - http://blog.imaou.com/opensource/2014/10/09/pixiv_api_for_ios_update.html
-    - https://gist.github.com/ZipFile/e14ff1a7e6d01456188a
-    """
-    def __init__(self, extractor):
-        self.session = extractor.session
-        self.log = extractor.log
-        self.username, self.password = extractor._get_auth_info()
-        self.user_info = None
-        self.session.headers.update({
-            "Referer": "https://www.pixiv.net/",
-            'App-OS': 'ios',
-            'App-OS-Version': '10.3.1',
-            'App-Version': '6.7.1',
-            'User-Agent': 'PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)',
-        })
-
-    def user(self, user_id):
-        """Query information about a pixiv user"""
-        endpoint = "users/" + user_id
-        return self._call(endpoint, {})["response"]
-
-    def work(self, illust_id):
-        """Query information about a single pixiv work/illustration"""
-        endpoint = "works/" + illust_id
-        params = {"image_sizes": "large"}
-        return self._call(endpoint, params)["response"]
-
-    def user_works(self, user_id):
-        """Query information about the works of a pixiv user"""
-        endpoint = "users/{user}/works".format(user=user_id)
-        params = {"image_sizes": "large"}
-        return self._pagination(endpoint, params)
-
-    def user_favorite_works(self, user_id):
-        """Query information about the favorite works of a pixiv user"""
-        endpoint = "users/{user}/favorite_works".format(user=user_id)
-        params = {"image_sizes": "large", "include_stats": False}
-        return self._pagination(endpoint, params)
-
-    def ranking(self, mode, content="all", date=None):
-        """Query pixiv's ranking lists"""
-        endpoint = "ranking/" + content
-        params = {"image_sizes": "large", "mode": mode, "date": date}
-        return self._pagination(endpoint, params)
-
-    def login(self):
-        """Login and gain a Pixiv Public-API access token"""
-        self.user_info, access_token = self._login_impl(
-            self.username, self.password)
-        self.session.headers["Authorization"] = access_token
-
-    @cache(maxage=50*60, keyarg=1)
-    def _login_impl(self, username, password):
-        """Actual login implementation"""
-        self.log.info("Logging in as %s", username)
-        data = {
-            "username": username,
-            "password": password,
-            "grant_type": "password",
-            "client_id": "bYGKuGVw91e0NMfPGp44euvGt59s",
-            "client_secret": "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK",
-            "get_secure_url": 1,
-        }
-        response = self.session.post(
-            "https://oauth.secure.pixiv.net/auth/token", data=data
-        )
-        if response.status_code != 200:
-            raise exception.AuthenticationError()
-        try:
-            response = response.json()["response"]
-            token = response["access_token"]
-            user = response["user"]
-        except KeyError:
-            raise Exception("Get token error! Response: %s" % (response))
-        return user, "Bearer " + token
-
-    def _call(self, endpoint, params, _empty=[None]):
-        url = "https://public-api.secure.pixiv.net/v1/" + endpoint + ".json"
-
-        self.login()
-        data = self.session.get(url, params=params).json()
-
-        status = data.get("status")
-        response = data.get("response", _empty)
-        if status == "failure" or response == _empty:
-            raise exception.NotFoundError()
-        return data
-
-    def _pagination(self, endpoint, params):
-        while True:
-            data = self._call(endpoint, params)
-            yield from data["response"]
-
-            pinfo = data["pagination"]
-            if pinfo["current"] == pinfo["pages"]:
-                return
-            params["page"] = pinfo["next"]
+        return {"ranking": self.ranking_info}
 
 
 class PixivAppAPI():
-    """Minimal interface for the Pixiv App-API for mobile devices
+    """Minimal interface for the Pixiv App API for mobile devices
 
-    For a more complete implementation, see
+    For a more complete implementation or documentation, see
     - https://github.com/upbit/pixivpy
+    - https://gist.github.com/ZipFile/3ba99b47162c23f8aea5d5942bb557b1
     """
     CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
     CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
@@ -456,37 +332,17 @@ class PixivAppAPI():
             "App-OS-Version": "10.3.1",
             "App-Version": "6.7.1",
             "User-Agent": "PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)",
+            "Referer": "https://app-api.pixiv.net/",
         })
 
-    def illust_detail(self, illust_id):
-        params = {"illust_id": illust_id}
-        return self._call("v1/illust/detail", params)
-
-    def illust_ranking(self, mode="day", date=None):
-        params = {"mode": mode, "date": date}
-        return self._pagination("v1/illust/ranking", params)
-
-    def user_detail(self, user_id):
-        params = {"user_id": user_id}
-        return self._call("v1/user/detail", params)
-
-    def user_illusts(self, user_id, illust_type=None):
-        params = {"user_id": user_id, "type": illust_type}
-        return self._pagination("v1/user/illusts", params)
-
-    def ugoira_metadata(self, illust_id):
-        params = {"illust_id": illust_id}
-        return self._call("v1/ugoira/metadata", params)
-
-    def authenticate(self):
-        """Authenticate the application by requesting an access token"""
-        self.user_info, auth = self._authenticate_impl(
+    def login(self):
+        """Login and gain an access token"""
+        self.user_info, auth = self._login_impl(
             self.username, self.password)
         self.session.headers["Authorization"] = auth
 
     @cache(maxage=3590, keyarg=1)
-    def _authenticate_impl(self, username, password):
-        """Actual authenticate implementation"""
+    def _login_impl(self, username, password):
         self.log.info("Logging in as %s", username)
 
         url = "https://oauth.secure.pixiv.net/auth/token"
@@ -506,10 +362,34 @@ class PixivAppAPI():
         data = response.json()["response"]
         return data["user"], "Bearer " + data["access_token"]
 
+    def illust_detail(self, illust_id):
+        params = {"illust_id": illust_id}
+        return self._call("v1/illust/detail", params)["illust"]
+
+    def illust_ranking(self, mode="day", date=None):
+        params = {"mode": mode, "date": date}
+        return self._pagination("v1/illust/ranking", params)
+
+    def user_bookmarks_illust(self, user_id, tag=None):
+        params = {"user_id": user_id, "restrict": "public", "tag": tag}
+        return self._pagination("v1/user/bookmarks/illust", params)
+
+    def user_detail(self, user_id):
+        params = {"user_id": user_id}
+        return self._call("v1/user/detail", params)["user"]
+
+    def user_illusts(self, user_id, illust_type=None):
+        params = {"user_id": user_id, "type": illust_type}
+        return self._pagination("v1/user/illusts", params)
+
+    def ugoira_metadata(self, illust_id):
+        params = {"illust_id": illust_id}
+        return self._call("v1/ugoira/metadata", params)["ugoira_metadata"]
+
     def _call(self, endpoint, params=None):
         url = "https://app-api.pixiv.net/" + endpoint
 
-        self.authenticate()
+        self.login()
         response = self.session.get(url, params=params)
 
         if 200 <= response.status_code < 400:

From fe69d010835706ed2e78abfed6c1ec7acbef6ad0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Mon, 14 May 2018 14:46:05 +0200
Subject: [PATCH 17/21] [pixiv] add extractor for search results

---
 gallery_dl/extractor/pixiv.py | 67 +++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index c1f8f011..5f278bae 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -254,8 +254,7 @@ class PixivRankingExtractor(PixivExtractor):
     pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
                r"/ranking\.php(?:\?([^#]*))?"]
     test = [
-        (("https://www.pixiv.net/ranking.php"
-          "?mode=daily&date=20170818"), None),
+        ("https://www.pixiv.net/ranking.php?mode=daily&date=20170818", None),
         ("https://www.pixiv.net/ranking.php", None),
         ("https://touch.pixiv.net/ranking.php", None),
     ]
@@ -306,6 +305,65 @@ class PixivRankingExtractor(PixivExtractor):
         return {"ranking": self.ranking_info}
 
 
+class PixivSearchExtractor(PixivExtractor):
+    """Extractor for pixiv search results"""
+    subcategory = "search"
+    archive_fmt = "s_{search[word]}_{id}{num}.{extension}"
+    directory_fmt = ["{category}", "search", "{search[word]}"]
+    pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
+               r"/search\.php\?([^#]+)"]
+    test = [
+        ("https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", None),
+        ("https://touch.pixiv.net/search.php?word=Original", None),
+    ]
+
+    def __init__(self, match):
+        PixivExtractor.__init__(self)
+
+        query = text.parse_query(match.group(1))
+
+        if "word" in query:
+            self.word = text.unescape(query["word"])
+        else:
+            self.log.error("missing search term")
+            self.word = None
+
+        sort = query.get("order", "date_d")
+        sort_map = {
+            "date": "date_asc",
+            "date_d": "date_desc",
+        }
+        if sort not in sort_map:
+            self.log.warning("invalid sort order '%s'", sort)
+            sort = "date_d"
+        self.sort = sort_map[sort]
+
+        target = query.get("s_mode", "s_tag")
+        target_map = {
+            "s_tag": "partial_match_for_tags",
+            "s_tag_full": "exact_match_for_tags",
+            "s_tc": "title_and_caption",
+        }
+        if target not in target_map:
+            self.log.warning("invalid search target '%s'", target)
+            target = "s_tag"
+        self.target = target_map[target]
+
+        self.search_info = {
+            "word": self.word,
+            "sort": self.sort,
+            "target": self.target,
+        }
+
+    def works(self):
+        if not self.word:
+            return ()
+        return self.api.search_illust(self.word, self.sort, self.target)
+
+    def get_metadata(self, user=None):
+        return {"search": self.search_info}
+
+
 class PixivAppAPI():
     """Minimal interface for the Pixiv App API for mobile devices
 
@@ -370,6 +428,11 @@ class PixivAppAPI():
         params = {"mode": mode, "date": date}
         return self._pagination("v1/illust/ranking", params)
 
+    def search_illust(self, word, sort=None, target=None, duration=None):
+        params = {"word": word, "search_target": target,
+                  "sort": sort, "duration": duration}
+        return self._pagination("v1/search/illust", params)
+
     def user_bookmarks_illust(self, user_id, tag=None):
         params = {"user_id": user_id, "restrict": "public", "tag": tag}
         return self._pagination("v1/user/bookmarks/illust", params)

From 909d105ae6f786cb143fcc056e955c8f0ac787f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Tue, 15 May 2018 13:02:49 +0200
Subject: [PATCH 18/21] [pixiv] add extractor for illusts from followed users

---
 docs/supportedsites.rst       |  4 ++--
 gallery_dl/extractor/pixiv.py | 35 ++++++++++++++++++++++++++++++-----
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst
index da6b774a..cdf5c697 100644
--- a/docs/supportedsites.rst
+++ b/docs/supportedsites.rst
@@ -88,8 +88,8 @@ Turboimagehost       https://turboimagehost.com/         individual Images
 ==================== =================================== ================================================== ================
 
 .. |Images from Use-0| replace:: Images from Users, Albums, Challenges, individual Images, Likes, Search Results
-.. |Collections, De-1| replace:: Collections, Deviations, Favorites, Folders, Galleries, Journals
+.. |Collections, De-1| replace:: Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images
 .. |Images from Use-2| replace:: Images from Users, Albums, Favorites, Galleries, Groups, individual Images, Search Results
 .. |Images from Use-3| replace:: Images from Users, Doujin, Favorites, individual Images
-.. |Images from Use-4| replace:: Images from Users, Bookmarks, Favorites, pixiv.me Links, Rankings, Individual Images
+.. |Images from Use-4| replace:: Images from Users, Bookmarks, Favorites, Follows, pixiv.me Links, Rankings, Search Results, Individual Images
 .. |Albums, individ-5| replace:: Albums, individual Images, Images from Users and Folders
diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index 5f278bae..7ef5b5b7 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -240,9 +240,7 @@ class PixivBookmarkExtractor(PixivFavoriteExtractor):
 
     def get_metadata(self, user=None):
         self.api.login()
-        user = self.api.user_info
-        self.user_id = user["id"]
-        return PixivFavoriteExtractor.get_metadata(self, user)
+        return PixivFavoriteExtractor.get_metadata(self, self.api.user)
 
 
 class PixivRankingExtractor(PixivExtractor):
@@ -364,6 +362,29 @@ class PixivSearchExtractor(PixivExtractor):
         return {"search": self.search_info}
 
 
+class PixivFollowExtractor(PixivExtractor):
+    """Extractor for new illustrations from your followed artists"""
+    subcategory = "follow"
+    archive_fmt = "F_{user_follow[id]}_{id}{num}.{extension}"
+    directory_fmt = ["{category}", "following"]
+    pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
+               r"/bookmark_new_illust\.php"]
+    test = [
+        ("https://www.pixiv.net/bookmark_new_illust.php", None),
+        ("https://touch.pixiv.net/bookmark_new_illust.php", None),
+    ]
+
+    def __init__(self, match):
+        PixivExtractor.__init__(self)
+
+    def works(self):
+        return self.api.illust_follow()
+
+    def get_metadata(self, user=None):
+        self.api.login()
+        return {"user_follow": self.api.user}
+
+
 class PixivAppAPI():
     """Minimal interface for the Pixiv App API for mobile devices
 
@@ -378,7 +399,7 @@ class PixivAppAPI():
         self.session = extractor.session
         self.log = extractor.log
         self.username, self.password = extractor._get_auth_info()
-        self.user_info = None
+        self.user = None
 
         self.client_id = extractor.config(
             "client-id", self.CLIENT_ID)
@@ -395,7 +416,7 @@ class PixivAppAPI():
 
     def login(self):
         """Login and gain an access token"""
-        self.user_info, auth = self._login_impl(
+        self.user, auth = self._login_impl(
             self.username, self.password)
         self.session.headers["Authorization"] = auth
 
@@ -424,6 +445,10 @@ class PixivAppAPI():
         params = {"illust_id": illust_id}
         return self._call("v1/illust/detail", params)["illust"]
 
+    def illust_follow(self, restrict="all"):
+        params = {"restrict": restrict}
+        return self._pagination("v2/illust/follow", params)
+
     def illust_ranking(self, mode="day", date=None):
         params = {"mode": mode, "date": date}
         return self._pagination("v1/illust/ranking", params)

From b8e53b8c6b3fca38fc25397ab3e009e8cbc1ff75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Tue, 15 May 2018 13:28:08 +0200
Subject: [PATCH 19/21] [pixiv] move query parsing out of constructor

better exception handling, among other things
---
 gallery_dl/extractor/pixiv.py | 64 +++++++++++++++++------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index 7ef5b5b7..07ec107e 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -112,17 +112,17 @@ class PixivUserExtractor(PixivExtractor):
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
-        self.user_id, tag = match.groups()
-        if tag:
-            self.tag = text.unquote(tag).lower()
-            self.works = self._tagged_works
+        self.user_id, self.tag = match.groups()
 
     def works(self):
+        if self.tag:
+            return self._tagged_works()
         return self.api.user_illusts(self.user_id)
 
     def _tagged_works(self):
+        tag = text.unquote(self.tag).lower()
         for work in self.api.user_illusts(self.user_id):
-            if self.tag in [tag["name"].lower() for tag in work["tags"]]:
+            if tag in [tag["name"].lower() for tag in work["tags"]]:
                 yield work
 
 
@@ -259,8 +259,17 @@ class PixivRankingExtractor(PixivExtractor):
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
+        self.query = match.group(1)
+        self.mode = self.date = None
 
-        modes = {
+    def works(self):
+        return self.api.illust_ranking(self.mode, self.date)
+
+    def get_metadata(self, user=None):
+        query = text.parse_query(self.query)
+
+        mode = query.get("mode", "daily").lower()
+        mode_map = {
             "daily": "day",
             "daily_r18": "day_r18",
             "weekly": "week",
@@ -274,14 +283,10 @@ class PixivRankingExtractor(PixivExtractor):
             "rookie": "week_rookie",
             "r18g": "week_r18g",
         }
-
-        query = text.parse_query(match.group(1))
-
-        mode = query.get("mode", "daily").lower()
-        if mode not in modes:
+        if mode not in mode_map:
             self.log.warning("invalid mode '%s'", mode)
             mode = "daily"
-        self.mode = modes[mode]
+        self.mode = mode_map[mode]
 
         date = query.get("date")
         if date:
@@ -294,13 +299,10 @@ class PixivRankingExtractor(PixivExtractor):
             date = (datetime.utcnow() - timedelta(days=1)).strftime("%Y-%m-%d")
         self.date = date
 
-        self.ranking_info = {"mode": mode, "date": self.date}
-
-    def works(self):
-        return self.api.illust_ranking(self.mode, self.date)
-
-    def get_metadata(self, user=None):
-        return {"ranking": self.ranking_info}
+        return {"ranking": {
+            "mode": mode,
+            "date": self.date,
+        }}
 
 
 class PixivSearchExtractor(PixivExtractor):
@@ -317,14 +319,20 @@ class PixivSearchExtractor(PixivExtractor):
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
+        self.query = match.group(1)
+        self.word = self.sort = self.target = None
 
-        query = text.parse_query(match.group(1))
+    def works(self):
+        return self.api.search_illust(self.word, self.sort, self.target)
+
+    def get_metadata(self, user=None):
+        query = text.parse_query(self.query)
 
         if "word" in query:
             self.word = text.unescape(query["word"])
         else:
             self.log.error("missing search term")
-            self.word = None
+            raise exception.StopExtraction()
 
         sort = query.get("order", "date_d")
         sort_map = {
@@ -347,19 +355,11 @@ class PixivSearchExtractor(PixivExtractor):
             target = "s_tag"
         self.target = target_map[target]
 
-        self.search_info = {
+        return {"search": {
             "word": self.word,
             "sort": self.sort,
             "target": self.target,
-        }
-
-    def works(self):
-        if not self.word:
-            return ()
-        return self.api.search_illust(self.word, self.sort, self.target)
-
-    def get_metadata(self, user=None):
-        return {"search": self.search_info}
+        }}
 
 
 class PixivFollowExtractor(PixivExtractor):
@@ -374,7 +374,7 @@ class PixivFollowExtractor(PixivExtractor):
         ("https://touch.pixiv.net/bookmark_new_illust.php", None),
     ]
 
-    def __init__(self, match):
+    def __init__(self, _):
         PixivExtractor.__init__(self)
 
     def works(self):

From 0a1863fce3d62f8fdf107bdc11579d63e8f185bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 18 May 2018 15:30:06 +0200
Subject: [PATCH 20/21] [pixiv] respect more query parameters for user URLs

The API endpoint responsible for user illustrations does not
provide sufficient filter capabilities* to match the actual
website, so we are spinning our own filters.

Respected parameters are
    'type': illust, manga, ugoira
    'tag' : any image tag (this was already supported)
    'p'   : the page to start on

*
- API can filter for illustrations and manga, but not for ugoira.
- 'offset' is applied before filtering
- no 'tag' filter
---
 gallery_dl/extractor/pixiv.py | 44 ++++++++++++++++++++++++-----------
 test/test_results.py          |  1 -
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index 07ec107e..779dbd31 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -9,7 +9,7 @@
 """Extract images and ugoira from https://www.pixiv.net/"""
 
 from .common import Extractor, Message
-from .. import text, exception
+from .. import text, util, exception
 from ..cache import cache
 from datetime import datetime, timedelta
 
@@ -89,7 +89,7 @@ class PixivUserExtractor(PixivExtractor):
     """Extractor for works of a pixiv-user"""
     subcategory = "user"
     pattern = [(r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
-                r"/member(?:_illust)?\.php\?id=(\d+)(?:.*&tag=([^&#]+))?"),
+                r"/member(?:_illust)?\.php\?id=(\d+)(?:&([^#]+))?"),
                (r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
                 r"/(?:u(?:ser)?/|(?:mypage\.php)?#id=)(\d+)()")]
     test = [
@@ -100,6 +100,10 @@ class PixivUserExtractor(PixivExtractor):
           "&tag=%E6%89%8B%E3%81%B6%E3%82%8D"), {
             "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658",
         }),
+        (("https://www.pixiv.net/member_illust.php?id=3137110"
+          "&tag=%E3%83%96%E3%82%A4%E3%82%BA&type=illust&p=2"), {
+            "count": ">= 55",
+        }),
         ("http://www.pixiv.net/member_illust.php?id=173531", {
             "exception": exception.NotFoundError,
         }),
@@ -112,18 +116,32 @@ class PixivUserExtractor(PixivExtractor):
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
-        self.user_id, self.tag = match.groups()
+        self.user_id, self.query = match.groups()
 
     def works(self):
-        if self.tag:
-            return self._tagged_works()
-        return self.api.user_illusts(self.user_id)
+        works = self.api.user_illusts(self.user_id)
 
-    def _tagged_works(self):
-        tag = text.unquote(self.tag).lower()
-        for work in self.api.user_illusts(self.user_id):
-            if tag in [tag["name"].lower() for tag in work["tags"]]:
-                yield work
+        if self.query:
+            qdict = text.parse_query(self.query)
+            if "type" in qdict:
+                type_ = qdict["type"].lower()
+                works = filter(self._is_type(type_), works)
+            if "tag" in qdict:
+                tag = text.unquote(qdict["tag"]).lower()
+                works = filter(self._has_tag(tag), works)
+            if "p" in qdict:  # apply page-offset last
+                offset = (text.parse_int(qdict["p"], 1) - 1) * 20
+                works = util.advance(works, offset)
+
+        return works
+
+    @staticmethod
+    def _has_tag(tag):
+        return lambda work: tag in [t["name"].lower() for t in work["tags"]]
+
+    @staticmethod
+    def _is_type(type_):
+        return lambda work: work["type"] == type_
 
 
 class PixivMeExtractor(PixivExtractor):
@@ -466,8 +484,8 @@ class PixivAppAPI():
         params = {"user_id": user_id}
         return self._call("v1/user/detail", params)["user"]
 
-    def user_illusts(self, user_id, illust_type=None):
-        params = {"user_id": user_id, "type": illust_type}
+    def user_illusts(self, user_id):
+        params = {"user_id": user_id}
         return self._pagination("v1/user/illusts", params)
 
     def ugoira_metadata(self, illust_id):
diff --git a/test/test_results.py b/test/test_results.py
index 90fc1b96..0a604f08 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -22,7 +22,6 @@ TRAVIS_SKIP = {
 
 # temporary issues, etc.
 BROKEN = {
-    "pixiv",  # /users/<id>/favorite_works API endpoint is gone
 }
 
 

From 1ff626db975ff84156b690c19c4b283d7ef17fbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 18 May 2018 16:55:24 +0200
Subject: [PATCH 21/21] [pixiv] improve bookmark extraction

- combine 'favorite' and 'bookmark' extractors
  - it is now one extractor class, but its subcategory still
    distinguishes between your own bookmarks ('bookmark') and other
    user's bookmarks ('favorite') like before
- allow filtering by bookmark tags and public/private bookmarks
- fix pagination for bookmark results
---
 gallery_dl/extractor/pixiv.py | 63 +++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 22 deletions(-)

diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index 779dbd31..bd481e62 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -96,10 +96,12 @@ class PixivUserExtractor(PixivExtractor):
         ("http://www.pixiv.net/member_illust.php?id=173530", {
             "url": "852c31ad83b6840bacbce824d85f2a997889efb7",
         }),
+        # illusts with specific tag
         (("https://www.pixiv.net/member_illust.php?id=173530"
           "&tag=%E6%89%8B%E3%81%B6%E3%82%8D"), {
             "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658",
         }),
+        # all sorts of query parameters
         (("https://www.pixiv.net/member_illust.php?id=3137110"
           "&tag=%E3%83%96%E3%82%A4%E3%82%BA&type=illust&p=2"), {
             "count": ">= 55",
@@ -226,39 +228,55 @@ class PixivFavoriteExtractor(PixivExtractor):
                      "{user_bookmark[id]} {user_bookmark[account]}"]
     archive_fmt = "f_{user_bookmark[id]}_{id}{num}.{extension}"
     pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net"
-               r"/bookmark\.php\?id=(\d+)"]
+               r"/bookmark\.php(?:\?([^#]*))?"]
     test = [
         ("https://www.pixiv.net/bookmark.php?id=173530", {
             "url": "e717eb511500f2fa3497aaee796a468ecf685cc4",
         }),
+        # bookmarks with specific tag
+        (("https://www.pixiv.net/bookmark.php?id=3137110"
+          "&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1"), {
+            "count": 2,
+        }),
+        # own bookmarks
+        ("https://www.pixiv.net/bookmark.php", {
+            "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba",
+        }),
+        # touch URLs
         ("https://touch.pixiv.net/bookmark.php?id=173530", None),
+        ("https://touch.pixiv.net/bookmark.php", None),
     ]
 
     def __init__(self, match):
         PixivExtractor.__init__(self)
-        self.user_id = match.group(1)
-        self.user = None
+        self.query = text.parse_query(match.group(1))
+        if "id" not in self.query:
+            self.subcategory = "bookmark"
 
     def works(self):
-        return self.api.user_bookmarks_illust(self.user_id)
+        tag = None
+        restrict = "public"
+        offset = 0
+
+        if "tag" in self.query:
+            tag = text.unquote(self.query["tag"])
+        if "rest" in self.query and self.query["rest"] == "hide":
+            restrict = "private"
+        if "p" in self.query:
+            offset = (text.parse_int(self.query["p"], 1) - 1) * 20
+
+        works = self.api.user_bookmarks_illust(self.user_id, tag, restrict)
+        return util.advance(works, offset)
 
     def get_metadata(self, user=None):
-        self.user = user or self.api.user_detail(self.user_id)
-        return {"user_bookmark": self.user}
+        if "id" in self.query:
+            user = self.api.user_detail(self.query["id"])
+        else:
+            self.api.login()
+            user = self.api.user
 
-
-class PixivBookmarkExtractor(PixivFavoriteExtractor):
-    """Extractor for all favorites/bookmarks of your own account"""
-    subcategory = "bookmark"
-    pattern = [r"(?:https?://)?(?:www\.|touch\.)?pixiv\.net/bookmark\.php()$"]
-    test = [
-        ("https://www.pixiv.net/bookmark.php", None),
-        ("https://touch.pixiv.net/bookmark.php", None),
-    ]
-
-    def get_metadata(self, user=None):
-        self.api.login()
-        return PixivFavoriteExtractor.get_metadata(self, self.api.user)
+        self.user_id = user["id"]
+        return {"user_bookmark": user}
 
 
 class PixivRankingExtractor(PixivExtractor):
@@ -476,8 +494,8 @@ class PixivAppAPI():
                   "sort": sort, "duration": duration}
         return self._pagination("v1/search/illust", params)
 
-    def user_bookmarks_illust(self, user_id, tag=None):
-        params = {"user_id": user_id, "restrict": "public", "tag": tag}
+    def user_bookmarks_illust(self, user_id, tag=None, restrict="public"):
+        params = {"user_id": user_id, "tag": tag, "restrict": restrict}
         return self._pagination("v1/user/bookmarks/illust", params)
 
     def user_detail(self, user_id):
@@ -512,4 +530,5 @@ class PixivAppAPI():
 
             if not data["next_url"]:
                 return
-            params["offset"] = data["next_url"].rpartition("=")[2]
+            query = data["next_url"].rpartition("?")[2]
+            params = text.parse_query(query)