replace AsynchronousMixin Extractor with a Mixin

gallery_dl/extractor/common.py

@@ -175,38 +175,6 @@ class Extractor():
                 setcookie(cookie)
 
 
-class AsynchronousExtractor(Extractor):
-
-    def __init__(self):
-        Extractor.__init__(self)
-        queue_size = int(config.get(("queue-size",), 5))
-        self.__queue = queue.Queue(queue_size)
-        self.__thread = threading.Thread(target=self.async_items, daemon=True)
-
-    def __iter__(self):
-        get = self.__queue.get
-        done = self.__queue.task_done
-
-        self.__thread.start()
-        while True:
-            task = get()
-            if task is None:
-                return
-            if isinstance(task, Exception):
-                raise task
-            yield task
-            done()
-
-    def async_items(self):
-        put = self.__queue.put
-        try:
-            for task in self.items():
-                put(task)
-        except Exception as exc:
-            put(exc)
-        put(None)
-
-
 class ChapterExtractor(Extractor):
 
     subcategory = "chapter"
@@ -287,6 +255,38 @@ class MangaExtractor(Extractor):
         """Return a list of all (chapter-url, metadata)-tuples"""
 
 
+class AsynchronousMixin():
+    """Run info extraction in a separate thread"""
+
+    def __iter__(self):
+        messages = queue.Queue(5)
+        thread = threading.Thread(
+            target=self.async_items,
+            args=(messages,),
+            daemon=True,
+        )
+
+        thread.start()
+        while True:
+            msg = messages.get()
+            if msg is None:
+                thread.join()
+                return
+            if isinstance(msg, Exception):
+                thread.join()
+                raise msg
+            yield msg
+            messages.task_done()
+
+    def async_items(self, messages):
+        try:
+            for msg in self.items():
+                messages.put(msg)
+        except Exception as exc:
+            messages.put(exc)
+        messages.put(None)
+
+
 class SharedConfigMixin():
     """Enable sharing of config settings based on 'basecategory'"""
     basecategory = ""

gallery_dl/extractor/imgbox.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014-2018 Mike Fährmann
+# Copyright 2014-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -8,7 +8,7 @@
 
 """Extract images from galleries at https://imgbox.com/"""
 
-from .common import Extractor, AsynchronousExtractor, Message
+from .common import Extractor, Message, AsynchronousMixin
 from .. import text, exception
 import re
 
@@ -58,7 +58,7 @@ class ImgboxExtractor(Extractor):
         return text.extract(page, '<a href="', '"', pos)[0]
 
 
-class ImgboxGalleryExtractor(AsynchronousExtractor, ImgboxExtractor):
+class ImgboxGalleryExtractor(AsynchronousMixin, ImgboxExtractor):
     """Extractor for image galleries from imgbox.com"""
     subcategory = "gallery"
     directory_fmt = ["{category}", "{title} - {gallery_key}"]
@@ -81,7 +81,7 @@ class ImgboxGalleryExtractor(AsynchronousExtractor, ImgboxExtractor):
     ]
 
     def __init__(self, match):
-        AsynchronousExtractor.__init__(self)
+        ImgboxExtractor.__init__(self)
         self.gallery_key = match.group(1)
         self.image_keys = []
 

gallery_dl/extractor/khinsider.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2016-2018 Mike Fährmann
+# Copyright 2016-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -8,11 +8,11 @@
 
 """Extract soundtracks from https://downloads.khinsider.com/"""
 
-from .common import AsynchronousExtractor, Message
+from .common import Extractor, Message, AsynchronousMixin
 from .. import text, exception
 
 
-class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
+class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor):
     """Extractor for soundtracks from khinsider.com"""
     category = "khinsider"
     subcategory = "soundtrack"
@@ -30,7 +30,7 @@ class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
     root = "https://downloads.khinsider.com"
 
     def __init__(self, match):
-        AsynchronousExtractor.__init__(self)
+        Extractor.__init__(self)
         self.album = match.group(1)
 
     def items(self):

gallery_dl/extractor/luscious.py

@@ -8,7 +8,7 @@
 
 """Extractors for https://luscious.net/"""
 
-from .common import Extractor, Message
+from .common import Extractor, Message, AsynchronousMixin
 from .. import text, util, exception
 from ..cache import cache
 
@@ -46,7 +46,7 @@ class LusciousExtractor(Extractor):
         raise exception.AuthenticationError()
 
 
-class LusciousAlbumExtractor(LusciousExtractor):
+class LusciousAlbumExtractor(AsynchronousMixin, LusciousExtractor):
     """Extractor for image albums from luscious.net"""
     subcategory = "album"
     directory_fmt = ["{category}", "{gallery_id} {title}"]

gallery_dl/extractor/nijie.py

@@ -8,12 +8,12 @@
 
 """Extract images from https://nijie.info/"""
 
-from .common import AsynchronousExtractor, Message
+from .common import Extractor, Message, AsynchronousMixin
 from .. import text, exception
 from ..cache import cache
 
 
-class NijieExtractor(AsynchronousExtractor):
+class NijieExtractor(AsynchronousMixin, Extractor):
     """Base class for nijie extractors"""
     category = "nijie"
     directory_fmt = ["{category}", "{user_id}"]
@@ -26,7 +26,7 @@ class NijieExtractor(AsynchronousExtractor):
     popup_url = "https://nijie.info/view_popup.php?id="
 
     def __init__(self, match=None):
-        AsynchronousExtractor.__init__(self)
+        Extractor.__init__(self)
         self.session.headers["Referer"] = self.root + "/"
         self.user_id = match.group(1) if match else None