add extractor '3dbooru'

gallery_dl/extractor/3dbooru.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extract image-urls from http://behoimi.org/"""
+
+from .booru import JSONBooruExtractor
+
+info = {
+    "category": "3dbooru",
+    "extractor": "ThreeDeeBooruExtractor",
+    "directory": ["{category}", "{tags}"],
+    "filename": "{category}_{id}_{name}",
+    "pattern": [
+        r"(?:https?://)?(?:www\.)?http://behoimi\.org/post(?:/(?:index)?)?\?tags=([^&]+).*",
+    ],
+}
+
+class ThreeDeeBooruExtractor(JSONBooruExtractor):
+
+    def __init__(self, match, config):
+        JSONBooruExtractor.__init__(self, match, config, info)
+        self.api_url = "http://behoimi.org/post/index.json"
+        self.headers = {
+            "Referer": "http://behoimi.org/post/show/",
+            "User-Agent": "Mozilla/5.0"
+        }

gallery_dl/extractor/booru.py

@@ -27,10 +27,12 @@ class BooruExtractor(SequentialExtractor):
         self.tags = urllib.parse.unquote(match.group(1))
         self.page = "page"
         self.params = {"tags": self.tags}
+        self.headers = {}
 
     def items(self):
         yield Message.Version, 1
         yield Message.Directory, self.get_job_metadata()
+        yield Message.Headers, self.headers
         for data in self.items_impl():
             yield Message.Url, self.get_file_url(data), self.get_file_metadata(data)
 
@@ -76,7 +78,8 @@ class JSONBooruExtractor(BooruExtractor):
         self.update_page(reset=True)
         while True:
             images = json.loads(
-                self.request(self.api_url, verify=True, params=self.params).text
+                self.request(self.api_url, verify=True, params=self.params,
+                             headers=self.headers).text
             )
             if len(images) == 0:
                 return