[idolcomplex] add support for idol.sankakucomplex.com

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
 ## Unreleased
+- Added support for:
+  - `puremashiro` - http://reader.puremashiro.moe/ ([#66](https://github.com/mikf/gallery-dl/issues/66))
+  - `idolcomplex` - https://idol.sankakucomplex.com/
 - Added an option to filter reblogs on `tumblr` ([#61](https://github.com/mikf/gallery-dl/issues/61))
 - Improved pagination for various …booru sites to work around page limits
 - Fixed chapter information parsing for certain manga on `kissmanga` ([#58](https://github.com/mikf/gallery-dl/issues/58)) and `batoto` ([#60](https://github.com/mikf/gallery-dl/issues/60))

docs/supportedsites.rst

@@ -30,6 +30,7 @@ Hentai Foundry       https://www.hentai-foundry.com/     Images from Users, indi
 Hentai2Read          https://hentai2read.com/            Chapters, Manga
 HentaiHere           https://hentaihere.com/             Chapters, Manga
 Hitomi.la            https://hitomi.la/                  Galleries
+Idol Complex         https://idol.sankakucomplex.com/    Pools, Posts, Tag-Searches                         Optional
 ImageBam             http://www.imagebam.com/            Galleries, individual Images
 ImageFap             http://imagefap.com/                Images from Users, Galleries, individual Images
 imgbox               https://imgbox.com/                 Galleries, individual Images

gallery_dl/extractor/__init__.py

@@ -37,6 +37,7 @@ modules = [
     "hentaifoundry",
     "hentaihere",
     "hitomi",
+    "idolcomplex",
     "imagebam",
     "imagefap",
     "imgbox",

gallery_dl/extractor/deviantart.py

@@ -299,7 +299,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor):
     test = [(("https://pencilshadings.deviantart.com"
               "/favourites/70595441/3D-Favorites"), {
         "url": "742f92199d5bc6a89cda6ec6133d46c7a523824d",
-        "keyword": "22aa30e064cf8cd8b9ca7944f4877f5ac9793cfa",
+        "keyword": "9210c976b5274eff6ea1d2b8a4f891c9f35ce340",
         "options": (("original", False),),
     })]
 

gallery_dl/extractor/idolcomplex.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extract images from https://idol.sankakucomplex.com/"""
+
+from . import sankaku
+
+
+class IdolcomplexExtractor(sankaku.SankakuExtractor):
+    """Base class for idolcomplex extractors"""
+    category = "idolcomplex"
+    subdomain = "idol"
+
+
+class IdolcomplexTagExtractor(IdolcomplexExtractor,
+                              sankaku.SankakuTagExtractor):
+    """Extractor for images from idol.sankakucomplex.com by search-tags"""
+    pattern = [r"(?:https?://)?idol\.sankakucomplex\.com"
+               r"/\?(?:[^&#]*&)*tags=([^&#]+)"]
+    test = [("https://idol.sankakucomplex.com/?tags=lyumos+wreath", {
+        "count": ">= 6",
+        "pattern": (r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}"
+                    r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+"),
+    })]
+
+
+class IdolcomplexPoolExtractor(IdolcomplexExtractor,
+                               sankaku.SankakuPoolExtractor):
+    """Extractor for image-pools from idol.sankakucomplex.com"""
+    pattern = [r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)"]
+    test = [("https://idol.sankakucomplex.com/pool/show/145", {
+        "count": 3,
+    })]
+
+
+class IdolcomplexPostExtractor(IdolcomplexExtractor,
+                               sankaku.SankakuPostExtractor):
+    """Extractor for single images from idol.sankakucomplex.com"""
+    pattern = [r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)"]
+    test = [("https://idol.sankakucomplex.com/post/show/694215", {
+        "content": "694ec2491240787d75bf5d0c75d0082b53a85afd",
+        "count": 1,
+    })]

gallery_dl/extractor/sankaku.py

@@ -20,12 +20,14 @@ class SankakuExtractor(SharedConfigExtractor):
     basecategory = "booru"
     category = "sankaku"
     filename_fmt = "{category}_{id}_{md5}.{extension}"
-    root = "https://chan.sankakucomplex.com"
     cookienames = ("login", "pass_hash")
-    cookiedomain = "chan.sankakucomplex.com"
+    subdomain = "chan"
 
     def __init__(self):
         SharedConfigExtractor.__init__(self)
+        self.cookiedomain = self.subdomain + ".sankakucomplex.com"
+        self.root = "https://" + self.cookiedomain
+
         self.logged_in = True
         self.start_page = 1
         self.start_post = 0
@@ -61,7 +63,7 @@ class SankakuExtractor(SharedConfigExtractor):
         url = self.root + "/post/show/" + post_id
         page = self.request(url, retries=10).text
 
-        tags   , pos = extr(page, "<title>", " | Sankaku Channel</title>")
+        tags   , pos = extr(page, "<title>", " | ")
         vavg   , pos = extr(page, "itemprop=ratingValue>", "<", pos)
         vcnt   , pos = extr(page, "itemprop=reviewCount>", "<", pos)
         _      , pos = extr(page, "Posted: <", "", pos)
@@ -100,7 +102,7 @@ class SankakuExtractor(SharedConfigExtractor):
             return
         username, password = self._get_auth_info()
         if username:
-            cookies = self._login_impl(username, password)
+            cookies = self._login_impl((username, self.subdomain), password)
             for key, value in cookies.items():
                 self.session.cookies.set(
                     key, value, domain=self.cookiedomain)
@@ -108,8 +110,9 @@ class SankakuExtractor(SharedConfigExtractor):
             self.logged_in = False
 
     @cache(maxage=90*24*60*60, keyarg=1)
-    def _login_impl(self, username, password):
+    def _login_impl(self, usertuple, password):
         """Actual login implementation"""
+        username = usertuple[0]
         self.log.info("Logging in as %s", username)
         params = {
             "url": "",

scripts/build_supportedsites.py

@@ -25,6 +25,7 @@ CATEGORY_MAP = {
     "hentaifoundry"  : "Hentai Foundry",
     "hentaihere"     : "HentaiHere",
     "hitomi"         : "Hitomi.la",
+    "idolcomplex"    : "Idol Complex",
     "imagebam"       : "ImageBam",
     "imagefap"       : "ImageFap",
     "imgbox"         : "imgbox",
@@ -77,15 +78,16 @@ SUBCATEGORY_MAP = {
 }
 
 AUTH_MAP = {
-    "batoto"    : "Optional",
+    "batoto"     : "Optional",
-    "deviantart": "Optional (OAuth)",
+    "deviantart" : "Optional (OAuth)",
-    "exhentai"  : "Optional",
+    "exhentai"   : "Optional",
-    "flickr"    : "Optional (OAuth)",
+    "flickr"     : "Optional (OAuth)",
-    "nijie"     : "Required",
+    "idolcomplex": "Optional",
-    "pixiv"     : "Required",
+    "nijie"      : "Required",
-    "reddit"    : "Optional (OAuth)",
+    "pixiv"      : "Required",
-    "sankaku"   : "Optional",
+    "reddit"     : "Optional (OAuth)",
-    "seiga"     : "Required",
+    "sankaku"    : "Optional",
+    "seiga"      : "Required",
 }
 
 IGNORE_LIST = (

test/test_extractors.py

@@ -18,7 +18,9 @@ SKIP = {
     "archivedmoe", "archiveofsins", "thebarchive",
 
     # temporary issues
-
+    "batoto",  # R.I.P.
+    "imgyt",  # server maintenance
+    "loveisover",
 }