[imagevenue] detect 404 image files (#7570)

gallery_dl/extractor/imagehosts.py

@@ -23,6 +23,7 @@ class ImagehostImageExtractor(Extractor):
     _params = None
     _cookies = None
     _encoding = None
+    _validate = None
 
     def __init__(self, match):
         Extractor.__init__(self, match)
@@ -57,6 +58,8 @@ class ImagehostImageExtractor(Extractor):
         data.update(self.metadata(page))
         if self._https and url.startswith("http:"):
             url = "https:" + url[5:]
+        if self._validate is not None:
+            data["_http_validate"] = self._validate
 
         yield Message.Directory, data
         yield Message.Url, url, data
@@ -164,6 +167,14 @@ class ImagevenueImageExtractor(ImagehostImageExtractor):
         filename, pos = text.extract(page, 'alt="', '"', pos)
         return url, text.unescape(filename)
 
+    def _validate(self, response):
+        hget = response.headers.get
+        return not (
+            hget("content-length") == "14396" and
+            hget("content-type") == "image/jpeg" and
+            hget("last-modified") == "Mon, 04 May 2020 07:19:52 GMT"
+        )
+
 
 class ImagetwistImageExtractor(ImagehostImageExtractor):
     """Extractor for single images from imagetwist.com"""

test/results/imagevenue.py

@@ -12,15 +12,20 @@ __tests__ = (
     "#url"     : "https://www.imagevenue.com/ME13LS07",
     "#category": ("imagehost", "imagevenue", "image"),
     "#class"   : imagehosts.ImagevenueImageExtractor,
-    "#pattern"      : r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png",
+    "#urls"         : "https://cdn-images.imagevenue.com/10/ac/05/ME13LS07_o.png",
     "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36",
     "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee",
+
+    "extension": "png",
+    "filename" : "test-テスト-\"&>",
+    "token"    : "ME13LS07",
 },
 
 {
     "#url"     : "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486",
     "#category": ("imagehost", "imagevenue", "image"),
     "#class"   : imagehosts.ImagevenueImageExtractor,
+    "#urls"    : "https://cdno-data.imagevenue.com/html.img150/upload2328/loc486/92518_13732377annakarina424200712535AM_122_486lo.jpg",
     "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980",
 },
 
@@ -28,7 +33,17 @@ __tests__ = (
     "#url"     : "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg",
     "#category": ("imagehost", "imagevenue", "image"),
     "#class"   : imagehosts.ImagevenueImageExtractor,
+    "#urls"    : "https://cdno-data.imagevenue.com/html.img8116/upload2328/loc64/th_52709_test_122_64lo.jpg",
     "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331",
 },
 
+{
+    "#url"     : "http://img159.imagevenue.com/img.php?image=73874_203_123_83lo.jpg",
+    "#comment" : "404 image (#7570)",
+    "#category": ("imagehost", "imagevenue", "image"),
+    "#class"   : imagehosts.ImagevenueImageExtractor,
+    "#urls"        : "https://cdno-data.imagevenue.com/html.img159/upload2328/loc83/73874_203_123_83lo.jpg",
+    "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+},
+
 )