From cd931e1139b2146375eb722bd6fa505e8a71b09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 8 Dec 2022 18:58:29 +0100 Subject: [PATCH] update extractor test results --- gallery_dl/extractor/35photo.py | 1 + gallery_dl/extractor/8chan.py | 4 +-- gallery_dl/extractor/deviantart.py | 14 ----------- gallery_dl/extractor/erome.py | 37 +++++++++++++++++++--------- gallery_dl/extractor/exhentai.py | 8 ++++-- gallery_dl/extractor/gelbooru_v02.py | 3 ++- gallery_dl/extractor/mangadex.py | 2 +- gallery_dl/extractor/redgifs.py | 4 +-- gallery_dl/extractor/slickpic.py | 3 ++- gallery_dl/extractor/smugmug.py | 2 +- gallery_dl/extractor/twibooru.py | 2 +- gallery_dl/extractor/twitter.py | 6 ++--- gallery_dl/extractor/unsplash.py | 18 +++++++------- gallery_dl/extractor/webtoons.py | 1 + 14 files changed, 56 insertions(+), 49 deletions(-) diff --git a/gallery_dl/extractor/35photo.py b/gallery_dl/extractor/35photo.py index 28acc3d5..f86691d4 100644 --- a/gallery_dl/extractor/35photo.py +++ b/gallery_dl/extractor/35photo.py @@ -146,6 +146,7 @@ class _35photoTagExtractor(_35photoExtractor): test = ("https://35photo.pro/tags/landscape/", { "range": "1-25", "count": 25, + "archive": False, }) def __init__(self, match): diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 1e020c25..0e128c3a 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -92,8 +92,8 @@ class _8chanThreadExtractor(_8chanExtractor): "uniquePosters": 9, "usesCustomCss": True, "usesCustomJs": False, - "wsPort": 8880, - "wssPort": 2087, + "?wsPort": 8880, + "?wssPort": 2087, }, }), ("https://8chan.se/vhs/res/4.html"), diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 45beddf3..df59be4a 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -896,20 +896,6 @@ class DeviantartDeviationExtractor(DeviantartExtractor): "range": "2-", "count": 4, }), - # video - ("https://www.deviantart.com/chi-u/art/-VIDEO-Brushes-330774593", { - "pattern": r"https://wixmp-.+wixmp.com/v/mp4/.+\.720p\.\w+.mp4", - "keyword": { - "filename": r"re:_video____brushes_\w+_by_chi_u-d5gxnb5", - "extension": "mp4", - "target": { - "duration": 306, - "filesize": 19367585, - "quality": "720p", - "src": str, - }, - } - }), # journal ("https://www.deviantart.com/shimoda7/journal/ARTility-583755752", { "url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", diff --git a/gallery_dl/extractor/erome.py b/gallery_dl/extractor/erome.py index b4dadc7e..ad3f16ba 100644 --- a/gallery_dl/extractor/erome.py +++ b/gallery_dl/extractor/erome.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -92,16 +92,29 @@ class EromeAlbumExtractor(EromeExtractor): """Extractor for albums on erome.com""" subcategory = "album" pattern = BASE_PATTERN + r"/a/(\w+)" - test = ("https://www.erome.com/a/TyFMI7ik", { - "pattern": r"https://s\d+\.erome\.com/\d+/TyFMI7ik/\w+", - "count": 9, - "keyword": { - "album_id": "TyFMI7ik", - "num": int, - "title": "Ryan Ryans", - "user": "xanub", - }, - }) + test = ( + ("https://www.erome.com/a/NQgdlWvk", { + "pattern": r"https://v\d+\.erome\.com/\d+" + r"/NQgdlWvk/j7jlzmYB_480p\.mp4", + "count": 1, + "keyword": { + "album_id": "NQgdlWvk", + "num": 1, + "title": "porn", + "user": "yYgWBZw8o8qsMzM", + }, + }), + ("https://www.erome.com/a/TdbZ4ogi", { + "pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", + "count": 6, + "keyword": { + "album_id": "TdbZ4ogi", + "num": int, + "title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", + "user": "yYgWBZw8o8qsMzM", + }, + }), + ) def albums(self): return (self.item,) @@ -110,7 +123,7 @@ class EromeAlbumExtractor(EromeExtractor): class EromeUserExtractor(EromeExtractor): subcategory = "user" pattern = BASE_PATTERN + r"/(?!a/|search\?)([^/?#]+)" - test = ("https://www.erome.com/xanub", { + test = ("https://www.erome.com/yYgWBZw8o8qsMzM", { "range": "1-25", "count": 25, }) diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 01375d81..dccc74e4 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -117,9 +117,10 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): r"|/s/([\da-f]{10})/(\d+)-(\d+))") test = ( ("https://exhentai.org/g/1200119/d55c44d3d0/", { + "options": (("original", False),), "keyword": { "cost": int, - "date": "dt:2018-03-18 20:15:00", + "date": "dt:2018-03-18 20:14:00", "eh_category": "Non-H", "expunged": False, "favorites": r"re:^[12]\d$", @@ -150,7 +151,8 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): "uploader": "klorpa", "width": int, }, - "content": "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", + "content": ("2c68cff8a7ca540a78c36fdbf5fbae0260484f87", + "e9891a4c017ed0bb734cd1efba5cd03f594d31ff"), }), ("https://exhentai.org/g/960461/4f0e369d82/", { "exception": exception.NotFoundError, @@ -159,9 +161,11 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): "exception": exception.AuthorizationError, }), ("https://exhentai.org/s/f68367b4c8/1200119-3", { + "options": (("original", False),), "count": 2, }), ("https://e-hentai.org/s/f68367b4c8/1200119-3", { + "options": (("original", False),), "count": 2, }), ("https://g.e-hentai.org/g/1200119/d55c44d3d0/"), diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index da87b8f1..facd3dbe 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -174,7 +174,8 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor): pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" test = ( ("https://rule34.xxx/index.php?page=post&s=list&tags=danraku", { - "content": "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "content": ("5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "622e80be3f496672c44aab5c47fbc6941c61bc79"), "pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", "count": 2, }), diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index 0bc35274..dae203e7 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -109,7 +109,7 @@ class MangadexChapterExtractor(MangadexExtractor): }), # 'externalUrl', but still downloadable (#2503) ("https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", { - "count": 39, + "count": 0, # 404 }), ) diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index 53e5e790..ad4282c8 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -72,7 +72,7 @@ class RedgifsUserExtractor(RedgifsExtractor): pattern = r"(?:https?://)?(?:www\.)?redgifs\.com/users/([^/?#]+)" test = ("https://www.redgifs.com/users/Natalifiction", { "pattern": r"https://\w+\.redgifs\.com/[A-Za-z]+\.mp4", - "count": ">= 120", + "count": ">= 100", }) def metadata(self): @@ -89,7 +89,7 @@ class RedgifsSearchExtractor(RedgifsExtractor): pattern = r"(?:https?://)?(?:www\.)?redgifs\.com/browse/?\?([^#]+)" test = ( ("https://www.redgifs.com/browse?tags=JAV", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.mp4", + "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", "range": "1-10", "count": 10, }), diff --git a/gallery_dl/extractor/slickpic.py b/gallery_dl/extractor/slickpic.py index ae4e2e8a..3727c0b0 100644 --- a/gallery_dl/extractor/slickpic.py +++ b/gallery_dl/extractor/slickpic.py @@ -43,7 +43,8 @@ class SlickpicAlbumExtractor(SlickpicExtractor): }), ("https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", { "range": "34", - "content": ("52b5a310587de1048030ab13a912f6a3a9cc7dab", + "content": ("276eb2c902187bb177ae8013e310e1d6641fba9a", + "52b5a310587de1048030ab13a912f6a3a9cc7dab", "cec6630e659dc72db1ee1a9a6f3b525189261988", "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", "22e83645fc242bc3584eca7ec982c8a53a4d8a44"), diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py index 2264fe48..713d4c41 100644 --- a/gallery_dl/extractor/smugmug.py +++ b/gallery_dl/extractor/smugmug.py @@ -117,7 +117,7 @@ class SmugmugImageExtractor(SmugmugExtractor): # video ("https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", { "url": "04d0ab1ff829ca7d78f5acb5548953df08e9a5ee", - "keyword": "4cef98133ace511adc874c9d9abac5817ba0d856", + "keyword": "2b545184592c282b365fcbb7df6ca7952b8a3173", }), ) diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index f010f926..30bf2f15 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -83,7 +83,7 @@ class TwibooruPostExtractor(TwibooruExtractor): "tag_ids": list, "tags": list, "thumbnails_generated": True, - "updated_at": "2022-09-21T14:31:50.441Z", + "updated_at": "2022-11-27T00:34:50.483Z", "upvotes": int, "view_url": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", "width": 576, diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 22d4a6ec..d0411acf 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -633,7 +633,7 @@ class TwitterEventExtractor(TwitterExtractor): pattern = BASE_PATTERN + r"/i/events/(\d+)" test = ("https://twitter.com/i/events/1484669206993903616", { "range": "1-20", - "count": ">5", + "count": ">=1", }) def metadata(self): @@ -759,7 +759,7 @@ class TwitterTweetExtractor(TwitterExtractor): # retweet with missing media entities (#1555) ("https://twitter.com/morino_ya/status/1392763691599237121", { "options": (("retweets", True),), - "count": 4, + "count": 0, # private }), # deleted quote tweet (#2225) ("https://twitter.com/i/web/status/1460044411165888515", { @@ -782,7 +782,7 @@ class TwitterTweetExtractor(TwitterExtractor): # '?format=...&name=...'-style URLs ("https://twitter.com/poco_dandy/status/1150646424461176832", { "options": (("cards", True),), - "pattern": r"https://pbs.twimg.com/card_img/157\d+/\w+" + "pattern": r"https://pbs.twimg.com/card_img/157\d+/[\w-]+" r"\?format=(jpg|png)&name=orig$", "range": "1-2", }), diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index 8bea18c7..b298c27e 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -78,11 +78,11 @@ class UnsplashImageExtractor(UnsplashExtractor): pattern = BASE_PATTERN + r"/photos/([^/?#]+)" test = ("https://unsplash.com/photos/lsoogGC_5dg", { "pattern": r"https://images\.unsplash\.com/photo-1586348943529-" - r"beaae6c28db9\?ixid=\w+&ixlib=rb-1.2.1", + r"beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", "keyword": { "alt_description": "re:silhouette of trees near body of water ", "blur_hash": "LZP4uQS4jboe%#o0WCa}2doJNaaz", - "categories": list, + "? categories": list, "color": "#f3c08c", "created_at": "2020-04-08T12:29:42Z", "date": "dt:2020-04-08 12:29:42", @@ -108,9 +108,8 @@ class UnsplashImageExtractor(UnsplashExtractor): "name": "Beaver Dam, WI 53916, USA", "position": { "latitude": 43.457769, - "longitude": -88.837329 + "longitude": -88.837329, }, - "title": "Beaver Dam, WI 53916, USA" }, "promoted_at": "2020-04-08T15:12:03Z", "sponsorship": None, @@ -149,7 +148,7 @@ class UnsplashUserExtractor(UnsplashExtractor): pattern = BASE_PATTERN + r"/@(\w+)/?$" test = ("https://unsplash.com/@davehoefler", { "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-1\.2\.1$", + r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", "range": "1-30", "count": 30, }) @@ -166,7 +165,7 @@ class UnsplashFavoriteExtractor(UnsplashExtractor): pattern = BASE_PATTERN + r"/@(\w+)/likes" test = ("https://unsplash.com/@davehoefler/likes", { "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-1\.2\.1$", + r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", "range": "1-30", "count": 30, }) @@ -184,7 +183,7 @@ class UnsplashCollectionExtractor(UnsplashExtractor): test = ( ("https://unsplash.com/collections/3178572/winter", { "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-1\.2\.1$", + r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", "keyword": {"collection_id": "3178572", "collection_title": "winter"}, "range": "1-30", @@ -212,8 +211,9 @@ class UnsplashSearchExtractor(UnsplashExtractor): subcategory = "search" pattern = BASE_PATTERN + r"/s/photos/([^/?#]+)(?:\?([^#]+))?" test = ("https://unsplash.com/s/photos/hair-style", { - "pattern": r"https://images\.unsplash\.com/((flagged/)?photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-1\.2\.1$", + "pattern": r"https://(images|plus)\.unsplash\.com" + r"/((flagged/|premium_)?photo-\d+-\w+" + r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", "range": "1-30", "count": 30, }) diff --git a/gallery_dl/extractor/webtoons.py b/gallery_dl/extractor/webtoons.py index 8a22fcb7..21f7c21e 100644 --- a/gallery_dl/extractor/webtoons.py +++ b/gallery_dl/extractor/webtoons.py @@ -57,6 +57,7 @@ class WebtoonsEpisodeExtractor(WebtoonsBase, GalleryExtractor): }), (("https://www.webtoons.com/en/challenge/punderworld" "/happy-earth-day-/viewer?title_no=312584&episode_no=40"), { + "exception": exception.NotFoundError, "keyword": { "comic": "punderworld", "description": str,