diff --git a/gallery_dl/extractor/2ch.py b/gallery_dl/extractor/2ch.py index 912a2519..a4621399 100644 --- a/gallery_dl/extractor/2ch.py +++ b/gallery_dl/extractor/2ch.py @@ -46,7 +46,7 @@ class _2chThreadExtractor(Extractor): for post in posts: if files := post.get("files"): post["post_name"] = post["name"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) del post["files"] del post["name"] diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index ee3510cd..af3d6b9c 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -65,7 +65,7 @@ class _2chenThreadExtractor(Extractor): extr = text.extract_from(post) return { "name" : text.unescape(extr("", "")), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr("")[2], "%d %b %Y (%a) %H:%M:%S" ), diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 6a4d0c69..18948907 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -61,7 +61,7 @@ class _4archiveThreadExtractor(Extractor): extr = text.extract_from(post) data = { "name": extr('class="name">', ""), - "date": text.parse_datetime( + "date": self.parse_datetime( (extr('class="dateTime">', "<") or extr('class="dateTime postNum" >', "<")).strip(), "%Y-%m-%d %H:%M:%S"), diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index 120cd8a2..dfc34224 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -85,7 +85,7 @@ class _8musesAlbumExtractor(Extractor): "parent" : text.parse_int(album["parentId"]), "views" : text.parse_int(album["numberViews"]), "likes" : text.parse_int(album["numberLikes"]), - "date" : text.parse_datetime( + "date" : self.parse_datetime( album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), } diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 3249ae6f..e9adf97a 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -33,7 +33,7 @@ class AdultempireGalleryExtractor(GalleryExtractor): "gallery_id": text.parse_int(self.gallery_id), "title" : text.unescape(extr('title="', '"')), "studio" : extr(">studio", "<").strip(), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( ">released", "<").strip(), "%m/%d/%Y"), "actors" : sorted(text.split_html(extr( '"), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "
  • Posted: ", "<"), "%Y-%m-%d"), } diff --git a/gallery_dl/extractor/tiktok.py b/gallery_dl/extractor/tiktok.py index f4508066..c80cdd33 100644 --- a/gallery_dl/extractor/tiktok.py +++ b/gallery_dl/extractor/tiktok.py @@ -43,7 +43,7 @@ class TiktokExtractor(Extractor): post = video_detail["itemInfo"]["itemStruct"] post["user"] = (a := post.get("author")) and a["uniqueId"] or "" - post["date"] = text.parse_timestamp(post["createTime"]) + post["date"] = self.parse_timestamp(post["createTime"]) original_title = title = post["desc"] yield Message.Directory, post diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 7add79ac..0963cd63 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -51,7 +51,7 @@ class ToyhouseExtractor(Extractor): extr = text.extract_from(post) return { "url": extr(needle, '"'), - "date": text.parse_datetime(extr( + "date": self.parse_datetime(extr( '\n
    ', '<'), "%d %b %Y, %I:%M:%S %p"), "artists": [ diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index 8732c604..1ccdafb1 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -65,7 +65,7 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): "title_jp" : title_jp, "thumbnail" : extr('"og:image" content="', '"'), "uploader" : text.remove_html(extr('id="Uploader">', '
    ')), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('id="Uploaded">', '').strip(), "%Y %B %d"), "rating" : text.parse_float(extr( 'id="Rating">', '').partition(" ")[0]), diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index beaa5c77..f68808b8 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -99,7 +99,7 @@ class TumblrExtractor(Extractor): if "trail" in post: del post["trail"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) posts = [] if "photos" in post: # type "photo" or "link" diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py index 45836a95..e53cfaa2 100644 --- a/gallery_dl/extractor/tungsten.py +++ b/gallery_dl/extractor/tungsten.py @@ -23,7 +23,7 @@ class TungstenExtractor(Extractor): def items(self): for post in self.posts(): url = post["original_url"] - post["date"] = text.parse_datetime(post["created_at"]) + post["date"] = self.parse_datetime(post["created_at"]) post["filename"] = url[url.rfind("/")+1:] post["extension"] = "webp" yield Message.Directory, post diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index 4f9fe845..ec862fe2 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -37,7 +37,7 @@ class TwibooruExtractor(BooruExtractor): return post["view_url"] def _prepare(self, post): - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ") if "name" in post: @@ -146,7 +146,7 @@ class TwibooruAPI(): return response.json() if response.status_code == 429: - until = text.parse_datetime( + until = self.parse_datetime( response.headers["X-RL-Reset"], "%Y-%m-%d %H:%M:%S %Z") # wait an extra minute, just to be safe self.extractor.wait(until=until, adjust=60.0) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index bf125a60..7f0d21f3 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -363,11 +363,11 @@ class TwitterExtractor(Extractor): tweet_id = int(legacy["id_str"]) if tweet_id >= 300000000000000: - date = text.parse_timestamp( + date = self.parse_timestamp( ((tweet_id >> 22) + 1288834974657) // 1000) else: try: - date = text.parse_datetime( + date = self.parse_datetime( legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") except Exception: date = util.NONE @@ -455,7 +455,7 @@ class TwitterExtractor(Extractor): tdata, legacy["extended_entities"]["media"][0]) if tdata["retweet_id"]: tdata["content"] = f"RT @{author['name']}: {tdata['content']}" - tdata["date_original"] = text.parse_timestamp( + tdata["date_original"] = self.parse_timestamp( ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) return tdata @@ -492,7 +492,7 @@ class TwitterExtractor(Extractor): "id": text.parse_int(cid), "name": com.get("name"), "description": com.get("description"), - "date": text.parse_timestamp(com.get("created_at", 0) // 1000), + "date": self.parse_timestamp(com.get("created_at", 0) // 1000), "nsfw": com.get("is_nsfw"), "role": com.get("role"), "member_count": com.get("member_count"), @@ -529,7 +529,7 @@ class TwitterExtractor(Extractor): "name" : core.get("screen_name"), "nick" : core.get("name"), "location" : user["location"]["location"], - "date" : text.parse_datetime( + "date" : self.parse_datetime( core["created_at"], "%a %b %d %H:%M:%S %z %Y"), "verified" : user["verification"]["verified"], "protected" : user["privacy"]["protected"], @@ -898,7 +898,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) - tdata["date_bookmarked"] = text.parse_timestamp( + tdata["date_bookmarked"] = self.parse_timestamp( (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index cf6631fc..9d9702af 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -41,7 +41,7 @@ class UnsplashExtractor(Extractor): if metadata: photo.update(metadata) photo["extension"] = "jpg" - photo["date"] = text.parse_datetime(photo["created_at"]) + photo["date"] = self.parse_datetime(photo["created_at"]) if "tags" in photo: photo["tags"] = [t["title"] for t in photo["tags"]] diff --git a/gallery_dl/extractor/urlgalleries.py b/gallery_dl/extractor/urlgalleries.py index 4369ac63..53e348e4 100644 --- a/gallery_dl/extractor/urlgalleries.py +++ b/gallery_dl/extractor/urlgalleries.py @@ -52,7 +52,7 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor): "blog" : text.unescape(extr(' title="', '"')), "_rprt": extr(' title="', '"'), # report button "title": text.unescape(extr(' title="', '"').strip()), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr(" images in gallery | ", "<"), "%B %d, %Y"), } diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index e0107f36..e0834b62 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -47,7 +47,7 @@ class VanillarockPostExtractor(VanillarockExtractor): "count": len(imgs), "title": text.unescape(name), "path" : self.path.strip("/"), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '
    ', '
    '), "%Y-%m-%d %H:%M"), "tags" : text.split_html(extr( '
    ', '
    '))[::2], diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 22d4b9ab..2ef24a8a 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -72,7 +72,7 @@ class VkExtractor(Extractor): photo["width"] = photo["height"] = 0 photo["id"] = photo["id"].rpartition("_")[2] - photo["date"] = text.parse_timestamp(text.extr( + photo["date"] = self.parse_timestamp(text.extr( photo["date"], 'data-date="', '"')) photo["description"] = text.unescape(text.extr( photo.get("desc", ""), ">", "<")) diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index df09fce2..19464c22 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -62,7 +62,7 @@ class VscoExtractor(Extractor): "grid" : img["grid_name"], "meta" : img.get("image_meta") or {}, "tags" : [tag["text"] for tag in img.get("tags") or ()], - "date" : text.parse_timestamp(img["upload_date"] // 1000), + "date" : self.parse_timestamp(img["upload_date"] // 1000), "video" : img["is_video"], "width" : img["width"], "height": img["height"], diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index f0f27e0c..861d0c99 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -43,7 +43,7 @@ class WallhavenExtractor(Extractor): wp["url"] = wp.pop("path") if "tags" in wp: wp["tags"] = [t["name"] for t in wp["tags"]] - wp["date"] = text.parse_datetime( + wp["date"] = self.parse_datetime( wp.pop("created_at"), "%Y-%m-%d %H:%M:%S") wp["width"] = wp.pop("dimension_x") wp["height"] = wp.pop("dimension_y") diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index 8ae2a49a..5463448f 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -42,7 +42,7 @@ class WarosuThreadExtractor(Extractor): if "image" in post: for key in ("w", "h", "no", "time", "tim"): post[key] = text.parse_int(post[key]) - dt = text.parse_timestamp(post["time"]) + dt = self.parse_timestamp(post["time"]) # avoid zero-padding 'day' with %d post["now"] = dt.strftime(f"%a, %b {dt.day}, %Y %H:%M:%S") post.update(data) diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index a69f3a85..48c26f77 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -24,7 +24,7 @@ class WeasylExtractor(Extractor): # Some submissions don't have content and can be skipped if "submission" in data["media"]: data["url"] = data["media"]["submission"][0]["url"] - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") text.nameext_from_url(data["url"], data) return True @@ -42,7 +42,7 @@ class WeasylExtractor(Extractor): f"{self.root}/api/journals/{journalid}/view") data["extension"] = "html" data["html"] = "text:" + data["content"] - data["date"] = text.parse_datetime(data["posted_at"]) + data["date"] = self.parse_datetime(data["posted_at"]) return data def submissions(self, owner_login, folderid=None): diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index cc41b039..10b5aa51 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -40,7 +40,7 @@ class WebmshareVideoExtractor(Extractor): 'property="og:video:width" content="', '"')), "height": text.parse_int(extr( 'property="og:video:height" content="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "Added ", "<"), "%B %d, %Y"), "views": text.parse_int(extr('glyphicon-eye-open">', '<')), "id" : self.video_id, diff --git a/gallery_dl/extractor/weebcentral.py b/gallery_dl/extractor/weebcentral.py index 03cbf293..cddceb76 100644 --- a/gallery_dl/extractor/weebcentral.py +++ b/gallery_dl/extractor/weebcentral.py @@ -127,7 +127,7 @@ class WeebcentralMangaExtractor(WeebcentralBase, MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor": sep + minor, "chapter_type" : type, - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr(' datetime="', '"')[:-5], "%Y-%m-%dT%H:%M:%S"), } chapter.update(data) diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 3c0f0776..023f92e0 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -98,7 +98,7 @@ class WeiboExtractor(Extractor): files = [] self._extract_status(status, files) - status["date"] = text.parse_datetime( + status["date"] = self.parse_datetime( status["created_at"], "%a %b %d %H:%M:%S %z %Y") status["count"] = len(files) yield Message.Directory, status diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 31dc9cd1..7c168e9e 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -34,7 +34,7 @@ class WikifeetGalleryExtractor(GalleryExtractor): "celeb" : self.celeb, "type" : self.type, "birthplace": text.unescape(extr('"bplace":"', '"')), - "birthday" : text.parse_datetime(text.unescape( + "birthday" : self.parse_datetime(text.unescape( extr('"bdate":"', '"'))[:10], "%Y-%m-%d"), "shoesize" : text.unescape(extr('"ssize":', ',')), "rating" : text.parse_float(extr('"score":', ',')), diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index ba020d50..83220889 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -75,7 +75,7 @@ class WikimediaExtractor(BaseExtractor): for m in image["commonmetadata"] or ()} text.nameext_from_url(image["canonicaltitle"].partition(":")[2], image) - image["date"] = text.parse_datetime( + image["date"] = self.parse_datetime( image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") def items(self): diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index 6c971754..f5261d66 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -67,7 +67,7 @@ class XhamsterGalleryExtractor(XhamsterExtractor): { "id" : text.parse_int(gallery["id"]), "tags" : [t["label"] for t in info["categoriesTags"]], - "date" : text.parse_timestamp(model["created"]), + "date" : self.parse_timestamp(model["created"]), "views" : text.parse_int(model["views"]), "likes" : text.parse_int(model["rating"]["likes"]), "dislikes" : text.parse_int(model["rating"]["dislikes"]), diff --git a/gallery_dl/extractor/yiffverse.py b/gallery_dl/extractor/yiffverse.py index 1595b4d6..33640fc1 100644 --- a/gallery_dl/extractor/yiffverse.py +++ b/gallery_dl/extractor/yiffverse.py @@ -55,7 +55,7 @@ class YiffverseExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index 7bff83b4..602f994b 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -76,7 +76,7 @@ class ZerochanExtractor(BooruExtractor): data = { "id" : text.parse_int(entry_id), "file_url": jsonld["contentUrl"], - "date" : text.parse_datetime(jsonld["datePublished"]), + "date" : self.parse_datetime(jsonld["datePublished"]), "width" : text.parse_int(jsonld["width"][:-3]), "height" : text.parse_int(jsonld["height"][:-3]), "size" : text.parse_bytes(jsonld["contentSize"][:-1]), diff --git a/gallery_dl/text.py b/gallery_dl/text.py index 98bba484..e1a27ef5 100644 --- a/gallery_dl/text.py +++ b/gallery_dl/text.py @@ -8,10 +8,7 @@ """Collection of functions that work on strings/text""" -import sys import html -import time -import datetime import urllib.parse import re as re_module @@ -322,46 +319,6 @@ def build_query(params): ]) -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - return datetime.datetime.utcfromtimestamp(int(ts)) - except Exception: - return default -else: - # Python >= 3.12 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - Y, m, d, H, M, S, _, _, _ = time.gmtime(int(ts)) - return datetime.datetime(Y, m, d, H, M, S) - except Exception: - return default - - -def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0): - """Create a datetime object by parsing 'date_string'""" - try: - d = datetime.datetime.strptime(date_string, format) - o = d.utcoffset() - if o is not None: - # convert to naive UTC - d = d.replace(tzinfo=None, microsecond=0) - o - else: - if d.microsecond: - d = d.replace(microsecond=0) - if utcoffset: - # apply manual UTC offset - d += datetime.timedelta(0, utcoffset * -3600) - return d - except (TypeError, IndexError, KeyError): - return None - except (ValueError, OverflowError): - return date_string - - urljoin = urllib.parse.urljoin quote = urllib.parse.quote