diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py new file mode 100644 index 00000000..b37ebf38 --- /dev/null +++ b/gallery_dl/dt.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Date/Time utilities""" + +import sys +import time +from datetime import datetime, date, timedelta, timezone # noqa F401 + + +class NullDatetime(datetime): + + def __bool__(self): + return False + + def __str__(self): + return "[Invalid DateTime]" + + def __format__(self, format_spec): + return "[Invalid DateTime]" + + +NONE = NullDatetime(1, 1, 1) +EPOCH = datetime(1970, 1, 1) +SECOND = timedelta(0, 1) + + +def normalize(dt): + # if (o := dt.utcoffset()) is not None: + # return dt.replace(tzinfo=None, microsecond=0) - o + if dt.tzinfo is not None: + return dt.astimezone(timezone.utc).replace(tzinfo=None, microsecond=0) + if dt.microsecond: + return dt.replace(microsecond=0) + return dt + + +def convert(value): + """Convert 'value' to a naive UTC datetime object""" + if not value: + return NONE + if isinstance(value, datetime): + return normalize(value) + if isinstance(value, str) and (dt := parse_iso(value)) is not NONE: + return dt + return parse_ts(value) + + +def parse(dt_string, format): + """Parse 'dt_string' according to 'format'""" + try: + return normalize(datetime.strptime(dt_string, format)) + except Exception: + return NONE + + +if sys.hexversion < 0x30c0000: + # Python <= 3.11 + def parse_iso(dt_string): + """Parse 'dt_string' as ISO 8601 value""" + try: + if dt_string[-1] == "Z": + # compat for Python < 3.11 + dt_string = dt_string[:-1] + elif dt_string[-5] in "+-": + # compat for Python < 3.11 + dt_string = f"{dt_string[:-2]}:{dt_string[-2:]}" + return normalize(datetime.fromisoformat(dt_string)) + except Exception: + return NONE + + from_ts = datetime.utcfromtimestamp + now = datetime.utcnow + +else: + # Python >= 3.12 + def parse_iso(dt_string): + """Parse 'dt_string' as ISO 8601 value""" + try: + return normalize(datetime.fromisoformat(dt_string)) + except Exception: + return NONE + + def from_ts(ts=None): + """Convert Unix timestamp to naive UTC datetime""" + Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) + return datetime(Y, m, d, H, M, S) + + now = from_ts + + +def parse_ts(ts, default=NONE): + """Create a datetime object from a Unix timestamp""" + try: + return from_ts(int(ts)) + except Exception: + return default + + +def to_ts(dt): + """Convert naive UTC datetime to Unix timestamp""" + return (dt - EPOCH) / SECOND + + +def to_ts_string(dt): + """Convert naive UTC datetime to Unix timestamp string""" + try: + return str((dt - EPOCH) // SECOND) + except Exception: + return "" diff --git a/gallery_dl/extractor/2ch.py b/gallery_dl/extractor/2ch.py index 912a2519..a4621399 100644 --- a/gallery_dl/extractor/2ch.py +++ b/gallery_dl/extractor/2ch.py @@ -46,7 +46,7 @@ class _2chThreadExtractor(Extractor): for post in posts: if files := post.get("files"): post["post_name"] = post["name"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) del post["files"] del post["name"] diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index ee3510cd..af3d6b9c 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -65,7 +65,7 @@ class _2chenThreadExtractor(Extractor): extr = text.extract_from(post) return { "name" : text.unescape(extr("", "")), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr("")[2], "%d %b %Y (%a) %H:%M:%S" ), diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 4c434643..a8bb5d9c 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -7,7 +7,7 @@ """Extractors for https://4archive.org/""" from .common import Extractor, Message -from .. import text, util +from .. import text, dt class _4archiveThreadExtractor(Extractor): @@ -37,7 +37,7 @@ class _4archiveThreadExtractor(Extractor): for post in posts: post.update(data) - post["time"] = int(util.datetime_to_timestamp(post["date"])) + post["time"] = int(dt.to_ts(post["date"])) yield Message.Directory, post if "url" in post: yield Message.Url, post["url"], text.nameext_from_url( @@ -61,10 +61,9 @@ class _4archiveThreadExtractor(Extractor): extr = text.extract_from(post) data = { "name": extr('class="name">', ""), - "date": text.parse_datetime( + "date": self.parse_datetime_iso( (extr('class="dateTime">', "<") or - extr('class="dateTime postNum" >', "<")).strip(), - "%Y-%m-%d %H:%M:%S"), + extr('class="dateTime postNum" >', "<")).strip()), "no" : text.parse_int(extr(">Post No.", "<")), } if 'class="file"' in post: diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 03850676..6524821b 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -9,9 +9,8 @@ """Extractors for https://8chan.moe/""" from .common import Extractor, Message -from .. import text, util +from .. import text, dt from ..cache import memcache -from datetime import timedelta import itertools BASE_PATTERN = r"(?:https?://)?8chan\.(moe|se|cc)" @@ -44,7 +43,7 @@ class _8chanExtractor(Extractor): def cookies_prepare(self): # fetch captcha cookies # (necessary to download without getting interrupted) - now = util.datetime_utcnow() + now = dt.now() url = self.root + "/captcha.js" params = {"d": now.strftime("%a %b %d %Y %H:%M:%S GMT+0000 (UTC)")} self.request(url, params=params).content @@ -57,7 +56,7 @@ class _8chanExtractor(Extractor): if cookie.domain.endswith(domain): cookie.expires = None if cookie.name == "captchaexpiration": - cookie.value = (now + timedelta(30, 300)).strftime( + cookie.value = (now + dt.timedelta(30, 300)).strftime( "%a, %d %b %Y %H:%M:%S GMT") return self.cookies diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index 120cd8a2..7cb6eee9 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -85,8 +85,7 @@ class _8musesAlbumExtractor(Extractor): "parent" : text.parse_int(album["parentId"]), "views" : text.parse_int(album["numberViews"]), "likes" : text.parse_int(album["numberLikes"]), - "date" : text.parse_datetime( - album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), + "date" : self.parse_datetime_iso(album["updatedAt"]), } def _unobfuscate(self, data): diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 3249ae6f..e9adf97a 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -33,7 +33,7 @@ class AdultempireGalleryExtractor(GalleryExtractor): "gallery_id": text.parse_int(self.gallery_id), "title" : text.unescape(extr('title="', '"')), "studio" : extr(">studio", "<").strip(), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( ">released", "<").strip(), "%m/%d/%Y"), "actors" : sorted(text.split_html(extr( '"), - "date" : text.parse_datetime(extr( - "
  • Posted: ", "<"), "%Y-%m-%d"), + "date" : self.parse_datetime_iso(extr("
  • Posted: ", "<")), } if (c := url[27]) == "v": diff --git a/gallery_dl/extractor/tiktok.py b/gallery_dl/extractor/tiktok.py index 3ca72c51..5e75380f 100644 --- a/gallery_dl/extractor/tiktok.py +++ b/gallery_dl/extractor/tiktok.py @@ -43,7 +43,7 @@ class TiktokExtractor(Extractor): post = video_detail["itemInfo"]["itemStruct"] post["user"] = (a := post.get("author")) and a["uniqueId"] or "" - post["date"] = text.parse_timestamp(post["createTime"]) + post["date"] = self.parse_timestamp(post["createTime"]) original_title = title = post["desc"] yield Message.Directory, post diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 7add79ac..0963cd63 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -51,7 +51,7 @@ class ToyhouseExtractor(Extractor): extr = text.extract_from(post) return { "url": extr(needle, '"'), - "date": text.parse_datetime(extr( + "date": self.parse_datetime(extr( '\n
    ', '<'), "%d %b %Y, %I:%M:%S %p"), "artists": [ diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index 8732c604..1ccdafb1 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -65,7 +65,7 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): "title_jp" : title_jp, "thumbnail" : extr('"og:image" content="', '"'), "uploader" : text.remove_html(extr('id="Uploader">', '
    ')), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('id="Uploaded">', '').strip(), "%Y %B %d"), "rating" : text.parse_float(extr( 'id="Rating">', '').partition(" ")[0]), diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 92fc8314..f68808b8 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -9,8 +9,7 @@ """Extractors for https://www.tumblr.com/""" from .common import Extractor, Message -from .. import text, util, oauth, exception -from datetime import datetime, date, timedelta +from .. import text, util, dt, oauth, exception BASE_PATTERN = ( @@ -100,7 +99,7 @@ class TumblrExtractor(Extractor): if "trail" in post: del post["trail"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) posts = [] if "photos" in post: # type "photo" or "link" @@ -313,7 +312,7 @@ class TumblrDayExtractor(TumblrExtractor): def posts(self): year, month, day = self.groups[3].split("/") - ordinal = date(int(year), int(month), int(day)).toordinal() + ordinal = dt.date(int(year), int(month), int(day)).toordinal() # 719163 == date(1970, 1, 1).toordinal() self.date_min = (ordinal - 719163) * 86400 @@ -514,7 +513,7 @@ class TumblrAPI(oauth.OAuth1API): self.extractor.wait(seconds=reset) continue - t = (datetime.now() + timedelta(0, float(reset))).time() + t = (dt.now() + dt.timedelta(0, float(reset))).time() raise exception.AbortExtraction( f"Aborting - Rate limit will reset at " f"{t.hour:02}:{t.minute:02}:{t.second:02}") diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py index 45836a95..6b56035f 100644 --- a/gallery_dl/extractor/tungsten.py +++ b/gallery_dl/extractor/tungsten.py @@ -23,7 +23,7 @@ class TungstenExtractor(Extractor): def items(self): for post in self.posts(): url = post["original_url"] - post["date"] = text.parse_datetime(post["created_at"]) + post["date"] = self.parse_datetime_iso(post["created_at"]) post["filename"] = url[url.rfind("/")+1:] post["extension"] = "webp" yield Message.Directory, post diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index 4f9fe845..4558e212 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -37,8 +37,7 @@ class TwibooruExtractor(BooruExtractor): return post["view_url"] def _prepare(self, post): - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created_at"]) if "name" in post: name, sep, rest = post["name"].rpartition(".") @@ -146,8 +145,8 @@ class TwibooruAPI(): return response.json() if response.status_code == 429: - until = text.parse_datetime( - response.headers["X-RL-Reset"], "%Y-%m-%d %H:%M:%S %Z") + until = self.parse_datetime_iso( + response.headers["X-RL-Reset"][:19]) # wait an extra minute, just to be safe self.extractor.wait(until=until, adjust=60.0) continue diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 239418ff..fd10cd6f 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -362,11 +362,11 @@ class TwitterExtractor(Extractor): tweet_id = int(legacy["id_str"]) if tweet_id >= 300000000000000: - date = text.parse_timestamp( + date = self.parse_timestamp( ((tweet_id >> 22) + 1288834974657) // 1000) else: try: - date = text.parse_datetime( + date = self.parse_datetime( legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") except Exception: date = util.NONE @@ -454,7 +454,7 @@ class TwitterExtractor(Extractor): tdata, legacy["extended_entities"]["media"][0]) if tdata["retweet_id"]: tdata["content"] = f"RT @{author['name']}: {tdata['content']}" - tdata["date_original"] = text.parse_timestamp( + tdata["date_original"] = self.parse_timestamp( ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) return tdata @@ -491,7 +491,7 @@ class TwitterExtractor(Extractor): "id": text.parse_int(cid), "name": com.get("name"), "description": com.get("description"), - "date": text.parse_timestamp(com.get("created_at", 0) // 1000), + "date": self.parse_timestamp(com.get("created_at", 0) // 1000), "nsfw": com.get("is_nsfw"), "role": com.get("role"), "member_count": com.get("member_count"), @@ -528,7 +528,7 @@ class TwitterExtractor(Extractor): "name" : core.get("screen_name"), "nick" : core.get("name"), "location" : user["location"].get("location"), - "date" : text.parse_datetime( + "date" : self.parse_datetime( core["created_at"], "%a %b %d %H:%M:%S %z %Y"), "verified" : user["verification"]["verified"], "protected" : user["privacy"]["protected"], @@ -897,7 +897,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) - tdata["date_bookmarked"] = text.parse_timestamp( + tdata["date_bookmarked"] = self.parse_timestamp( (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index cf6631fc..f8a82585 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -41,7 +41,7 @@ class UnsplashExtractor(Extractor): if metadata: photo.update(metadata) photo["extension"] = "jpg" - photo["date"] = text.parse_datetime(photo["created_at"]) + photo["date"] = self.parse_datetime_iso(photo["created_at"]) if "tags" in photo: photo["tags"] = [t["title"] for t in photo["tags"]] diff --git a/gallery_dl/extractor/urlgalleries.py b/gallery_dl/extractor/urlgalleries.py index 4369ac63..53e348e4 100644 --- a/gallery_dl/extractor/urlgalleries.py +++ b/gallery_dl/extractor/urlgalleries.py @@ -52,7 +52,7 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor): "blog" : text.unescape(extr(' title="', '"')), "_rprt": extr(' title="', '"'), # report button "title": text.unescape(extr(' title="', '"').strip()), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr(" images in gallery | ", "<"), "%B %d, %Y"), } diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index e0107f36..a6a10ff3 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -47,8 +47,8 @@ class VanillarockPostExtractor(VanillarockExtractor): "count": len(imgs), "title": text.unescape(name), "path" : self.path.strip("/"), - "date" : text.parse_datetime(extr( - '
    ', '
    '), "%Y-%m-%d %H:%M"), + "date" : self.parse_datetime_iso(extr( + '
    ', '
    ')), "tags" : text.split_html(extr( '
    ', '
    '))[::2], } diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 22d4b9ab..2ef24a8a 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -72,7 +72,7 @@ class VkExtractor(Extractor): photo["width"] = photo["height"] = 0 photo["id"] = photo["id"].rpartition("_")[2] - photo["date"] = text.parse_timestamp(text.extr( + photo["date"] = self.parse_timestamp(text.extr( photo["date"], 'data-date="', '"')) photo["description"] = text.unescape(text.extr( photo.get("desc", ""), ">", "<")) diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index df09fce2..19464c22 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -62,7 +62,7 @@ class VscoExtractor(Extractor): "grid" : img["grid_name"], "meta" : img.get("image_meta") or {}, "tags" : [tag["text"] for tag in img.get("tags") or ()], - "date" : text.parse_timestamp(img["upload_date"] // 1000), + "date" : self.parse_timestamp(img["upload_date"] // 1000), "video" : img["is_video"], "width" : img["width"], "height": img["height"], diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index f0f27e0c..623de8bf 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -43,8 +43,7 @@ class WallhavenExtractor(Extractor): wp["url"] = wp.pop("path") if "tags" in wp: wp["tags"] = [t["name"] for t in wp["tags"]] - wp["date"] = text.parse_datetime( - wp.pop("created_at"), "%Y-%m-%d %H:%M:%S") + wp["date"] = self.parse_datetime_iso(wp.pop("created_at")) wp["width"] = wp.pop("dimension_x") wp["height"] = wp.pop("dimension_y") wp["wh_category"] = wp["category"] diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index 8ae2a49a..5463448f 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -42,7 +42,7 @@ class WarosuThreadExtractor(Extractor): if "image" in post: for key in ("w", "h", "no", "time", "tim"): post[key] = text.parse_int(post[key]) - dt = text.parse_timestamp(post["time"]) + dt = self.parse_timestamp(post["time"]) # avoid zero-padding 'day' with %d post["now"] = dt.strftime(f"%a, %b {dt.day}, %Y %H:%M:%S") post.update(data) diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index a69f3a85..333113b8 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -24,8 +24,7 @@ class WeasylExtractor(Extractor): # Some submissions don't have content and can be skipped if "submission" in data["media"]: data["url"] = data["media"]["submission"][0]["url"] - data["date"] = text.parse_datetime( - data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") + data["date"] = self.parse_datetime_iso(data["posted_at"][:19]) text.nameext_from_url(data["url"], data) return True return False @@ -42,7 +41,7 @@ class WeasylExtractor(Extractor): f"{self.root}/api/journals/{journalid}/view") data["extension"] = "html" data["html"] = "text:" + data["content"] - data["date"] = text.parse_datetime(data["posted_at"]) + data["date"] = self.parse_datetime_iso(data["posted_at"]) return data def submissions(self, owner_login, folderid=None): diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index cc41b039..10b5aa51 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -40,7 +40,7 @@ class WebmshareVideoExtractor(Extractor): 'property="og:video:width" content="', '"')), "height": text.parse_int(extr( 'property="og:video:height" content="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "Added ", "<"), "%B %d, %Y"), "views": text.parse_int(extr('glyphicon-eye-open">', '<')), "id" : self.video_id, diff --git a/gallery_dl/extractor/weebcentral.py b/gallery_dl/extractor/weebcentral.py index 03cbf293..c5496e32 100644 --- a/gallery_dl/extractor/weebcentral.py +++ b/gallery_dl/extractor/weebcentral.py @@ -127,8 +127,8 @@ class WeebcentralMangaExtractor(WeebcentralBase, MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor": sep + minor, "chapter_type" : type, - "date" : text.parse_datetime( - extr(' datetime="', '"')[:-5], "%Y-%m-%dT%H:%M:%S"), + "date" : self.parse_datetime_iso(extr( + ' datetime="', '"')[:-5]), } chapter.update(data) results.append((base + chapter_id, chapter)) diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 428a4bf9..cd18ae4f 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -103,7 +103,7 @@ class WeiboExtractor(Extractor): status["text"].endswith('class="expand">展开'): status = self._status_by_id(status["id"]) - status["date"] = text.parse_datetime( + status["date"] = self.parse_datetime( status["created_at"], "%a %b %d %H:%M:%S %z %Y") status["count"] = len(files) yield Message.Directory, status diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 31dc9cd1..a07fd840 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -34,8 +34,8 @@ class WikifeetGalleryExtractor(GalleryExtractor): "celeb" : self.celeb, "type" : self.type, "birthplace": text.unescape(extr('"bplace":"', '"')), - "birthday" : text.parse_datetime(text.unescape( - extr('"bdate":"', '"'))[:10], "%Y-%m-%d"), + "birthday" : self.parse_datetime_iso(text.unescape(extr( + '"bdate":"', '"'))[:10]), "shoesize" : text.unescape(extr('"ssize":', ',')), "rating" : text.parse_float(extr('"score":', ',')), "celebrity" : text.unescape(extr('"cname":"', '"')), diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index ba020d50..040e4f6a 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -75,8 +75,7 @@ class WikimediaExtractor(BaseExtractor): for m in image["commonmetadata"] or ()} text.nameext_from_url(image["canonicaltitle"].partition(":")[2], image) - image["date"] = text.parse_datetime( - image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + image["date"] = self.parse_datetime_iso(image["timestamp"]) def items(self): for info in self._pagination(self.params): diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index 6c971754..f5261d66 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -67,7 +67,7 @@ class XhamsterGalleryExtractor(XhamsterExtractor): { "id" : text.parse_int(gallery["id"]), "tags" : [t["label"] for t in info["categoriesTags"]], - "date" : text.parse_timestamp(model["created"]), + "date" : self.parse_timestamp(model["created"]), "views" : text.parse_int(model["views"]), "likes" : text.parse_int(model["rating"]["likes"]), "dislikes" : text.parse_int(model["rating"]["dislikes"]), diff --git a/gallery_dl/extractor/yiffverse.py b/gallery_dl/extractor/yiffverse.py index 1595b4d6..f1073ed6 100644 --- a/gallery_dl/extractor/yiffverse.py +++ b/gallery_dl/extractor/yiffverse.py @@ -55,8 +55,7 @@ class YiffverseExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = text.parse_datetime( - post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created"]) post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: post["tags"] = [t["value"] for t in post["tags"]] diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index 7bff83b4..35222da9 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -76,7 +76,7 @@ class ZerochanExtractor(BooruExtractor): data = { "id" : text.parse_int(entry_id), "file_url": jsonld["contentUrl"], - "date" : text.parse_datetime(jsonld["datePublished"]), + "date" : self.parse_datetime_iso(jsonld["datePublished"]), "width" : text.parse_int(jsonld["width"][:-3]), "height" : text.parse_int(jsonld["height"][:-3]), "size" : text.parse_bytes(jsonld["contentSize"][:-1]), diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 5246f663..8b5e7adf 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -13,9 +13,8 @@ import sys import time import string import _string -import datetime import operator -from . import text, util +from . import text, util, dt NONE = util.NONE @@ -68,8 +67,8 @@ class StringFormatter(): - "g": calls text.slugify() - "j": calls json.dumps - "t": calls str.strip - - "T": calls util.datetime_to_timestamp_string() - - "d": calls text.parse_timestamp + - "T": calls dt.to_ts_string() + - "d": calls dt.parse_ts() - "s": calls str() - "S": calls util.to_string() - "U": calls urllib.parse.unescape @@ -471,9 +470,9 @@ def _parse_datetime(format_spec, default): dt_format = dt_format[1:] fmt = _build_format_func(format_spec, default) - def dt(obj): - return fmt(text.parse_datetime(obj, dt_format)) - return dt + def dt_parse(obj): + return fmt(dt.parse(obj, dt_format)) + return dt_parse def _parse_offset(format_spec, default): @@ -482,15 +481,15 @@ def _parse_offset(format_spec, default): fmt = _build_format_func(format_spec, default) if not offset or offset == "local": - def off(dt): - local = time.localtime(util.datetime_to_timestamp(dt)) - return fmt(dt + datetime.timedelta(0, local.tm_gmtoff)) + def off(dt_utc): + local = time.localtime(dt.to_ts(dt_utc)) + return fmt(dt_utc + dt.timedelta(0, local.tm_gmtoff)) else: hours, _, minutes = offset.partition(":") offset = 3600 * int(hours) if minutes: offset += 60 * (int(minutes) if offset > 0 else -int(minutes)) - offset = datetime.timedelta(0, offset) + offset = dt.timedelta(0, offset) def off(obj): return fmt(obj + offset) @@ -557,7 +556,7 @@ _FORMATTERS = { _GLOBALS = { "_env": lambda: os.environ, "_lit": lambda: _literal, - "_now": datetime.datetime.now, + "_now": dt.datetime.now, "_nul": lambda: util.NONE, } _CONVERSIONS = { @@ -569,9 +568,9 @@ _CONVERSIONS = { "t": str.strip, "n": len, "L": util.code_to_language, - "T": util.datetime_to_timestamp_string, - "d": text.parse_timestamp, - "D": util.to_datetime, + "T": dt.to_ts_string, + "d": dt.parse_ts, + "D": dt.convert, "U": text.unescape, "H": lambda s: text.unescape(text.remove_html(s)), "g": text.slugify, diff --git a/gallery_dl/postprocessor/mtime.py b/gallery_dl/postprocessor/mtime.py index b1269dd5..7d4796e7 100644 --- a/gallery_dl/postprocessor/mtime.py +++ b/gallery_dl/postprocessor/mtime.py @@ -9,8 +9,7 @@ """Use metadata as file modification time""" from .common import PostProcessor -from .. import text, util, formatter -from datetime import datetime +from .. import text, util, dt, formatter class MtimePP(PostProcessor): @@ -36,8 +35,8 @@ class MtimePP(PostProcessor): return pathfmt.kwdict["_mtime_meta"] = ( - util.datetime_to_timestamp(mtime) - if isinstance(mtime, datetime) else + dt.to_ts(mtime) + if isinstance(mtime, dt.datetime) else text.parse_int(mtime) ) diff --git a/gallery_dl/text.py b/gallery_dl/text.py index 4b6f6ba4..9738edb2 100644 --- a/gallery_dl/text.py +++ b/gallery_dl/text.py @@ -8,10 +8,7 @@ """Collection of functions that work on strings/text""" -import sys import html -import time -import datetime import urllib.parse import re as re_module @@ -340,46 +337,6 @@ def build_query(params): ]) -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - return datetime.datetime.utcfromtimestamp(int(ts)) - except Exception: - return default -else: - # Python >= 3.12 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - Y, m, d, H, M, S, _, _, _ = time.gmtime(int(ts)) - return datetime.datetime(Y, m, d, H, M, S) - except Exception: - return default - - -def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0): - """Create a datetime object by parsing 'date_string'""" - try: - d = datetime.datetime.strptime(date_string, format) - o = d.utcoffset() - if o is not None: - # convert to naive UTC - d = d.replace(tzinfo=None, microsecond=0) - o - else: - if d.microsecond: - d = d.replace(microsecond=0) - if utcoffset: - # apply manual UTC offset - d += datetime.timedelta(0, utcoffset * -3600) - return d - except (TypeError, IndexError, KeyError): - return None - except (ValueError, OverflowError): - return date_string - - urljoin = urllib.parse.urljoin quote = urllib.parse.quote diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 49c1ba8e..7fc3363c 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -16,7 +16,6 @@ import random import getpass import hashlib import binascii -import datetime import functools import itertools import subprocess @@ -24,7 +23,7 @@ import collections import urllib.parse from http.cookiejar import Cookie from email.utils import mktime_tz, parsedate_tz -from . import text, version, exception +from . import text, dt, version, exception def bencode(num, alphabet="0123456789"): @@ -228,63 +227,6 @@ def to_string(value): return str(value) -def to_datetime(value): - """Convert 'value' to a datetime object""" - if not value: - return EPOCH - - if isinstance(value, datetime.datetime): - return value - - if isinstance(value, str): - try: - if value[-1] == "Z": - # compat for Python < 3.11 - value = value[:-1] - dt = datetime.datetime.fromisoformat(value) - if dt.tzinfo is None: - if dt.microsecond: - dt = dt.replace(microsecond=0) - else: - # convert to naive UTC - dt = dt.astimezone(datetime.timezone.utc).replace( - microsecond=0, tzinfo=None) - return dt - except Exception: - pass - - return text.parse_timestamp(value, EPOCH) - - -def datetime_to_timestamp(dt): - """Convert naive UTC datetime to Unix timestamp""" - return (dt - EPOCH) / SECOND - - -def datetime_to_timestamp_string(dt): - """Convert naive UTC datetime to Unix timestamp string""" - try: - return str((dt - EPOCH) // SECOND) - except Exception: - return "" - - -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - datetime_utcfromtimestamp = datetime.datetime.utcfromtimestamp - datetime_utcnow = datetime.datetime.utcnow - datetime_from_timestamp = datetime_utcfromtimestamp -else: - # Python >= 3.12 - def datetime_from_timestamp(ts=None): - """Convert Unix timestamp to naive UTC datetime""" - Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) - return datetime.datetime(Y, m, d, H, M, S) - - datetime_utcfromtimestamp = datetime_from_timestamp - datetime_utcnow = datetime_from_timestamp - - def json_default(obj): if isinstance(obj, CustomNone): return None @@ -379,7 +321,7 @@ def extract_headers(response): text.nameext_from_url(name, data) if hlm := headers.get("last-modified"): - data["date"] = datetime.datetime(*parsedate_tz(hlm)[:6]) + data["date"] = dt.datetime(*parsedate_tz(hlm)[:6]) return data @@ -751,11 +693,11 @@ class Flags(): # 735506 == 739342 - 137 * 28 # v135.0 release of Chrome on 2025-04-01 has ordinal 739342 # 735562 == 739342 - 135 * 28 -# _ord_today = datetime.date.today().toordinal() +# _ord_today = dt.date.today().toordinal() # _ff_ver = (_ord_today - 735506) // 28 # _ch_ver = (_ord_today - 735562) // 28 -_ff_ver = (datetime.date.today().toordinal() - 735506) // 28 +_ff_ver = (dt.date.today().toordinal() - 735506) // 28 # _ch_ver = _ff_ver - 2 re = text.re @@ -763,8 +705,6 @@ re_compile = text.re_compile NONE = CustomNone() FLAGS = Flags() -EPOCH = datetime.datetime(1970, 1, 1) -SECOND = datetime.timedelta(0, 1) WINDOWS = (os.name == "nt") SENTINEL = object() EXECUTABLE = getattr(sys, "frozen", False) @@ -786,8 +726,8 @@ GLOBALS = { "contains" : contains, "parse_int": text.parse_int, "urlsplit" : urllib.parse.urlsplit, - "datetime" : datetime.datetime, - "timedelta": datetime.timedelta, + "datetime" : dt.datetime, + "timedelta": dt.timedelta, "abort" : raises(exception.StopExtraction), "error" : raises(exception.AbortExtraction), "terminate": raises(exception.TerminateExtraction), diff --git a/test/results/2chen.py b/test/results/2chen.py index 589053fa..132f75d7 100644 --- a/test/results/2chen.py +++ b/test/results/2chen.py @@ -6,7 +6,6 @@ gallery_dl = __import__("gallery_dl.extractor.2chen") _2chen = getattr(gallery_dl.extractor, "2chen") -import datetime __tests__ = ( @@ -18,7 +17,7 @@ __tests__ = ( "#count" : ">= 179", "board" : "tv", - "date" : datetime.datetime, + "date" : "type:datetime", "hash" : r"re:[0-9a-f]{40}", "name" : "Anonymous", "no" : r"re:\d+", diff --git a/test/results/4archive.py b/test/results/4archive.py index cebec6fc..f907577f 100644 --- a/test/results/4archive.py +++ b/test/results/4archive.py @@ -6,7 +6,6 @@ gallery_dl = __import__("gallery_dl.extractor.4archive") _4archive = getattr(gallery_dl.extractor, "4archive") -import datetime __tests__ = ( @@ -19,7 +18,7 @@ __tests__ = ( "board" : "u", "com" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "name" : "Anonymous", "no" : range(2397221, 2418158), "thread": 2397221, diff --git a/test/results/deviantart.py b/test/results/deviantart.py index c3a6c537..ee911e99 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import deviantart -import datetime from gallery_dl import exception @@ -60,7 +59,7 @@ __tests__ = ( "transparency": bool, "width" : int, }, - "date" : datetime.datetime, + "date" : "type:datetime", "deviationid" : str, "?download_filesize": int, "extension" : str, diff --git a/test/results/facebook.py b/test/results/facebook.py index 7eda027d..64f49690 100644 --- a/test/results/facebook.py +++ b/test/results/facebook.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import facebook from gallery_dl import exception -import datetime __tests__ = ( @@ -191,7 +190,7 @@ __tests__ = ( "#count" : 1, "caption" : "They were on a break... #FriendsReunion #MoreTogether", - "date" : datetime.datetime(2021, 5, 27, 21, 55, 19), + "date" : "dt:2021-05-27 21:55:19", "filename" : "191053255_10160743390471729_9001965649022744000_n", "extension": "jpg", "id" : "10160743390456729", @@ -212,7 +211,7 @@ __tests__ = ( "#count" : 1, "caption" : "", - "date" : datetime.datetime(2014, 5, 3, 0, 44, 47), + "date" : "dt:2014-05-03 00:44:47", "filename" : str, "extension": "png", "id" : "10152716011076729", @@ -272,7 +271,7 @@ __tests__ = ( "#class" : facebook.FacebookVideoExtractor, "#count" : 1, - "date" : datetime.datetime(2024, 4, 19, 17, 25, 48), + "date" : "dt:2024-04-19 17:25:48", "filename" : str, "id" : "1165557851291824", "url" : str, diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py index 6da50035..65aaafcf 100644 --- a/test/results/hentaifoundry.py +++ b/test/results/hentaifoundry.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import hentaifoundry -import datetime __tests__ = ( @@ -190,7 +189,7 @@ Sorry for the bad quality, I made it on after effect because Flash works like sh "author" : "SnowWolf35", "chapters" : int, "comments" : int, - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "index" : int, "rating" : int, diff --git a/test/results/horne.py b/test/results/horne.py index 6d2a0b49..418e656b 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import nijie -import datetime __tests__ = ( @@ -29,7 +28,7 @@ __tests__ = ( "artist_id" : 58000, "artist_name": "のえるわ", - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "image_id" : int, "num" : int, diff --git a/test/results/imgur.py b/test/results/imgur.py index bd99245f..8d6cf3ae 100644 --- a/test/results/imgur.py +++ b/test/results/imgur.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import imgur from gallery_dl import exception -import datetime __tests__ = ( @@ -215,7 +214,7 @@ __tests__ = ( }, "account_id" : 0, "count" : 19, - "date" : datetime.datetime, + "date" : "type:datetime", "description": "", "ext" : "jpg", "has_sound" : False, diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py index 4e05da7c..78885f59 100644 --- a/test/results/inkbunny.py +++ b/test/results/inkbunny.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import inkbunny -import datetime __tests__ = ( @@ -16,7 +15,7 @@ __tests__ = ( "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", "#range" : "20-50", - "date" : datetime.datetime, + "date" : "type:datetime", "deleted" : bool, "file_id" : r"re:[0-9]+", "filename" : r"re:[0-9]+_soina_\w+", diff --git a/test/results/luscious.py b/test/results/luscious.py index 597b5699..32626485 100644 --- a/test/results/luscious.py +++ b/test/results/luscious.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import luscious -import datetime from gallery_dl import exception @@ -49,7 +48,7 @@ __tests__ = ( "aspect_ratio" : r"re:\d+:\d+", "category" : "luscious", "created" : int, - "date" : datetime.datetime, + "date" : "type:datetime", "height" : int, "id" : int, "is_animated" : False, diff --git a/test/results/mangadex.py b/test/results/mangadex.py index 5309db13..5fdd7150 100644 --- a/test/results/mangadex.py +++ b/test/results/mangadex.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import mangadex from gallery_dl import exception -import datetime __tests__ = ( @@ -132,7 +131,7 @@ __tests__ = ( "chapter" : 0, "chapter_minor": "", "chapter_id" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "lang" : "iso:lang", "artist" : ["Arakawa Hiromu"], "author" : ["Arakawa Hiromu"], diff --git a/test/results/mangafox.py b/test/results/mangafox.py index dc0cc9a0..04651496 100644 --- a/test/results/mangafox.py +++ b/test/results/mangafox.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import mangafox -import datetime __tests__ = ( @@ -40,7 +39,7 @@ __tests__ = ( "chapter" : int, "chapter_minor" : r"re:^(\.\d+)?$", "chapter_string": r"re:(v\d+/)?c\d+", - "date" : datetime.datetime, + "date" : "type:datetime", "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", "lang" : "en", "language" : "English", diff --git a/test/results/mangapark.py b/test/results/mangapark.py index b40ab3fc..2ad5d426 100644 --- a/test/results/mangapark.py +++ b/test/results/mangapark.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import mangapark -import datetime __tests__ = ( @@ -115,7 +114,7 @@ __tests__ = ( "chapter" : int, "chapter_id" : r"re:\d+", "chapter_minor": str, - "date" : datetime.datetime, + "date" : "type:datetime", "lang" : "en", "language" : "English", "manga_id" : 114972, diff --git a/test/results/nijie.py b/test/results/nijie.py index 4e06f871..b5a53fb1 100644 --- a/test/results/nijie.py +++ b/test/results/nijie.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import nijie -import datetime from gallery_dl import exception @@ -32,7 +31,7 @@ __tests__ = ( "artist_id" : 44, "artist_name": "ED", "count" : 1, - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "extension" : "jpg", "filename" : str, diff --git a/test/results/patreon.py b/test/results/patreon.py index f2aeee8b..426981de 100644 --- a/test/results/patreon.py +++ b/test/results/patreon.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import patreon -import datetime from gallery_dl import exception @@ -21,7 +20,7 @@ __tests__ = ( "comment_count": int, "content" : str, "creator" : dict, - "date" : datetime.datetime, + "date" : "type:datetime", "id" : int, "images" : list, "like_count" : int, diff --git a/test/results/photovogue.py b/test/results/photovogue.py index 6898f5d9..187c374a 100644 --- a/test/results/photovogue.py +++ b/test/results/photovogue.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import photovogue -import datetime __tests__ = ( @@ -21,7 +20,7 @@ __tests__ = ( "#class" : photovogue.PhotovogueUserExtractor, "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - "date" : datetime.datetime, + "date" : "type:datetime", "favorite_count" : int, "favorited" : list, "id" : int, diff --git a/test/results/picarto.py b/test/results/picarto.py index 07456260..5d2af21f 100644 --- a/test/results/picarto.py +++ b/test/results/picarto.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import picarto -import datetime __tests__ = ( @@ -16,7 +15,7 @@ __tests__ = ( "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", "#count" : ">= 7", - "date": datetime.datetime, + "date": "type:datetime", }, ) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py index 514697be..fc583614 100644 --- a/test/results/pillowfort.py +++ b/test/results/pillowfort.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import pillowfort -import datetime __tests__ = ( @@ -24,7 +23,7 @@ __tests__ = ( "content" : str, "count" : 4, "created_at" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "deleted" : None, "deleted_at" : None, "deleted_by_mod" : None, diff --git a/test/results/pixeldrain.py b/test/results/pixeldrain.py index 1afc49b9..563634a0 100644 --- a/test/results/pixeldrain.py +++ b/test/results/pixeldrain.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import pixeldrain -import datetime __tests__ = ( { @@ -81,7 +80,7 @@ __tests__ = ( "success" : True, "title" : "アルバム", }, - "date" : datetime.datetime, + "date" : "type:datetime", "description": "", "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", "hash_sha256": r"re:\w{64}", diff --git a/test/results/pixiv.py b/test/results/pixiv.py index 19dbb601..0e763f62 100644 --- a/test/results/pixiv.py +++ b/test/results/pixiv.py @@ -668,6 +668,8 @@ __tests__ = ( "#class" : pixiv.PixivSketchExtractor, "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", "#count" : ">= 35", + + "date": "type:datetime", }, ) diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py index f9f8a62c..a98e22a1 100644 --- a/test/results/subscribestar.py +++ b/test/results/subscribestar.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import subscribestar -import datetime __tests__ = ( @@ -20,7 +19,7 @@ __tests__ = ( "author_name": "subscribestar", "author_nick": "SubscribeStar", "content" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "id" : int, "num" : int, "post_id" : int, @@ -38,7 +37,7 @@ __tests__ = ( "#options" : {"metadata": True}, "#range" : "1", - "date": datetime.datetime, + "date": "type:datetime", }, { diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py index 21d13ee1..46d61954 100644 --- a/test/results/toyhouse.py +++ b/test/results/toyhouse.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import toyhouse -import datetime __tests__ = ( @@ -19,7 +18,7 @@ __tests__ = ( "artists" : list, "characters": list, - "date" : datetime.datetime, + "date" : "type:datetime", "hash" : r"re:\w+", "id" : r"re:\d+", "url" : str, diff --git a/test/test_dt.py b/test/test_dt.py new file mode 100644 index 00000000..02e3ac23 --- /dev/null +++ b/test/test_dt.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import unittest + +import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from gallery_dl import dt # noqa E402 + + +class TestDatetime(unittest.TestCase): + + def test_convert(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + self.assertIs(f(d), d) + + _assert(d , d) + _assert(1262304000 , d) + _assert(1262304000.0 , d) + _assert(1262304000.123, d) + _assert("1262304000" , d) + + _assert("2010-01-01" , d) + _assert("2010-01-01 00:00:00" , d) + _assert("2010-01-01T00:00:00" , d) + _assert("2010-01-01T00:00:00.123456" , d) + _assert("2009-12-31T19:00:00-05:00" , d) + _assert("2009-12-31T19:00:00.123456-05:00", d) + _assert("2010-01-01T00:00:00Z" , d) + _assert("2010-01-01T00:00:00.123456Z" , d) + _assert("2009-12-31T19:00:00-0500" , d) + _assert("2009-12-31T19:00:00.123456-0500" , d) + + _assert(0 , dt.NONE) + _assert("" , dt.NONE) + _assert("foo", dt.NONE) + _assert(None , dt.NONE) + _assert(() , dt.NONE) + _assert([] , dt.NONE) + _assert({} , dt.NONE) + _assert((1, 2, 3), dt.NONE) + + @unittest.skipIf(sys.hexversion < 0x30b0000, + "extended fromisoformat timezones") + def test_convert_tz(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + _assert("2009-12-31T19:00:00-05" , d) + _assert("2009-12-31T19:00:00.123456-05" , d) + + def test_to_timestamp(self, f=dt.to_ts): + self.assertEqual(f(dt.EPOCH), 0.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), + 1262304000.128000) + with self.assertRaises(TypeError): + f(None) + + def test_to_timestamp_string(self, f=dt.to_ts_string): + self.assertEqual(f(dt.EPOCH), "0") + self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") + self.assertEqual(f(None), "") + + def test_from_timestamp(self, f=dt.from_ts): + self.assertEqual(f(0.0), dt.EPOCH) + self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) + self.assertEqual(f(1262304000.128000).replace(microsecond=0), + datetime.datetime(2010, 1, 1, 0, 0, 0)) + + def test_now(self, f=dt.now): + self.assertIsInstance(f(), datetime.datetime) + + def test_parse_timestamp(self, f=dt.parse_ts): + null = dt.from_ts(0) + value = dt.from_ts(1555816235) + + self.assertEqual(f(0) , null) + self.assertEqual(f("0") , null) + self.assertEqual(f(1555816235) , value) + self.assertEqual(f("1555816235"), value) + + for value in ((), [], {}, None, ""): + self.assertEqual(f(value), dt.NONE) + self.assertEqual(f(value, "foo"), "foo") + + def test_parse(self, f=dt.parse): + self.assertEqual( + f("1970.01.01", "%Y.%m.%d"), + dt.EPOCH, + ) + self.assertEqual( + f("May 7, 2019 9:33 am", "%B %d, %Y %I:%M %p"), + datetime.datetime(2019, 5, 7, 9, 33, 0), + ) + self.assertEqual( + f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value, "%Y"), dt.NONE) + + def test_parse_iso(self, f=dt.parse_iso): + self.assertEqual( + f("1970-01-01T00:00:00+00:00"), + dt.from_ts(0), + ) + self.assertEqual( + f("2019-05-07T21:25:02+09:00"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07T12:25:02Z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07 21:25:02"), + datetime.datetime(2019, 5, 7, 21, 25, 2), + ) + self.assertEqual( + f("1970-01-01"), + dt.EPOCH, + ) + self.assertEqual( + f("1970.01.01"), + dt.NONE, + ) + self.assertEqual( + f("1970-01-01T00:00:00+0000"), + dt.EPOCH, + ) + self.assertEqual( + f("2019-05-07T21:25:02.753+0900"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value), dt.NONE) + + def test_none(self): + self.assertFalse(dt.NONE) + self.assertIsInstance(dt.NONE, dt.datetime) + self.assertEqual(str(dt.NONE), "[Invalid DateTime]") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_extractor.py b/test/test_extractor.py index a623e1dc..cc06c47c 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -14,10 +14,9 @@ from unittest.mock import patch import time import string -from datetime import datetime, timedelta sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util # noqa E402 +from gallery_dl import extractor, util, dt # noqa E402 from gallery_dl.extractor import mastodon # noqa E402 from gallery_dl.extractor.common import Extractor, Message # noqa E402 from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 @@ -233,8 +232,8 @@ class TestExtractorWait(unittest.TestCase): def test_wait_until_datetime(self): extr = extractor.find("generic:https://example.org/") - until = util.datetime_utcnow() + timedelta(seconds=5) - until_local = datetime.now() + timedelta(seconds=5) + until = dt.now() + dt.timedelta(seconds=5) + until_local = dt.datetime.now() + dt.timedelta(seconds=5) if not until.microsecond: until = until.replace(microsecond=until_local.microsecond) @@ -251,8 +250,8 @@ class TestExtractorWait(unittest.TestCase): self._assert_isotime(calls[0][1][1], until_local) def _assert_isotime(self, output, until): - if not isinstance(until, datetime): - until = datetime.fromtimestamp(until) + if not isinstance(until, dt.datetime): + until = dt.datetime.fromtimestamp(until) o = self._isotime_to_seconds(output) u = self._isotime_to_seconds(until.time().isoformat()[:8]) self.assertLessEqual(o-u, 1.0) diff --git a/test/test_formatter.py b/test/test_formatter.py index 01e3a88e..f08ae49c 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -15,7 +15,7 @@ import datetime import tempfile sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import formatter, text, util, config # noqa E402 +from gallery_dl import formatter, text, dt, util, config # noqa E402 try: import jinja2 @@ -154,7 +154,7 @@ class TestFormatter(unittest.TestCase): self._run_test("{t}" , self.kwdict["t"] , None, int) self._run_test("{t}" , self.kwdict["t"] , None, util.identity) self._run_test("{dt}", self.kwdict["dt"], None, util.identity) - self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime) + self._run_test("{ds}", self.kwdict["dt"], None, dt.parse_iso) self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], None, util.identity) @@ -271,8 +271,8 @@ class TestFormatter(unittest.TestCase): def test_specifier_datetime(self): self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00") - self._run_test("{ds:D%Y}", "2010-01-01T01:00:00+01:00") - self._run_test("{l:D%Y}", "None") + self._run_test("{ds:D%Y}", "[Invalid DateTime]") + self._run_test("{l2:D%Y}", "[Invalid DateTime]") def test_specifier_offset(self): self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00") diff --git a/test/test_text.py b/test/test_text.py index 019a2069..7b9a4247 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -11,8 +11,6 @@ import os import sys import unittest -import datetime - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import text, util # noqa E402 @@ -537,51 +535,6 @@ class TestText(unittest.TestCase): self.assertEqual(f({"ä&": "あと", "#": "?"}), "%C3%A4%26=%E3%81%82%E3%81%A8&%23=%3F") - def test_parse_timestamp(self, f=text.parse_timestamp): - null = util.datetime_utcfromtimestamp(0) - value = util.datetime_utcfromtimestamp(1555816235) - - self.assertEqual(f(0) , null) - self.assertEqual(f("0") , null) - self.assertEqual(f(1555816235) , value) - self.assertEqual(f("1555816235"), value) - - for value in INVALID_ALT: - self.assertEqual(f(value), None) - self.assertEqual(f(value, "foo"), "foo") - - def test_parse_datetime(self, f=text.parse_datetime): - null = util.datetime_utcfromtimestamp(0) - - self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - self.assertEqual(f("1970.01.01", "%Y.%m.%d") , null) - - self.assertEqual( - f("2019-05-07T21:25:02+09:00"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02+0900"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02", "%Y-%m-%dT%H:%M:%S", utcoffset=9), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07 21:25:02"), - "2019-05-07 21:25:02", - ) - - for value in INVALID: - self.assertEqual(f(value), None) - self.assertEqual(f("1970.01.01"), "1970.01.01") - if __name__ == "__main__": unittest.main() diff --git a/test/test_util.py b/test/test_util.py index bfaab01b..7f278a93 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -406,89 +406,6 @@ def hash(value): self.assertEqual(expr(value), result) -class TestDatetime(unittest.TestCase): - - def test_to_datetime(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - self.assertIs(f(dt), dt) - - _assert(dt , dt) - _assert(1262304000 , dt) - _assert(1262304000.0 , dt) - _assert(1262304000.123, dt) - _assert("1262304000" , dt) - - _assert("2010-01-01" , dt) - _assert("2010-01-01 00:00:00" , dt) - _assert("2010-01-01T00:00:00" , dt) - _assert("2010-01-01T00:00:00.123456" , dt) - _assert("2009-12-31T19:00:00-05:00" , dt) - _assert("2009-12-31T19:00:00.123456-05:00", dt) - _assert("2010-01-01T00:00:00Z" , dt) - _assert("2010-01-01T00:00:00.123456Z" , dt) - - _assert(0 , util.EPOCH) - _assert("" , util.EPOCH) - _assert("foo", util.EPOCH) - _assert(None , util.EPOCH) - _assert(() , util.EPOCH) - _assert([] , util.EPOCH) - _assert({} , util.EPOCH) - _assert((1, 2, 3), util.EPOCH) - - @unittest.skipIf(sys.hexversion < 0x30b0000, - "extended fromisoformat timezones") - def test_to_datetime_tz(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - - _assert("2009-12-31T19:00:00-05" , dt) - _assert("2009-12-31T19:00:00-0500" , dt) - _assert("2009-12-31T19:00:00.123456-05" , dt) - _assert("2009-12-31T19:00:00.123456-0500" , dt) - - def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp): - self.assertEqual(f(util.EPOCH), 0.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), - 1262304000.128000) - with self.assertRaises(TypeError): - f(None) - - def test_datetime_to_timestamp_string( - self, f=util.datetime_to_timestamp_string): - self.assertEqual(f(util.EPOCH), "0") - self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") - self.assertEqual(f(None), "") - - def test_datetime_from_timestamp( - self, f=util.datetime_from_timestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - self.assertEqual(f(1262304000.128000).replace(microsecond=0), - datetime.datetime(2010, 1, 1, 0, 0, 0)) - - def test_datetime_utcfromtimestamp( - self, f=util.datetime_utcfromtimestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - - def test_datetime_utcnow( - self, f=util.datetime_utcnow): - self.assertIsInstance(f(), datetime.datetime) - - class TestOther(unittest.TestCase): def test_bencode(self):