From 4845d39679902a3d7fb03056b2e648222ffb2a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 12 Oct 2025 21:36:18 +0200 Subject: [PATCH 01/13] [dt] create separate module for datetime utilities --- gallery_dl/dt.py | 108 ++++++++++++++++++++++++++++++++++++++++++ gallery_dl/version.py | 2 +- 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 gallery_dl/dt.py diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py new file mode 100644 index 00000000..2b99a291 --- /dev/null +++ b/gallery_dl/dt.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Date/Time utilities""" + +import sys +import time +from datetime import datetime, date, timedelta, timezone # noqa F401 + +EPOCH = datetime(1970, 1, 1) +SECOND = timedelta(0, 1) + + +def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0): + """Create a datetime object by parsing 'date_string'""" + try: + d = datetime.strptime(date_string, format) + o = d.utcoffset() + if o is not None: + # convert to naive UTC + d = d.replace(tzinfo=None, microsecond=0) - o + else: + if d.microsecond: + d = d.replace(microsecond=0) + if utcoffset: + # apply manual UTC offset + d += timedelta(0, utcoffset * -3600) + return d + except (TypeError, IndexError, KeyError): + return None + except (ValueError, OverflowError): + return date_string + + +def to_datetime(value): + """Convert 'value' to a datetime object""" + if not value: + return EPOCH + + if isinstance(value, datetime): + return value + + if isinstance(value, str): + try: + if value[-1] == "Z": + # compat for Python < 3.11 + value = value[:-1] + dt = datetime.fromisoformat(value) + if dt.tzinfo is None: + if dt.microsecond: + dt = dt.replace(microsecond=0) + else: + # convert to naive UTC + dt = dt.astimezone(timezone.utc).replace( + microsecond=0, tzinfo=None) + return dt + except Exception: + pass + + return parse_timestamp(value, EPOCH) + + +def datetime_to_timestamp(dt): + """Convert naive UTC datetime to Unix timestamp""" + return (dt - EPOCH) / SECOND + + +def datetime_to_timestamp_string(dt): + """Convert naive UTC datetime to Unix timestamp string""" + try: + return str((dt - EPOCH) // SECOND) + except Exception: + return "" + + +if sys.hexversion < 0x30c0000: + # Python <= 3.11 + datetime_utcfromtimestamp = datetime.utcfromtimestamp + datetime_utcnow = datetime.utcnow + datetime_from_timestamp = datetime_utcfromtimestamp + + def parse_timestamp(ts, default=None): + """Create a datetime object from a Unix timestamp""" + try: + return datetime_utcfromtimestamp(int(ts)) + except Exception: + return default +else: + # Python >= 3.12 + def datetime_from_timestamp(ts=None): + """Convert Unix timestamp to naive UTC datetime""" + Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) + return datetime(Y, m, d, H, M, S) + + def parse_timestamp(ts, default=None): + """Create a datetime object from a Unix timestamp""" + try: + return datetime_from_timestamp(int(ts)) + except Exception: + return default + + datetime_utcfromtimestamp = datetime_from_timestamp + datetime_utcnow = datetime_from_timestamp diff --git a/gallery_dl/version.py b/gallery_dl/version.py index bc70f747..fba1b71c 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,5 +6,5 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.30.10" +__version__ = "1.30.11-dev" __variant__ = None From dc00af8e907673e0399bceedbbaa67c855060a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 14 Oct 2025 21:37:08 +0200 Subject: [PATCH 02/13] [dt] rename & reorganize functions --- gallery_dl/dt.py | 133 +++++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index 2b99a291..0008cf57 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -16,29 +16,18 @@ EPOCH = datetime(1970, 1, 1) SECOND = timedelta(0, 1) -def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0): - """Create a datetime object by parsing 'date_string'""" - try: - d = datetime.strptime(date_string, format) - o = d.utcoffset() - if o is not None: - # convert to naive UTC - d = d.replace(tzinfo=None, microsecond=0) - o - else: - if d.microsecond: - d = d.replace(microsecond=0) - if utcoffset: - # apply manual UTC offset - d += timedelta(0, utcoffset * -3600) - return d - except (TypeError, IndexError, KeyError): - return None - except (ValueError, OverflowError): - return date_string +def normalize(dt): + # if (o := dt.utcoffset()) is not None: + # return dt.replace(tzinfo=None, microsecond=0) - o + if dt.tzinfo is not None: + return dt.astimezone(timezone.utc).replace(tzinfo=None, microsecond=0) + if dt.microsecond: + return dt.replace(microsecond=0) + return dt -def to_datetime(value): - """Convert 'value' to a datetime object""" +def convert(value): + """Convert 'value' to a naive UTC datetime object""" if not value: return EPOCH @@ -50,59 +39,77 @@ def to_datetime(value): if value[-1] == "Z": # compat for Python < 3.11 value = value[:-1] - dt = datetime.fromisoformat(value) - if dt.tzinfo is None: - if dt.microsecond: - dt = dt.replace(microsecond=0) - else: - # convert to naive UTC - dt = dt.astimezone(timezone.utc).replace( - microsecond=0, tzinfo=None) - return dt + return normalize(datetime.fromisoformat(value)) except Exception: pass - return parse_timestamp(value, EPOCH) + return parse_ts(value) -def datetime_to_timestamp(dt): +def parse(dt_string, format): + """Parse 'dt_string' according to 'format'""" + try: + return normalize(datetime.strptime(dt_string, format)) + except Exception: + return EPOCH + + +if sys.hexversion < 0x30c0000: + # Python <= 3.11 + def parse_iso(dt_string): + """Parse 'dt_string' as ISO 8601 value""" + try: + if dt_string[-1] == "Z": + # compat for Python < 3.11 + dt_string = dt_string[:-1] + return normalize(datetime.fromisoformat(dt_string)) + except Exception: + return EPOCH + + def parse_compat(dt_string, format): + """Parse 'dt_string' as ISO 8601 value using 'format'""" + return parse(dt_string, format) + + from_ts = datetime.utcfromtimestamp + now = datetime.utcnow + +else: + # Python >= 3.12 + def parse_iso(dt_string): + """Parse 'dt_string' as ISO 8601 value""" + try: + return normalize(datetime.fromisoformat(dt_string)) + except Exception: + return EPOCH + + def parse_compat(dt_string, format): + """Parse 'dt_string' as ISO 8601 value""" + return parse_iso(dt_string) + + def from_ts(ts=None): + """Convert Unix timestamp to naive UTC datetime""" + Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) + return datetime(Y, m, d, H, M, S) + + now = from_ts + + +def parse_ts(ts, default=EPOCH): + """Create a datetime object from a Unix timestamp""" + try: + return from_ts(int(ts)) + except Exception: + return default + + +def to_ts(dt): """Convert naive UTC datetime to Unix timestamp""" return (dt - EPOCH) / SECOND -def datetime_to_timestamp_string(dt): +def to_ts_string(dt): """Convert naive UTC datetime to Unix timestamp string""" try: return str((dt - EPOCH) // SECOND) except Exception: return "" - - -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - datetime_utcfromtimestamp = datetime.utcfromtimestamp - datetime_utcnow = datetime.utcnow - datetime_from_timestamp = datetime_utcfromtimestamp - - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - return datetime_utcfromtimestamp(int(ts)) - except Exception: - return default -else: - # Python >= 3.12 - def datetime_from_timestamp(ts=None): - """Convert Unix timestamp to naive UTC datetime""" - Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) - return datetime(Y, m, d, H, M, S) - - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - return datetime_from_timestamp(int(ts)) - except Exception: - return default - - datetime_utcfromtimestamp = datetime_from_timestamp - datetime_utcnow = datetime_from_timestamp From d0f5d4e0d34944f460a4e335655b4b827def59e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 14 Oct 2025 22:18:34 +0200 Subject: [PATCH 03/13] [tests/dt] add tests --- test/test_dt.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++ test/test_text.py | 47 -------------- test/test_util.py | 83 ------------------------ 3 files changed, 158 insertions(+), 130 deletions(-) create mode 100644 test/test_dt.py diff --git a/test/test_dt.py b/test/test_dt.py new file mode 100644 index 00000000..b9cc4d2c --- /dev/null +++ b/test/test_dt.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import unittest + +import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from gallery_dl import dt # noqa E402 + + +class TestDatetime(unittest.TestCase): + + def test_convert(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + self.assertIs(f(d), d) + + _assert(d , d) + _assert(1262304000 , d) + _assert(1262304000.0 , d) + _assert(1262304000.123, d) + _assert("1262304000" , d) + + _assert("2010-01-01" , d) + _assert("2010-01-01 00:00:00" , d) + _assert("2010-01-01T00:00:00" , d) + _assert("2010-01-01T00:00:00.123456" , d) + _assert("2009-12-31T19:00:00-05:00" , d) + _assert("2009-12-31T19:00:00.123456-05:00", d) + _assert("2010-01-01T00:00:00Z" , d) + _assert("2010-01-01T00:00:00.123456Z" , d) + + _assert(0 , dt.EPOCH) + _assert("" , dt.EPOCH) + _assert("foo", dt.EPOCH) + _assert(None , dt.EPOCH) + _assert(() , dt.EPOCH) + _assert([] , dt.EPOCH) + _assert({} , dt.EPOCH) + _assert((1, 2, 3), dt.EPOCH) + + @unittest.skipIf(sys.hexversion < 0x30b0000, + "extended fromisoformat timezones") + def test_convert_tz(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + + _assert("2009-12-31T19:00:00-05" , d) + _assert("2009-12-31T19:00:00-0500" , d) + _assert("2009-12-31T19:00:00.123456-05" , d) + _assert("2009-12-31T19:00:00.123456-0500" , d) + + def test_to_timestamp(self, f=dt.to_ts): + self.assertEqual(f(dt.EPOCH), 0.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), + 1262304000.128000) + with self.assertRaises(TypeError): + f(None) + + def test_to_timestamp_string(self, f=dt.to_ts_string): + self.assertEqual(f(dt.EPOCH), "0") + self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") + self.assertEqual(f(None), "") + + def test_from_timestamp(self, f=dt.from_ts): + self.assertEqual(f(0.0), dt.EPOCH) + self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) + self.assertEqual(f(1262304000.128000).replace(microsecond=0), + datetime.datetime(2010, 1, 1, 0, 0, 0)) + + def test_now(self, f=dt.now): + self.assertIsInstance(f(), datetime.datetime) + + def test_parse_timestamp(self, f=dt.parse_ts): + null = dt.from_ts(0) + value = dt.from_ts(1555816235) + + self.assertEqual(f(0) , null) + self.assertEqual(f("0") , null) + self.assertEqual(f(1555816235) , value) + self.assertEqual(f("1555816235"), value) + + for value in ((), [], {}, None, ""): + self.assertEqual(f(value), dt.EPOCH) + self.assertEqual(f(value, "foo"), "foo") + + def test_parse(self, f=dt.parse): + self.assertEqual( + f("1970.01.01", "%Y.%m.%d"), + dt.EPOCH, + ) + self.assertEqual( + f("May 7, 2019 9:33 am", "%B %d, %Y %I:%M %p"), + datetime.datetime(2019, 5, 7, 9, 33, 0), + ) + self.assertEqual( + f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value, "%Y"), dt.EPOCH) + + def test_parse_iso(self, f=dt.parse_iso): + null = dt.from_ts(0) + + self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) + self.assertEqual(f("1970-01-01T00:00:00+0000") , null) + + self.assertEqual( + f("2019-05-07T21:25:02+09:00"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07T12:25:02Z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07 21:25:02"), + datetime.datetime(2019, 5, 7, 21, 25, 2), + ) + self.assertEqual( + f("1970.01.01"), + dt.EPOCH, + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value), dt.EPOCH) + + def test_parse_compat(self, f=dt.parse_compat): + self.assertEqual( + f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_text.py b/test/test_text.py index 0e187d75..4ff34a3b 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -11,8 +11,6 @@ import os import sys import unittest -import datetime - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import text, util # noqa E402 @@ -519,51 +517,6 @@ class TestText(unittest.TestCase): self.assertEqual(f({"ä&": "あと", "#": "?"}), "%C3%A4%26=%E3%81%82%E3%81%A8&%23=%3F") - def test_parse_timestamp(self, f=text.parse_timestamp): - null = util.datetime_utcfromtimestamp(0) - value = util.datetime_utcfromtimestamp(1555816235) - - self.assertEqual(f(0) , null) - self.assertEqual(f("0") , null) - self.assertEqual(f(1555816235) , value) - self.assertEqual(f("1555816235"), value) - - for value in INVALID_ALT: - self.assertEqual(f(value), None) - self.assertEqual(f(value, "foo"), "foo") - - def test_parse_datetime(self, f=text.parse_datetime): - null = util.datetime_utcfromtimestamp(0) - - self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - self.assertEqual(f("1970.01.01", "%Y.%m.%d") , null) - - self.assertEqual( - f("2019-05-07T21:25:02+09:00"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02+0900"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02", "%Y-%m-%dT%H:%M:%S", utcoffset=9), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07 21:25:02"), - "2019-05-07 21:25:02", - ) - - for value in INVALID: - self.assertEqual(f(value), None) - self.assertEqual(f("1970.01.01"), "1970.01.01") - if __name__ == "__main__": unittest.main() diff --git a/test/test_util.py b/test/test_util.py index bfaab01b..7f278a93 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -406,89 +406,6 @@ def hash(value): self.assertEqual(expr(value), result) -class TestDatetime(unittest.TestCase): - - def test_to_datetime(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - self.assertIs(f(dt), dt) - - _assert(dt , dt) - _assert(1262304000 , dt) - _assert(1262304000.0 , dt) - _assert(1262304000.123, dt) - _assert("1262304000" , dt) - - _assert("2010-01-01" , dt) - _assert("2010-01-01 00:00:00" , dt) - _assert("2010-01-01T00:00:00" , dt) - _assert("2010-01-01T00:00:00.123456" , dt) - _assert("2009-12-31T19:00:00-05:00" , dt) - _assert("2009-12-31T19:00:00.123456-05:00", dt) - _assert("2010-01-01T00:00:00Z" , dt) - _assert("2010-01-01T00:00:00.123456Z" , dt) - - _assert(0 , util.EPOCH) - _assert("" , util.EPOCH) - _assert("foo", util.EPOCH) - _assert(None , util.EPOCH) - _assert(() , util.EPOCH) - _assert([] , util.EPOCH) - _assert({} , util.EPOCH) - _assert((1, 2, 3), util.EPOCH) - - @unittest.skipIf(sys.hexversion < 0x30b0000, - "extended fromisoformat timezones") - def test_to_datetime_tz(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - - _assert("2009-12-31T19:00:00-05" , dt) - _assert("2009-12-31T19:00:00-0500" , dt) - _assert("2009-12-31T19:00:00.123456-05" , dt) - _assert("2009-12-31T19:00:00.123456-0500" , dt) - - def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp): - self.assertEqual(f(util.EPOCH), 0.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), - 1262304000.128000) - with self.assertRaises(TypeError): - f(None) - - def test_datetime_to_timestamp_string( - self, f=util.datetime_to_timestamp_string): - self.assertEqual(f(util.EPOCH), "0") - self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") - self.assertEqual(f(None), "") - - def test_datetime_from_timestamp( - self, f=util.datetime_from_timestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - self.assertEqual(f(1262304000.128000).replace(microsecond=0), - datetime.datetime(2010, 1, 1, 0, 0, 0)) - - def test_datetime_utcfromtimestamp( - self, f=util.datetime_utcfromtimestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - - def test_datetime_utcnow( - self, f=util.datetime_utcnow): - self.assertIsInstance(f(), datetime.datetime) - - class TestOther(unittest.TestCase): def test_bencode(self): From 0eb3c8a99485233764a99440164456403452e62e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 15 Oct 2025 21:30:38 +0200 Subject: [PATCH 04/13] [dt] replace 'util' datetime functions --- gallery_dl/extractor/4archive.py | 4 +- gallery_dl/extractor/8chan.py | 7 ++- gallery_dl/extractor/common.py | 4 +- gallery_dl/extractor/deviantart.py | 6 +-- gallery_dl/extractor/motherless.py | 17 ++++--- gallery_dl/extractor/patreon.py | 5 +-- gallery_dl/extractor/pixiv.py | 27 +++++------ gallery_dl/formatter.py | 24 +++++----- gallery_dl/postprocessor/mtime.py | 7 ++- gallery_dl/util.py | 72 +++--------------------------- test/results/pixiv.py | 2 + test/test_extractor.py | 11 +++-- test/test_formatter.py | 4 +- 13 files changed, 62 insertions(+), 128 deletions(-) diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 4c434643..6a4d0c69 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -7,7 +7,7 @@ """Extractors for https://4archive.org/""" from .common import Extractor, Message -from .. import text, util +from .. import text, dt class _4archiveThreadExtractor(Extractor): @@ -37,7 +37,7 @@ class _4archiveThreadExtractor(Extractor): for post in posts: post.update(data) - post["time"] = int(util.datetime_to_timestamp(post["date"])) + post["time"] = int(dt.to_ts(post["date"])) yield Message.Directory, post if "url" in post: yield Message.Url, post["url"], text.nameext_from_url( diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 03850676..6524821b 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -9,9 +9,8 @@ """Extractors for https://8chan.moe/""" from .common import Extractor, Message -from .. import text, util +from .. import text, dt from ..cache import memcache -from datetime import timedelta import itertools BASE_PATTERN = r"(?:https?://)?8chan\.(moe|se|cc)" @@ -44,7 +43,7 @@ class _8chanExtractor(Extractor): def cookies_prepare(self): # fetch captcha cookies # (necessary to download without getting interrupted) - now = util.datetime_utcnow() + now = dt.now() url = self.root + "/captcha.js" params = {"d": now.strftime("%a %b %d %Y %H:%M:%S GMT+0000 (UTC)")} self.request(url, params=params).content @@ -57,7 +56,7 @@ class _8chanExtractor(Extractor): if cookie.domain.endswith(domain): cookie.expires = None if cookie.name == "captchaexpiration": - cookie.value = (now + timedelta(30, 300)).strftime( + cookie.value = (now + dt.timedelta(30, 300)).strftime( "%a, %d %b %Y %H:%M:%S GMT") return self.cookies diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 34e65c5c..83cb7a50 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -23,7 +23,7 @@ from datetime import datetime from xml.etree import ElementTree from requests.adapters import HTTPAdapter from .message import Message -from .. import config, output, text, util, cache, exception +from .. import config, output, text, util, dt, cache, exception urllib3 = requests.packages.urllib3 @@ -315,7 +315,7 @@ class Extractor(): elif until: if isinstance(until, datetime): # convert to UTC timestamp - until = util.datetime_to_timestamp(until) + until = dt.to_ts(until) else: until = float(until) seconds = until - now diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 39690da2..64b23594 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -9,7 +9,7 @@ """Extractors for https://www.deviantart.com/""" from .common import Extractor, Message, Dispatch -from .. import text, util, exception +from .. import text, util, dt, exception from ..cache import cache, memcache import collections import mimetypes @@ -1187,8 +1187,8 @@ class DeviantartStatusExtractor(DeviantartExtractor): deviation["username"] = deviation["author"]["username"] deviation["_username"] = deviation["username"].lower() - deviation["date"] = dt = text.parse_datetime(deviation["ts"]) - deviation["published_time"] = int(util.datetime_to_timestamp(dt)) + deviation["date"] = d = text.parse_datetime(deviation["ts"]) + deviation["published_time"] = int(dt.to_ts(d)) deviation["da_category"] = "Status" deviation["category_path"] = "status" diff --git a/gallery_dl/extractor/motherless.py b/gallery_dl/extractor/motherless.py index 48137ced..cadf8988 100644 --- a/gallery_dl/extractor/motherless.py +++ b/gallery_dl/extractor/motherless.py @@ -9,9 +9,8 @@ """Extractors for https://motherless.com/""" from .common import Extractor, Message -from .. import text, util, exception +from .. import text, dt, exception from ..cache import memcache -from datetime import timedelta BASE_PATTERN = r"(?:https?://)?motherless\.com" @@ -115,14 +114,14 @@ class MotherlessExtractor(Extractor): return data - def _parse_datetime(self, dt): - if " ago" not in dt: - return text.parse_datetime(dt, "%d %b %Y") + def _parse_datetime(self, dt_string): + if " ago" not in dt_string: + return dt.parse(dt_string, "%d %b %Y") - value = text.parse_int(dt[:-5]) - delta = timedelta(0, value*3600) if dt[-5] == "h" else timedelta(value) - return (util.datetime_utcnow() - delta).replace( - hour=0, minute=0, second=0) + value = text.parse_int(dt_string[:-5]) + delta = (dt.timedelta(0, value*3600) if dt_string[-5] == "h" else + dt.timedelta(value)) + return (dt.now() - delta).replace(hour=0, minute=0, second=0) @memcache(keyarg=2) def _extract_gallery_title(self, page, gallery_id): diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index cf1a6d61..552b3856 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -9,7 +9,7 @@ """Extractors for https://www.patreon.com/""" from .common import Extractor, Message -from .. import text, util, exception +from .. import text, util, dt, exception from ..cache import memcache import collections import itertools @@ -445,8 +445,7 @@ class PatreonUserExtractor(PatreonExtractor): def posts(self): if date_max := self._get_date_min_max(None, None)[1]: - self._cursor = cursor = \ - util.datetime_from_timestamp(date_max).isoformat() + self._cursor = cursor = dt.from_ts(date_max).isoformat() self._init_cursor = lambda: cursor url = self._build_url("stream", ( diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 6276a2ad..c4391f0f 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -9,9 +9,8 @@ """Extractors for https://www.pixiv.net/""" from .common import Extractor, Message, Dispatch -from .. import text, util, exception +from .. import text, util, dt, exception from ..cache import cache, memcache -from datetime import datetime, timedelta import itertools import hashlib @@ -96,7 +95,7 @@ class PixivExtractor(Extractor): if transform_tags: transform_tags(work) work["num"] = 0 - work["date"] = text.parse_datetime(work["create_date"]) + work["date"] = dt.parse_iso(work["create_date"]) work["rating"] = ratings.get(work["x_restrict"]) work["suffix"] = "" work.update(metadata) @@ -350,10 +349,10 @@ class PixivExtractor(Extractor): if fmt in urls: yield urls[fmt] - def _date_from_url(self, url, offset=timedelta(hours=9)): + def _date_from_url(self, url, offset=dt.timedelta(hours=9)): try: _, _, _, _, _, y, m, d, H, M, S, _ = url.split("/") - return datetime( + return dt.datetime( int(y), int(m), int(d), int(H), int(M), int(S)) - offset except Exception: return None @@ -712,8 +711,7 @@ class PixivRankingExtractor(PixivExtractor): self.log.warning("invalid date '%s'", date) date = None if not date: - now = util.datetime_utcnow() - date = (now - timedelta(days=1)).strftime("%Y-%m-%d") + date = (dt.now() - dt.timedelta(days=1)).strftime("%Y-%m-%d") self.date = date self.type = type = query.get("content") @@ -888,8 +886,7 @@ class PixivSketchExtractor(Extractor): for post in self.posts(): media = post["media"] post["post_id"] = post["id"] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = dt.parse_iso(post["created_at"]) util.delete_items(post, ("id", "media", "_links")) yield Message.Directory, post @@ -969,7 +966,7 @@ class PixivNovelExtractor(PixivExtractor): if transform_tags: transform_tags(novel) novel["num"] = 0 - novel["date"] = text.parse_datetime(novel["create_date"]) + novel["date"] = dt.parse_iso(novel["create_date"]) novel["rating"] = ratings.get(novel["x_restrict"]) novel["suffix"] = "" @@ -1151,7 +1148,7 @@ class PixivAppAPI(): "get_secure_url": "1", } - time = util.datetime_utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00") + time = dt.now().strftime("%Y-%m-%dT%H:%M:%S+00:00") headers = { "X-Client-Time": time, "X-Client-Hash": hashlib.md5( @@ -1326,11 +1323,11 @@ class PixivAppAPI(): sort = params["sort"] if sort == "date_desc": date_key = "end_date" - date_off = timedelta(days=1) + date_off = dt.timedelta(days=1) date_cmp = lambda lhs, rhs: lhs >= rhs # noqa E731 elif sort == "date_asc": date_key = "start_date" - date_off = timedelta(days=-1) + date_off = dt.timedelta(days=-1) date_cmp = lambda lhs, rhs: lhs <= rhs # noqa E731 else: date_key = None @@ -1357,8 +1354,8 @@ class PixivAppAPI(): if date_key and text.parse_int(params.get("offset")) >= 5000: date_last = data["illusts"][-1]["create_date"] - date_val = (text.parse_datetime( - date_last) + date_off).strftime("%Y-%m-%d") + date_val = (dt.parse_iso(date_last) + date_off).strftime( + "%Y-%m-%d") self.log.info("Reached 'offset' >= 5000; " "Updating '%s' to '%s'", date_key, date_val) params[date_key] = date_val diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 5246f663..39a5e4e2 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -15,7 +15,7 @@ import string import _string import datetime import operator -from . import text, util +from . import text, util, dt NONE = util.NONE @@ -68,8 +68,8 @@ class StringFormatter(): - "g": calls text.slugify() - "j": calls json.dumps - "t": calls str.strip - - "T": calls util.datetime_to_timestamp_string() - - "d": calls text.parse_timestamp + - "T": calls dt.to_ts_string() + - "d": calls dt.parse_ts() - "s": calls str() - "S": calls util.to_string() - "U": calls urllib.parse.unescape @@ -471,9 +471,9 @@ def _parse_datetime(format_spec, default): dt_format = dt_format[1:] fmt = _build_format_func(format_spec, default) - def dt(obj): - return fmt(text.parse_datetime(obj, dt_format)) - return dt + def dt_parse(obj): + return fmt(dt.parse(obj, dt_format)) + return dt_parse def _parse_offset(format_spec, default): @@ -482,9 +482,9 @@ def _parse_offset(format_spec, default): fmt = _build_format_func(format_spec, default) if not offset or offset == "local": - def off(dt): - local = time.localtime(util.datetime_to_timestamp(dt)) - return fmt(dt + datetime.timedelta(0, local.tm_gmtoff)) + def off(dt_utc): + local = time.localtime(dt.to_ts(dt_utc)) + return fmt(dt_utc + datetime.timedelta(0, local.tm_gmtoff)) else: hours, _, minutes = offset.partition(":") offset = 3600 * int(hours) @@ -569,9 +569,9 @@ _CONVERSIONS = { "t": str.strip, "n": len, "L": util.code_to_language, - "T": util.datetime_to_timestamp_string, - "d": text.parse_timestamp, - "D": util.to_datetime, + "T": dt.to_ts_string, + "d": dt.parse_ts, + "D": dt.convert, "U": text.unescape, "H": lambda s: text.unescape(text.remove_html(s)), "g": text.slugify, diff --git a/gallery_dl/postprocessor/mtime.py b/gallery_dl/postprocessor/mtime.py index b1269dd5..7d4796e7 100644 --- a/gallery_dl/postprocessor/mtime.py +++ b/gallery_dl/postprocessor/mtime.py @@ -9,8 +9,7 @@ """Use metadata as file modification time""" from .common import PostProcessor -from .. import text, util, formatter -from datetime import datetime +from .. import text, util, dt, formatter class MtimePP(PostProcessor): @@ -36,8 +35,8 @@ class MtimePP(PostProcessor): return pathfmt.kwdict["_mtime_meta"] = ( - util.datetime_to_timestamp(mtime) - if isinstance(mtime, datetime) else + dt.to_ts(mtime) + if isinstance(mtime, dt.datetime) else text.parse_int(mtime) ) diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 49c1ba8e..7fc3363c 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -16,7 +16,6 @@ import random import getpass import hashlib import binascii -import datetime import functools import itertools import subprocess @@ -24,7 +23,7 @@ import collections import urllib.parse from http.cookiejar import Cookie from email.utils import mktime_tz, parsedate_tz -from . import text, version, exception +from . import text, dt, version, exception def bencode(num, alphabet="0123456789"): @@ -228,63 +227,6 @@ def to_string(value): return str(value) -def to_datetime(value): - """Convert 'value' to a datetime object""" - if not value: - return EPOCH - - if isinstance(value, datetime.datetime): - return value - - if isinstance(value, str): - try: - if value[-1] == "Z": - # compat for Python < 3.11 - value = value[:-1] - dt = datetime.datetime.fromisoformat(value) - if dt.tzinfo is None: - if dt.microsecond: - dt = dt.replace(microsecond=0) - else: - # convert to naive UTC - dt = dt.astimezone(datetime.timezone.utc).replace( - microsecond=0, tzinfo=None) - return dt - except Exception: - pass - - return text.parse_timestamp(value, EPOCH) - - -def datetime_to_timestamp(dt): - """Convert naive UTC datetime to Unix timestamp""" - return (dt - EPOCH) / SECOND - - -def datetime_to_timestamp_string(dt): - """Convert naive UTC datetime to Unix timestamp string""" - try: - return str((dt - EPOCH) // SECOND) - except Exception: - return "" - - -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - datetime_utcfromtimestamp = datetime.datetime.utcfromtimestamp - datetime_utcnow = datetime.datetime.utcnow - datetime_from_timestamp = datetime_utcfromtimestamp -else: - # Python >= 3.12 - def datetime_from_timestamp(ts=None): - """Convert Unix timestamp to naive UTC datetime""" - Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) - return datetime.datetime(Y, m, d, H, M, S) - - datetime_utcfromtimestamp = datetime_from_timestamp - datetime_utcnow = datetime_from_timestamp - - def json_default(obj): if isinstance(obj, CustomNone): return None @@ -379,7 +321,7 @@ def extract_headers(response): text.nameext_from_url(name, data) if hlm := headers.get("last-modified"): - data["date"] = datetime.datetime(*parsedate_tz(hlm)[:6]) + data["date"] = dt.datetime(*parsedate_tz(hlm)[:6]) return data @@ -751,11 +693,11 @@ class Flags(): # 735506 == 739342 - 137 * 28 # v135.0 release of Chrome on 2025-04-01 has ordinal 739342 # 735562 == 739342 - 135 * 28 -# _ord_today = datetime.date.today().toordinal() +# _ord_today = dt.date.today().toordinal() # _ff_ver = (_ord_today - 735506) // 28 # _ch_ver = (_ord_today - 735562) // 28 -_ff_ver = (datetime.date.today().toordinal() - 735506) // 28 +_ff_ver = (dt.date.today().toordinal() - 735506) // 28 # _ch_ver = _ff_ver - 2 re = text.re @@ -763,8 +705,6 @@ re_compile = text.re_compile NONE = CustomNone() FLAGS = Flags() -EPOCH = datetime.datetime(1970, 1, 1) -SECOND = datetime.timedelta(0, 1) WINDOWS = (os.name == "nt") SENTINEL = object() EXECUTABLE = getattr(sys, "frozen", False) @@ -786,8 +726,8 @@ GLOBALS = { "contains" : contains, "parse_int": text.parse_int, "urlsplit" : urllib.parse.urlsplit, - "datetime" : datetime.datetime, - "timedelta": datetime.timedelta, + "datetime" : dt.datetime, + "timedelta": dt.timedelta, "abort" : raises(exception.StopExtraction), "error" : raises(exception.AbortExtraction), "terminate": raises(exception.TerminateExtraction), diff --git a/test/results/pixiv.py b/test/results/pixiv.py index 19dbb601..0e763f62 100644 --- a/test/results/pixiv.py +++ b/test/results/pixiv.py @@ -668,6 +668,8 @@ __tests__ = ( "#class" : pixiv.PixivSketchExtractor, "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", "#count" : ">= 35", + + "date": "type:datetime", }, ) diff --git a/test/test_extractor.py b/test/test_extractor.py index a623e1dc..cc06c47c 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -14,10 +14,9 @@ from unittest.mock import patch import time import string -from datetime import datetime, timedelta sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util # noqa E402 +from gallery_dl import extractor, util, dt # noqa E402 from gallery_dl.extractor import mastodon # noqa E402 from gallery_dl.extractor.common import Extractor, Message # noqa E402 from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 @@ -233,8 +232,8 @@ class TestExtractorWait(unittest.TestCase): def test_wait_until_datetime(self): extr = extractor.find("generic:https://example.org/") - until = util.datetime_utcnow() + timedelta(seconds=5) - until_local = datetime.now() + timedelta(seconds=5) + until = dt.now() + dt.timedelta(seconds=5) + until_local = dt.datetime.now() + dt.timedelta(seconds=5) if not until.microsecond: until = until.replace(microsecond=until_local.microsecond) @@ -251,8 +250,8 @@ class TestExtractorWait(unittest.TestCase): self._assert_isotime(calls[0][1][1], until_local) def _assert_isotime(self, output, until): - if not isinstance(until, datetime): - until = datetime.fromtimestamp(until) + if not isinstance(until, dt.datetime): + until = dt.datetime.fromtimestamp(until) o = self._isotime_to_seconds(output) u = self._isotime_to_seconds(until.time().isoformat()[:8]) self.assertLessEqual(o-u, 1.0) diff --git a/test/test_formatter.py b/test/test_formatter.py index 01e3a88e..7a286190 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -271,8 +271,8 @@ class TestFormatter(unittest.TestCase): def test_specifier_datetime(self): self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00") - self._run_test("{ds:D%Y}", "2010-01-01T01:00:00+01:00") - self._run_test("{l:D%Y}", "None") + self._run_test("{ds:D%Y}", "1970-01-01 00:00:00") + self._run_test("{l:D%Y}", "1970-01-01 00:00:00") def test_specifier_offset(self): self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00") From 21350c5084a2bfa451b881bfa45cdd853eb65269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 15 Oct 2025 22:42:29 +0200 Subject: [PATCH 05/13] [dt] introduce 'NullDatetime' to represent invalid datetimes --- gallery_dl/dt.py | 18 +++++++++++++----- test/test_dt.py | 33 +++++++++++++++++++++------------ test/test_formatter.py | 4 ++-- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index 0008cf57..9f1cd09e 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -12,6 +12,14 @@ import sys import time from datetime import datetime, date, timedelta, timezone # noqa F401 + +class NullDatetime(datetime): + + def __bool__(self): + return False + + +NONE = NullDatetime(101, 1, 1) EPOCH = datetime(1970, 1, 1) SECOND = timedelta(0, 1) @@ -29,7 +37,7 @@ def normalize(dt): def convert(value): """Convert 'value' to a naive UTC datetime object""" if not value: - return EPOCH + return NONE if isinstance(value, datetime): return value @@ -51,7 +59,7 @@ def parse(dt_string, format): try: return normalize(datetime.strptime(dt_string, format)) except Exception: - return EPOCH + return NONE if sys.hexversion < 0x30c0000: @@ -64,7 +72,7 @@ if sys.hexversion < 0x30c0000: dt_string = dt_string[:-1] return normalize(datetime.fromisoformat(dt_string)) except Exception: - return EPOCH + return NONE def parse_compat(dt_string, format): """Parse 'dt_string' as ISO 8601 value using 'format'""" @@ -80,7 +88,7 @@ else: try: return normalize(datetime.fromisoformat(dt_string)) except Exception: - return EPOCH + return NONE def parse_compat(dt_string, format): """Parse 'dt_string' as ISO 8601 value""" @@ -94,7 +102,7 @@ else: now = from_ts -def parse_ts(ts, default=EPOCH): +def parse_ts(ts, default=NONE): """Create a datetime object from a Unix timestamp""" try: return from_ts(int(ts)) diff --git a/test/test_dt.py b/test/test_dt.py index b9cc4d2c..6eea4df4 100644 --- a/test/test_dt.py +++ b/test/test_dt.py @@ -44,14 +44,14 @@ class TestDatetime(unittest.TestCase): _assert("2010-01-01T00:00:00Z" , d) _assert("2010-01-01T00:00:00.123456Z" , d) - _assert(0 , dt.EPOCH) - _assert("" , dt.EPOCH) - _assert("foo", dt.EPOCH) - _assert(None , dt.EPOCH) - _assert(() , dt.EPOCH) - _assert([] , dt.EPOCH) - _assert({} , dt.EPOCH) - _assert((1, 2, 3), dt.EPOCH) + _assert(0 , dt.NONE) + _assert("" , dt.NONE) + _assert("foo", dt.NONE) + _assert(None , dt.NONE) + _assert(() , dt.NONE) + _assert([] , dt.NONE) + _assert({} , dt.NONE) + _assert((1, 2, 3), dt.NONE) @unittest.skipIf(sys.hexversion < 0x30b0000, "extended fromisoformat timezones") @@ -101,7 +101,7 @@ class TestDatetime(unittest.TestCase): self.assertEqual(f("1555816235"), value) for value in ((), [], {}, None, ""): - self.assertEqual(f(value), dt.EPOCH) + self.assertEqual(f(value), dt.NONE) self.assertEqual(f(value, "foo"), "foo") def test_parse(self, f=dt.parse): @@ -119,7 +119,7 @@ class TestDatetime(unittest.TestCase): ) for value in ((), [], {}, None, 1, 2.3): - self.assertEqual(f(value, "%Y"), dt.EPOCH) + self.assertEqual(f(value, "%Y"), dt.NONE) def test_parse_iso(self, f=dt.parse_iso): null = dt.from_ts(0) @@ -140,12 +140,16 @@ class TestDatetime(unittest.TestCase): datetime.datetime(2019, 5, 7, 21, 25, 2), ) self.assertEqual( - f("1970.01.01"), + f("1970-01-01"), dt.EPOCH, ) + self.assertEqual( + f("1970.01.01"), + dt.NONE, + ) for value in ((), [], {}, None, 1, 2.3): - self.assertEqual(f(value), dt.EPOCH) + self.assertEqual(f(value), dt.NONE) def test_parse_compat(self, f=dt.parse_compat): self.assertEqual( @@ -153,6 +157,11 @@ class TestDatetime(unittest.TestCase): datetime.datetime(2019, 5, 7, 12, 25, 2), ) + def test_none(self): + self.assertFalse(dt.NONE) + self.assertIsInstance(dt.NONE, dt.datetime) + self.assertEqual(str(dt.NONE), "0101-01-01 00:00:00") + if __name__ == "__main__": unittest.main() diff --git a/test/test_formatter.py b/test/test_formatter.py index 7a286190..e4444346 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -271,8 +271,8 @@ class TestFormatter(unittest.TestCase): def test_specifier_datetime(self): self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00") - self._run_test("{ds:D%Y}", "1970-01-01 00:00:00") - self._run_test("{l:D%Y}", "1970-01-01 00:00:00") + self._run_test("{ds:D%Y}", "0101-01-01 00:00:00") + self._run_test("{l2:D%Y}", "0101-01-01 00:00:00") def test_specifier_offset(self): self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00") From 69f7cfdd0c567a4662207eda9d01b3ec0896e597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 16 Oct 2025 10:59:03 +0200 Subject: [PATCH 06/13] [dt] replace 'datetime' imports --- gallery_dl/extractor/aryion.py | 5 ++--- gallery_dl/extractor/common.py | 9 ++++----- gallery_dl/extractor/danbooru.py | 12 +++++------- gallery_dl/extractor/moebooru.py | 11 +++++------ gallery_dl/extractor/naverblog.py | 11 +++++------ gallery_dl/extractor/plurk.py | 11 ++++------- gallery_dl/extractor/sexcom.py | 11 +++++------ gallery_dl/extractor/tumblr.py | 7 +++---- gallery_dl/formatter.py | 7 +++---- test/test_dt.py | 13 ++++++++----- 10 files changed, 44 insertions(+), 53 deletions(-) diff --git a/gallery_dl/extractor/aryion.py b/gallery_dl/extractor/aryion.py index 38b8ee40..b584c462 100644 --- a/gallery_dl/extractor/aryion.py +++ b/gallery_dl/extractor/aryion.py @@ -9,10 +9,9 @@ """Extractors for https://aryion.com/""" from .common import Extractor, Message -from .. import text, util, exception +from .. import text, util, dt, exception from ..cache import cache from email.utils import parsedate_tz -from datetime import datetime BASE_PATTERN = r"(?:https?://)?(?:www\.)?aryion\.com/g4" @@ -156,7 +155,7 @@ class AryionExtractor(Extractor): "artist": artist, "path" : text.split_html(extr( "cookiecrumb'>", ':", "<").replace(",", "")), "width" : text.parse_int(extr("Resolution:", "x")), diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 83cb7a50..dd8f119e 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -19,7 +19,6 @@ import getpass import logging import requests import threading -from datetime import datetime from xml.etree import ElementTree from requests.adapters import HTTPAdapter from .message import Message @@ -313,7 +312,7 @@ class Extractor(): seconds = float(seconds) until = now + seconds elif until: - if isinstance(until, datetime): + if isinstance(until, dt.datetime): # convert to UTC timestamp until = dt.to_ts(until) else: @@ -327,7 +326,7 @@ class Extractor(): return if reason: - t = datetime.fromtimestamp(until).time() + t = dt.datetime.fromtimestamp(until).time() isotime = f"{t.hour:02}:{t.minute:02}:{t.second:02}" self.log.info("Waiting until %s (%s)", isotime, reason) time.sleep(seconds) @@ -652,7 +651,7 @@ class Extractor(): self.log.warning( "cookies: %s/%s expired at %s", cookie.domain.lstrip("."), cookie.name, - datetime.fromtimestamp(cookie.expires)) + dt.datetime.fromtimestamp(cookie.expires)) continue elif diff <= 86400: @@ -694,7 +693,7 @@ class Extractor(): ts = self.config(key, default) if isinstance(ts, str): try: - ts = int(datetime.strptime(ts, fmt).timestamp()) + ts = int(dt.parse(ts, fmt).timestamp()) except ValueError as exc: self.log.warning("Unable to parse '%s': %s", key, exc) ts = default diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 29c77634..bbe20ed3 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -9,8 +9,7 @@ """Extractors for https://danbooru.donmai.us/ and other Danbooru instances""" from .common import BaseExtractor, Message -from .. import text, util -import datetime +from .. import text, util, dt class DanbooruExtractor(BaseExtractor): @@ -69,8 +68,7 @@ class DanbooruExtractor(BaseExtractor): continue text.nameext_from_url(url, post) - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = dt.parse_iso(post["created_at"]) post["tags"] = ( post["tag_string"].split(" ") @@ -357,11 +355,11 @@ class DanbooruPopularExtractor(DanbooruExtractor): def metadata(self): self.params = params = text.parse_query(self.groups[-1]) scale = params.get("scale", "day") - date = params.get("date") or datetime.date.today().isoformat() + date = params.get("date") or dt.date.today().isoformat() if scale == "week": - date = datetime.date.fromisoformat(date) - date = (date - datetime.timedelta(days=date.weekday())).isoformat() + date = dt.date.fromisoformat(date) + date = (date - dt.timedelta(days=date.weekday())).isoformat() elif scale == "month": date = date[:-3] diff --git a/gallery_dl/extractor/moebooru.py b/gallery_dl/extractor/moebooru.py index ba279948..e4c8f1c7 100644 --- a/gallery_dl/extractor/moebooru.py +++ b/gallery_dl/extractor/moebooru.py @@ -9,9 +9,8 @@ """Extractors for Moebooru based sites""" from .booru import BooruExtractor -from .. import text, util +from .. import text, util, dt import collections -import datetime class MoebooruExtractor(BooruExtractor): @@ -21,7 +20,7 @@ class MoebooruExtractor(BooruExtractor): page_start = 1 def _prepare(self, post): - post["date"] = text.parse_timestamp(post["created_at"]) + post["date"] = dt.parse_ts(post["created_at"]) def _html(self, post): url = f"{self.root}/post/show/{post['id']}" @@ -164,14 +163,14 @@ class MoebooruPopularExtractor(MoebooruExtractor): date = (f"{params['year']:>04}-{params.get('month', '01'):>02}-" f"{params.get('day', '01'):>02}") else: - date = datetime.date.today().isoformat() + date = dt.date.today().isoformat() scale = self.scale if scale.startswith("by_"): scale = scale[3:] if scale == "week": - date = datetime.date.fromisoformat(date) - date = (date - datetime.timedelta(days=date.weekday())).isoformat() + date = dt.date.fromisoformat(date) + date = (date - dt.timedelta(days=date.weekday())).isoformat() elif scale == "month": date = date[:-3] diff --git a/gallery_dl/extractor/naverblog.py b/gallery_dl/extractor/naverblog.py index b55e0012..cc96e099 100644 --- a/gallery_dl/extractor/naverblog.py +++ b/gallery_dl/extractor/naverblog.py @@ -9,8 +9,7 @@ """Extractors for https://blog.naver.com/""" from .common import GalleryExtractor, Extractor, Message -from .. import text, util -import datetime +from .. import text, util, dt import time @@ -67,11 +66,11 @@ class NaverBlogPostExtractor(NaverBlogBase, GalleryExtractor): return data - def _parse_datetime(self, date_string): - if "전" in date_string: + def _parse_datetime(self, dt_string): + if "전" in dt_string: ts = time.gmtime() - return datetime.datetime(ts.tm_year, ts.tm_mon, ts.tm_mday) - return text.parse_datetime(date_string, "%Y. %m. %d. %H:%M") + return dt.datetime(ts.tm_year, ts.tm_mon, ts.tm_mday) + return dt.parse(dt_string, "%Y. %m. %d. %H:%M") def images(self, page): files = [] diff --git a/gallery_dl/extractor/plurk.py b/gallery_dl/extractor/plurk.py index 37b9b100..ed7e0105 100644 --- a/gallery_dl/extractor/plurk.py +++ b/gallery_dl/extractor/plurk.py @@ -9,8 +9,7 @@ """Extractors for https://www.plurk.com/""" from .common import Extractor, Message -from .. import text, util, exception -import datetime +from .. import text, util, dt, exception class PlurkExtractor(Extractor): @@ -88,12 +87,10 @@ class PlurkTimelineExtractor(PlurkExtractor): while plurks: yield from plurks - offset = datetime.datetime.strptime( - plurks[-1]["posted"], "%a, %d %b %Y %H:%M:%S %Z") + offset = dt.parse(plurks[-1]["posted"], "%a, %d %b %Y %H:%M:%S %Z") data["offset"] = offset.strftime("%Y-%m-%dT%H:%M:%S.000Z") - response = self.request( - url, method="POST", headers=headers, data=data) - plurks = response.json()["plurks"] + plurks = self.request_json( + url, method="POST", headers=headers, data=data)["plurks"] class PlurkPostExtractor(PlurkExtractor): diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py index 2feb64eb..6a52f823 100644 --- a/gallery_dl/extractor/sexcom.py +++ b/gallery_dl/extractor/sexcom.py @@ -9,8 +9,7 @@ """Extractors for https://www.sex.com/""" from .common import Extractor, Message -from .. import text -from datetime import datetime +from .. import text, dt BASE_PATTERN = r"(?:https?://)?(?:www\.)?sex\.com(?:/[a-z]{2})?" @@ -34,10 +33,10 @@ class SexcomExtractor(Extractor): url = pin["url"] parts = url.rsplit("/", 4) try: - pin["date_url"] = dt = datetime( + pin["date_url"] = d = dt.datetime( int(parts[1]), int(parts[2]), int(parts[3])) if "date" not in pin: - pin["date"] = dt + pin["date"] = d except Exception: pass pin["tags"] = [t[1:] for t in pin["tags"]] @@ -136,7 +135,7 @@ class SexcomExtractor(Extractor): text.nameext_from_url(data["url"], data) data["uploader"] = extr('itemprop="author">', '<') - data["date"] = text.parse_datetime(extr('datetime="', '"')) + data["date"] = dt.parse_iso(extr('datetime="', '"')) data["tags"] = text.split_html(extr('class="tags"> Tags', '')) data["comments"] = text.parse_int(extr('Comments (', ')')) @@ -314,7 +313,7 @@ class SexcomSearchExtractor(SexcomExtractor): parts = path.rsplit("/", 4) try: - pin["date_url"] = pin["date"] = datetime( + pin["date_url"] = pin["date"] = dt.datetime( int(parts[1]), int(parts[2]), int(parts[3])) except Exception: pass diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 92fc8314..beaa5c77 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -9,8 +9,7 @@ """Extractors for https://www.tumblr.com/""" from .common import Extractor, Message -from .. import text, util, oauth, exception -from datetime import datetime, date, timedelta +from .. import text, util, dt, oauth, exception BASE_PATTERN = ( @@ -313,7 +312,7 @@ class TumblrDayExtractor(TumblrExtractor): def posts(self): year, month, day = self.groups[3].split("/") - ordinal = date(int(year), int(month), int(day)).toordinal() + ordinal = dt.date(int(year), int(month), int(day)).toordinal() # 719163 == date(1970, 1, 1).toordinal() self.date_min = (ordinal - 719163) * 86400 @@ -514,7 +513,7 @@ class TumblrAPI(oauth.OAuth1API): self.extractor.wait(seconds=reset) continue - t = (datetime.now() + timedelta(0, float(reset))).time() + t = (dt.now() + dt.timedelta(0, float(reset))).time() raise exception.AbortExtraction( f"Aborting - Rate limit will reset at " f"{t.hour:02}:{t.minute:02}:{t.second:02}") diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 39a5e4e2..8b5e7adf 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -13,7 +13,6 @@ import sys import time import string import _string -import datetime import operator from . import text, util, dt @@ -484,13 +483,13 @@ def _parse_offset(format_spec, default): if not offset or offset == "local": def off(dt_utc): local = time.localtime(dt.to_ts(dt_utc)) - return fmt(dt_utc + datetime.timedelta(0, local.tm_gmtoff)) + return fmt(dt_utc + dt.timedelta(0, local.tm_gmtoff)) else: hours, _, minutes = offset.partition(":") offset = 3600 * int(hours) if minutes: offset += 60 * (int(minutes) if offset > 0 else -int(minutes)) - offset = datetime.timedelta(0, offset) + offset = dt.timedelta(0, offset) def off(obj): return fmt(obj + offset) @@ -557,7 +556,7 @@ _FORMATTERS = { _GLOBALS = { "_env": lambda: os.environ, "_lit": lambda: _literal, - "_now": datetime.datetime.now, + "_now": dt.datetime.now, "_nul": lambda: util.NONE, } _CONVERSIONS = { diff --git a/test/test_dt.py b/test/test_dt.py index 6eea4df4..1c0aa16b 100644 --- a/test/test_dt.py +++ b/test/test_dt.py @@ -122,11 +122,10 @@ class TestDatetime(unittest.TestCase): self.assertEqual(f(value, "%Y"), dt.NONE) def test_parse_iso(self, f=dt.parse_iso): - null = dt.from_ts(0) - - self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - + self.assertEqual( + f("1970-01-01T00:00:00+00:00"), + dt.from_ts(0), + ) self.assertEqual( f("2019-05-07T21:25:02+09:00"), datetime.datetime(2019, 5, 7, 12, 25, 2), @@ -152,6 +151,10 @@ class TestDatetime(unittest.TestCase): self.assertEqual(f(value), dt.NONE) def test_parse_compat(self, f=dt.parse_compat): + self.assertEqual( + f("1970-01-01T00:00:00+0000", "%Y-%m-%dT%H:%M:%S%z"), + dt.EPOCH, + ) self.assertEqual( f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), datetime.datetime(2019, 5, 7, 12, 25, 2), From d57dc48dcdb13019555ce11f0f5980051aefe761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 16 Oct 2025 19:58:15 +0200 Subject: [PATCH 07/13] [tests/results] replace 'datetime' usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test for 'dt:…' to catch NullDatetime instances, which inherit from 'datetime.datetime' but are not exactly this class --- test/results/2chen.py | 3 +-- test/results/4archive.py | 3 +-- test/results/deviantart.py | 3 +-- test/results/facebook.py | 7 +++---- test/results/hentaifoundry.py | 3 +-- test/results/horne.py | 3 +-- test/results/imgur.py | 3 +-- test/results/inkbunny.py | 3 +-- test/results/luscious.py | 3 +-- test/results/mangadex.py | 3 +-- test/results/mangafox.py | 3 +-- test/results/mangapark.py | 3 +-- test/results/nijie.py | 3 +-- test/results/patreon.py | 3 +-- test/results/photovogue.py | 3 +-- test/results/picarto.py | 3 +-- test/results/pillowfort.py | 3 +-- test/results/pixeldrain.py | 3 +-- test/results/subscribestar.py | 5 ++--- test/results/toyhouse.py | 3 +-- 20 files changed, 23 insertions(+), 43 deletions(-) diff --git a/test/results/2chen.py b/test/results/2chen.py index 589053fa..132f75d7 100644 --- a/test/results/2chen.py +++ b/test/results/2chen.py @@ -6,7 +6,6 @@ gallery_dl = __import__("gallery_dl.extractor.2chen") _2chen = getattr(gallery_dl.extractor, "2chen") -import datetime __tests__ = ( @@ -18,7 +17,7 @@ __tests__ = ( "#count" : ">= 179", "board" : "tv", - "date" : datetime.datetime, + "date" : "type:datetime", "hash" : r"re:[0-9a-f]{40}", "name" : "Anonymous", "no" : r"re:\d+", diff --git a/test/results/4archive.py b/test/results/4archive.py index cebec6fc..f907577f 100644 --- a/test/results/4archive.py +++ b/test/results/4archive.py @@ -6,7 +6,6 @@ gallery_dl = __import__("gallery_dl.extractor.4archive") _4archive = getattr(gallery_dl.extractor, "4archive") -import datetime __tests__ = ( @@ -19,7 +18,7 @@ __tests__ = ( "board" : "u", "com" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "name" : "Anonymous", "no" : range(2397221, 2418158), "thread": 2397221, diff --git a/test/results/deviantart.py b/test/results/deviantart.py index c3a6c537..ee911e99 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import deviantart -import datetime from gallery_dl import exception @@ -60,7 +59,7 @@ __tests__ = ( "transparency": bool, "width" : int, }, - "date" : datetime.datetime, + "date" : "type:datetime", "deviationid" : str, "?download_filesize": int, "extension" : str, diff --git a/test/results/facebook.py b/test/results/facebook.py index 7eda027d..64f49690 100644 --- a/test/results/facebook.py +++ b/test/results/facebook.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import facebook from gallery_dl import exception -import datetime __tests__ = ( @@ -191,7 +190,7 @@ __tests__ = ( "#count" : 1, "caption" : "They were on a break... #FriendsReunion #MoreTogether", - "date" : datetime.datetime(2021, 5, 27, 21, 55, 19), + "date" : "dt:2021-05-27 21:55:19", "filename" : "191053255_10160743390471729_9001965649022744000_n", "extension": "jpg", "id" : "10160743390456729", @@ -212,7 +211,7 @@ __tests__ = ( "#count" : 1, "caption" : "", - "date" : datetime.datetime(2014, 5, 3, 0, 44, 47), + "date" : "dt:2014-05-03 00:44:47", "filename" : str, "extension": "png", "id" : "10152716011076729", @@ -272,7 +271,7 @@ __tests__ = ( "#class" : facebook.FacebookVideoExtractor, "#count" : 1, - "date" : datetime.datetime(2024, 4, 19, 17, 25, 48), + "date" : "dt:2024-04-19 17:25:48", "filename" : str, "id" : "1165557851291824", "url" : str, diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py index 6da50035..65aaafcf 100644 --- a/test/results/hentaifoundry.py +++ b/test/results/hentaifoundry.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import hentaifoundry -import datetime __tests__ = ( @@ -190,7 +189,7 @@ Sorry for the bad quality, I made it on after effect because Flash works like sh "author" : "SnowWolf35", "chapters" : int, "comments" : int, - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "index" : int, "rating" : int, diff --git a/test/results/horne.py b/test/results/horne.py index 6d2a0b49..418e656b 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import nijie -import datetime __tests__ = ( @@ -29,7 +28,7 @@ __tests__ = ( "artist_id" : 58000, "artist_name": "のえるわ", - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "image_id" : int, "num" : int, diff --git a/test/results/imgur.py b/test/results/imgur.py index bd99245f..8d6cf3ae 100644 --- a/test/results/imgur.py +++ b/test/results/imgur.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import imgur from gallery_dl import exception -import datetime __tests__ = ( @@ -215,7 +214,7 @@ __tests__ = ( }, "account_id" : 0, "count" : 19, - "date" : datetime.datetime, + "date" : "type:datetime", "description": "", "ext" : "jpg", "has_sound" : False, diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py index 4e05da7c..78885f59 100644 --- a/test/results/inkbunny.py +++ b/test/results/inkbunny.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import inkbunny -import datetime __tests__ = ( @@ -16,7 +15,7 @@ __tests__ = ( "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", "#range" : "20-50", - "date" : datetime.datetime, + "date" : "type:datetime", "deleted" : bool, "file_id" : r"re:[0-9]+", "filename" : r"re:[0-9]+_soina_\w+", diff --git a/test/results/luscious.py b/test/results/luscious.py index 597b5699..32626485 100644 --- a/test/results/luscious.py +++ b/test/results/luscious.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import luscious -import datetime from gallery_dl import exception @@ -49,7 +48,7 @@ __tests__ = ( "aspect_ratio" : r"re:\d+:\d+", "category" : "luscious", "created" : int, - "date" : datetime.datetime, + "date" : "type:datetime", "height" : int, "id" : int, "is_animated" : False, diff --git a/test/results/mangadex.py b/test/results/mangadex.py index 5309db13..5fdd7150 100644 --- a/test/results/mangadex.py +++ b/test/results/mangadex.py @@ -6,7 +6,6 @@ from gallery_dl.extractor import mangadex from gallery_dl import exception -import datetime __tests__ = ( @@ -132,7 +131,7 @@ __tests__ = ( "chapter" : 0, "chapter_minor": "", "chapter_id" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "lang" : "iso:lang", "artist" : ["Arakawa Hiromu"], "author" : ["Arakawa Hiromu"], diff --git a/test/results/mangafox.py b/test/results/mangafox.py index dc0cc9a0..04651496 100644 --- a/test/results/mangafox.py +++ b/test/results/mangafox.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import mangafox -import datetime __tests__ = ( @@ -40,7 +39,7 @@ __tests__ = ( "chapter" : int, "chapter_minor" : r"re:^(\.\d+)?$", "chapter_string": r"re:(v\d+/)?c\d+", - "date" : datetime.datetime, + "date" : "type:datetime", "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", "lang" : "en", "language" : "English", diff --git a/test/results/mangapark.py b/test/results/mangapark.py index b40ab3fc..2ad5d426 100644 --- a/test/results/mangapark.py +++ b/test/results/mangapark.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import mangapark -import datetime __tests__ = ( @@ -115,7 +114,7 @@ __tests__ = ( "chapter" : int, "chapter_id" : r"re:\d+", "chapter_minor": str, - "date" : datetime.datetime, + "date" : "type:datetime", "lang" : "en", "language" : "English", "manga_id" : 114972, diff --git a/test/results/nijie.py b/test/results/nijie.py index 4e06f871..b5a53fb1 100644 --- a/test/results/nijie.py +++ b/test/results/nijie.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import nijie -import datetime from gallery_dl import exception @@ -32,7 +31,7 @@ __tests__ = ( "artist_id" : 44, "artist_name": "ED", "count" : 1, - "date" : datetime.datetime, + "date" : "type:datetime", "description": str, "extension" : "jpg", "filename" : str, diff --git a/test/results/patreon.py b/test/results/patreon.py index f2aeee8b..426981de 100644 --- a/test/results/patreon.py +++ b/test/results/patreon.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import patreon -import datetime from gallery_dl import exception @@ -21,7 +20,7 @@ __tests__ = ( "comment_count": int, "content" : str, "creator" : dict, - "date" : datetime.datetime, + "date" : "type:datetime", "id" : int, "images" : list, "like_count" : int, diff --git a/test/results/photovogue.py b/test/results/photovogue.py index 6898f5d9..187c374a 100644 --- a/test/results/photovogue.py +++ b/test/results/photovogue.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import photovogue -import datetime __tests__ = ( @@ -21,7 +20,7 @@ __tests__ = ( "#class" : photovogue.PhotovogueUserExtractor, "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - "date" : datetime.datetime, + "date" : "type:datetime", "favorite_count" : int, "favorited" : list, "id" : int, diff --git a/test/results/picarto.py b/test/results/picarto.py index 07456260..5d2af21f 100644 --- a/test/results/picarto.py +++ b/test/results/picarto.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import picarto -import datetime __tests__ = ( @@ -16,7 +15,7 @@ __tests__ = ( "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", "#count" : ">= 7", - "date": datetime.datetime, + "date": "type:datetime", }, ) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py index 514697be..fc583614 100644 --- a/test/results/pillowfort.py +++ b/test/results/pillowfort.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import pillowfort -import datetime __tests__ = ( @@ -24,7 +23,7 @@ __tests__ = ( "content" : str, "count" : 4, "created_at" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "deleted" : None, "deleted_at" : None, "deleted_by_mod" : None, diff --git a/test/results/pixeldrain.py b/test/results/pixeldrain.py index 1afc49b9..563634a0 100644 --- a/test/results/pixeldrain.py +++ b/test/results/pixeldrain.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import pixeldrain -import datetime __tests__ = ( { @@ -81,7 +80,7 @@ __tests__ = ( "success" : True, "title" : "アルバム", }, - "date" : datetime.datetime, + "date" : "type:datetime", "description": "", "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", "hash_sha256": r"re:\w{64}", diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py index 94850ac3..a8c7146a 100644 --- a/test/results/subscribestar.py +++ b/test/results/subscribestar.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import subscribestar -import datetime __tests__ = ( @@ -20,7 +19,7 @@ __tests__ = ( "author_name": "subscribestar", "author_nick": "SubscribeStar", "content" : str, - "date" : datetime.datetime, + "date" : "type:datetime", "id" : int, "num" : int, "post_id" : int, @@ -38,7 +37,7 @@ __tests__ = ( "#options" : {"metadata": True}, "#range" : "1", - "date": datetime.datetime, + "date": "type:datetime", }, { diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py index 21d13ee1..46d61954 100644 --- a/test/results/toyhouse.py +++ b/test/results/toyhouse.py @@ -5,7 +5,6 @@ # published by the Free Software Foundation. from gallery_dl.extractor import toyhouse -import datetime __tests__ = ( @@ -19,7 +18,7 @@ __tests__ = ( "artists" : list, "characters": list, - "date" : datetime.datetime, + "date" : "type:datetime", "hash" : r"re:\w+", "id" : r"re:\d+", "url" : str, From 085616e0a8d4dcaccd4940666ea49d9b341d83f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 17 Oct 2025 17:43:06 +0200 Subject: [PATCH 08/13] [dt] replace 'text.parse_datetime()' & 'text.parse_timestamp()' --- gallery_dl/extractor/2ch.py | 2 +- gallery_dl/extractor/2chen.py | 2 +- gallery_dl/extractor/4archive.py | 2 +- gallery_dl/extractor/8muses.py | 2 +- gallery_dl/extractor/adultempire.py | 2 +- gallery_dl/extractor/agnph.py | 2 +- gallery_dl/extractor/ao3.py | 6 ++-- gallery_dl/extractor/arcalive.py | 2 +- gallery_dl/extractor/artstation.py | 2 +- gallery_dl/extractor/batoto.py | 8 ++--- gallery_dl/extractor/behance.py | 2 +- gallery_dl/extractor/bellazon.py | 6 ++-- gallery_dl/extractor/blogger.py | 4 +-- gallery_dl/extractor/bluesky.py | 2 +- gallery_dl/extractor/boosty.py | 2 +- gallery_dl/extractor/booth.py | 2 +- gallery_dl/extractor/bunkr.py | 2 +- gallery_dl/extractor/catbox.py | 2 +- gallery_dl/extractor/chevereto.py | 4 +-- gallery_dl/extractor/cien.py | 2 +- gallery_dl/extractor/civitai.py | 12 +++---- gallery_dl/extractor/comick.py | 4 +-- gallery_dl/extractor/comicvine.py | 2 +- gallery_dl/extractor/common.py | 4 +++ gallery_dl/extractor/cyberdrop.py | 2 +- gallery_dl/extractor/cyberfile.py | 2 +- gallery_dl/extractor/dankefuerslesen.py | 2 +- gallery_dl/extractor/deviantart.py | 4 +-- gallery_dl/extractor/discord.py | 2 +- gallery_dl/extractor/dynastyscans.py | 6 ++-- gallery_dl/extractor/e621.py | 2 +- gallery_dl/extractor/erome.py | 2 +- gallery_dl/extractor/exhentai.py | 6 ++-- gallery_dl/extractor/facebook.py | 4 +-- gallery_dl/extractor/fanbox.py | 2 +- gallery_dl/extractor/fansly.py | 6 ++-- gallery_dl/extractor/fantia.py | 2 +- gallery_dl/extractor/flickr.py | 4 +-- gallery_dl/extractor/furaffinity.py | 2 +- gallery_dl/extractor/furry34.py | 2 +- gallery_dl/extractor/gelbooru.py | 4 +-- gallery_dl/extractor/gelbooru_v01.py | 2 +- gallery_dl/extractor/gelbooru_v02.py | 2 +- gallery_dl/extractor/girlsreleased.py | 2 +- gallery_dl/extractor/girlswithmuscle.py | 2 +- gallery_dl/extractor/hatenablog.py | 2 +- gallery_dl/extractor/hentaifoundry.py | 4 +-- gallery_dl/extractor/hentaihand.py | 2 +- gallery_dl/extractor/hitomi.py | 2 +- gallery_dl/extractor/imagechest.py | 4 +-- gallery_dl/extractor/imgbb.py | 2 +- gallery_dl/extractor/imgth.py | 2 +- gallery_dl/extractor/imgur.py | 4 +-- gallery_dl/extractor/inkbunny.py | 4 +-- gallery_dl/extractor/instagram.py | 12 +++---- gallery_dl/extractor/issuu.py | 2 +- gallery_dl/extractor/itaku.py | 6 ++-- gallery_dl/extractor/iwara.py | 6 ++-- gallery_dl/extractor/kabeuchi.py | 2 +- gallery_dl/extractor/kemono.py | 2 +- gallery_dl/extractor/lensdump.py | 2 +- gallery_dl/extractor/livedoor.py | 2 +- gallery_dl/extractor/lofter.py | 2 +- gallery_dl/extractor/luscious.py | 4 +-- gallery_dl/extractor/madokami.py | 2 +- gallery_dl/extractor/mangadex.py | 8 ++--- gallery_dl/extractor/mangafox.py | 2 +- gallery_dl/extractor/manganelo.py | 8 ++--- gallery_dl/extractor/mangapark.py | 4 +-- gallery_dl/extractor/mangataro.py | 4 +-- gallery_dl/extractor/mangoxo.py | 2 +- gallery_dl/extractor/mastodon.py | 4 +-- gallery_dl/extractor/misskey.py | 4 +-- gallery_dl/extractor/naverchzzk.py | 6 ++-- gallery_dl/extractor/nekohouse.py | 2 +- gallery_dl/extractor/newgrounds.py | 8 ++--- gallery_dl/extractor/nijie.py | 4 +-- gallery_dl/extractor/nitter.py | 6 ++-- gallery_dl/extractor/nozomi.py | 2 +- gallery_dl/extractor/paheal.py | 4 +-- gallery_dl/extractor/patreon.py | 6 ++-- gallery_dl/extractor/pexels.py | 2 +- gallery_dl/extractor/philomena.py | 2 +- gallery_dl/extractor/photovogue.py | 2 +- gallery_dl/extractor/picarto.py | 2 +- gallery_dl/extractor/piczel.py | 2 +- gallery_dl/extractor/pillowfort.py | 4 +-- gallery_dl/extractor/pixeldrain.py | 12 +++---- gallery_dl/extractor/pornhub.py | 2 +- gallery_dl/extractor/postmill.py | 2 +- gallery_dl/extractor/rawkuma.py | 2 +- gallery_dl/extractor/reactor.py | 2 +- gallery_dl/extractor/realbooru.py | 2 +- gallery_dl/extractor/redbust.py | 2 +- gallery_dl/extractor/reddit.py | 4 +-- gallery_dl/extractor/redgifs.py | 2 +- gallery_dl/extractor/rule34vault.py | 2 +- gallery_dl/extractor/rule34xyz.py | 2 +- gallery_dl/extractor/s3ndpics.py | 4 +-- gallery_dl/extractor/saint.py | 4 +-- gallery_dl/extractor/sankaku.py | 2 +- gallery_dl/extractor/sankakucomplex.py | 2 +- gallery_dl/extractor/schalenetwork.py | 2 +- gallery_dl/extractor/seiga.py | 2 +- gallery_dl/extractor/simpcity.py | 4 +-- gallery_dl/extractor/simplyhentai.py | 4 +-- gallery_dl/extractor/sizebooru.py | 4 +-- gallery_dl/extractor/slideshare.py | 3 +- gallery_dl/extractor/subscribestar.py | 4 +-- gallery_dl/extractor/szurubooru.py | 2 +- gallery_dl/extractor/tapas.py | 2 +- gallery_dl/extractor/telegraph.py | 2 +- gallery_dl/extractor/tenor.py | 2 +- gallery_dl/extractor/thehentaiworld.py | 2 +- gallery_dl/extractor/tiktok.py | 2 +- gallery_dl/extractor/toyhouse.py | 2 +- gallery_dl/extractor/tsumino.py | 2 +- gallery_dl/extractor/tumblr.py | 2 +- gallery_dl/extractor/tungsten.py | 2 +- gallery_dl/extractor/twibooru.py | 4 +-- gallery_dl/extractor/twitter.py | 12 +++---- gallery_dl/extractor/unsplash.py | 2 +- gallery_dl/extractor/urlgalleries.py | 2 +- gallery_dl/extractor/vanillarock.py | 2 +- gallery_dl/extractor/vk.py | 2 +- gallery_dl/extractor/vsco.py | 2 +- gallery_dl/extractor/wallhaven.py | 2 +- gallery_dl/extractor/warosu.py | 2 +- gallery_dl/extractor/weasyl.py | 4 +-- gallery_dl/extractor/webmshare.py | 2 +- gallery_dl/extractor/weebcentral.py | 2 +- gallery_dl/extractor/weibo.py | 2 +- gallery_dl/extractor/wikifeet.py | 2 +- gallery_dl/extractor/wikimedia.py | 2 +- gallery_dl/extractor/xhamster.py | 2 +- gallery_dl/extractor/yiffverse.py | 2 +- gallery_dl/extractor/zerochan.py | 2 +- gallery_dl/text.py | 43 ------------------------- 138 files changed, 220 insertions(+), 260 deletions(-) diff --git a/gallery_dl/extractor/2ch.py b/gallery_dl/extractor/2ch.py index 912a2519..a4621399 100644 --- a/gallery_dl/extractor/2ch.py +++ b/gallery_dl/extractor/2ch.py @@ -46,7 +46,7 @@ class _2chThreadExtractor(Extractor): for post in posts: if files := post.get("files"): post["post_name"] = post["name"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) del post["files"] del post["name"] diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index ee3510cd..af3d6b9c 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -65,7 +65,7 @@ class _2chenThreadExtractor(Extractor): extr = text.extract_from(post) return { "name" : text.unescape(extr("", "")), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr("")[2], "%d %b %Y (%a) %H:%M:%S" ), diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 6a4d0c69..18948907 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -61,7 +61,7 @@ class _4archiveThreadExtractor(Extractor): extr = text.extract_from(post) data = { "name": extr('class="name">', ""), - "date": text.parse_datetime( + "date": self.parse_datetime( (extr('class="dateTime">', "<") or extr('class="dateTime postNum" >', "<")).strip(), "%Y-%m-%d %H:%M:%S"), diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index 120cd8a2..dfc34224 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -85,7 +85,7 @@ class _8musesAlbumExtractor(Extractor): "parent" : text.parse_int(album["parentId"]), "views" : text.parse_int(album["numberViews"]), "likes" : text.parse_int(album["numberLikes"]), - "date" : text.parse_datetime( + "date" : self.parse_datetime( album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), } diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 3249ae6f..e9adf97a 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -33,7 +33,7 @@ class AdultempireGalleryExtractor(GalleryExtractor): "gallery_id": text.parse_int(self.gallery_id), "title" : text.unescape(extr('title="', '"')), "studio" : extr(">studio", "<").strip(), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( ">released", "<").strip(), "%m/%d/%Y"), "actors" : sorted(text.split_html(extr( '
    ', "")), "lang" : extr('
    ', "
    "), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('
    ', "<"), "%Y-%m-%d"), - "date_completed": text.parse_datetime( + "date_completed": self.parse_datetime( extr('>Completed:
    ', "<"), "%Y-%m-%d"), - "date_updated" : text.parse_timestamp( + "date_updated" : self.parse_timestamp( path.rpartition("updated_at=")[2]), "words" : text.parse_int( extr('
    ', "<").replace(",", "")), diff --git a/gallery_dl/extractor/arcalive.py b/gallery_dl/extractor/arcalive.py index 1df7e0fc..94e49988 100644 --- a/gallery_dl/extractor/arcalive.py +++ b/gallery_dl/extractor/arcalive.py @@ -49,7 +49,7 @@ class ArcalivePostExtractor(ArcaliveExtractor): files = self._extract_files(post) post["count"] = len(files) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") post["post_url"] = post_url = \ f"{self.root}/b/{post['boardSlug']}/{post['id']}" diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index fdb92c4e..9a14f5e8 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -126,7 +126,7 @@ class ArtstationExtractor(Extractor): data["title"] = text.unescape(data["title"]) data["description"] = text.unescape(text.remove_html( data["description"])) - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( data["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") assets = data["assets"] diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index a7d1b78c..7495751b 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -123,7 +123,7 @@ class BatotoChapterExtractor(BatotoBase, ChapterExtractor): "chapter_minor" : minor, "chapter_string": info, "chapter_id" : text.parse_int(self.chapter_id), - "date" : text.parse_timestamp(extr(' time="', '"')[:-3]), + "date" : self.parse_timestamp(extr(' time="', '"')[:-3]), } def images(self, page): @@ -167,7 +167,7 @@ class BatotoMangaExtractor(BatotoBase, MangaExtractor): data["chapter"] = text.parse_int(chapter) data["chapter_minor"] = sep + minor - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") url = f"{self.root}/title/{href}" @@ -188,9 +188,9 @@ def _manga_info(self, manga_id, page=None): "manga" : data["name"][1], "manga_id" : text.parse_int(manga_id), "manga_slug" : data["slug"][1], - "manga_date" : text.parse_timestamp( + "manga_date" : self.parse_timestamp( data["dateCreate"][1] // 1000), - "manga_date_updated": text.parse_timestamp( + "manga_date_updated": self.parse_timestamp( data["dateUpdate"][1] / 1000), "author" : json_list(data["authors"]), "artist" : json_list(data["artists"]), diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index 4a7c0740..88d9e569 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -67,7 +67,7 @@ class BehanceExtractor(Extractor): tags = [tag["title"] for tag in tags] data["tags"] = tags - data["date"] = text.parse_timestamp( + data["date"] = self.parse_timestamp( data.get("publishedOn") or data.get("conceived_on") or 0) if creator := data.get("creator"): diff --git a/gallery_dl/extractor/bellazon.py b/gallery_dl/extractor/bellazon.py index ce50a918..0e557cb6 100644 --- a/gallery_dl/extractor/bellazon.py +++ b/gallery_dl/extractor/bellazon.py @@ -141,8 +141,8 @@ class BellazonExtractor(Extractor): "title": schema["headline"], "views": stats[0]["userInteractionCount"], "posts": stats[1]["userInteractionCount"], - "date" : text.parse_datetime(schema["datePublished"]), - "date_updated": text.parse_datetime(schema["dateModified"]), + "date" : self.parse_datetime(schema["datePublished"]), + "date_updated": self.parse_datetime(schema["dateModified"]), "description" : text.unescape(schema["text"]).strip(), "section" : path[-2], "author" : author["name"], @@ -162,7 +162,7 @@ class BellazonExtractor(Extractor): post = { "id": extr('id="elComment_', '"'), "author_url": extr(" href='", "'"), - "date": text.parse_datetime(extr("datetime='", "'")), + "date": self.parse_datetime(extr("datetime='", "'")), "content": extr("", "\n\t\t"), } diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index af434460..d2395daa 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -40,7 +40,7 @@ class BloggerExtractor(BaseExtractor): blog = self.api.blog_by_url("http://" + self.blog) blog["pages"] = blog["pages"]["totalItems"] blog["posts"] = blog["posts"]["totalItems"] - blog["date"] = text.parse_datetime(blog["published"]) + blog["date"] = self.parse_datetime(blog["published"]) del blog["selfLink"] findall_image = util.re( @@ -65,7 +65,7 @@ class BloggerExtractor(BaseExtractor): post["author"] = post["author"]["displayName"] post["replies"] = post["replies"]["totalItems"] post["content"] = text.remove_html(content) - post["date"] = text.parse_datetime(post["published"]) + post["date"] = self.parse_datetime(post["published"]) del post["selfLink"] del post["blog"] diff --git a/gallery_dl/extractor/bluesky.py b/gallery_dl/extractor/bluesky.py index e8c57077..f1fd2eee 100644 --- a/gallery_dl/extractor/bluesky.py +++ b/gallery_dl/extractor/bluesky.py @@ -135,7 +135,7 @@ class BlueskyExtractor(Extractor): post["instance"] = self.instance post["post_id"] = self._pid(post) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") def _extract_files(self, post): diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index 22f3259c..3e01f71b 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -78,7 +78,7 @@ class BoostyExtractor(Extractor): post["links"] = links = [] if "createdAt" in post: - post["date"] = text.parse_timestamp(post["createdAt"]) + post["date"] = self.parse_timestamp(post["createdAt"]) for block in post["data"]: try: diff --git a/gallery_dl/extractor/booth.py b/gallery_dl/extractor/booth.py index 0fcb1cb3..17679da0 100644 --- a/gallery_dl/extractor/booth.py +++ b/gallery_dl/extractor/booth.py @@ -70,7 +70,7 @@ class BoothItemExtractor(BoothExtractor): url + ".json", headers=headers, interval=False) item["booth_category"] = item.pop("category", None) - item["date"] = text.parse_datetime( + item["date"] = self.parse_datetime( item["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") item["tags"] = [t["name"] for t in item["tags"]] diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 14ebc485..b79ba9be 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -167,7 +167,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): item, 'name: "', ".") file["size"] = text.parse_int(text.extr( item, "size: ", " ,\n")) - file["date"] = text.parse_datetime(text.extr( + file["date"] = self.parse_datetime(text.extr( item, 'timestamp: "', '"'), "%H:%M:%S %d/%m/%Y") yield file diff --git a/gallery_dl/extractor/catbox.py b/gallery_dl/extractor/catbox.py index 22f7a976..de21aca2 100644 --- a/gallery_dl/extractor/catbox.py +++ b/gallery_dl/extractor/catbox.py @@ -28,7 +28,7 @@ class CatboxAlbumExtractor(GalleryExtractor): return { "album_id" : self.page_url.rpartition("/")[2], "album_name" : text.unescape(extr("

    ", "<")), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "

    Created ", "<"), "%B %d %Y"), "description": text.unescape(extr("

    ", "<")), } diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index 15528990..465da8dd 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -79,7 +79,7 @@ class CheveretoImageExtractor(CheveretoExtractor): "url" : url, "album": text.remove_html(extr( "Added to ").rpartition(">")[2]), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '', "—"), "album": text.remove_html(extr( "Added to ").rpartition(">")[2]), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '', "B")), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '

    ', '<'), "%d.%m.%Y"), "description": text.unescape(text.unescape( # double desc.rpartition(" [R")[0])), diff --git a/gallery_dl/extractor/cyberfile.py b/gallery_dl/extractor/cyberfile.py index 2ea81d60..bdcc044d 100644 --- a/gallery_dl/extractor/cyberfile.py +++ b/gallery_dl/extractor/cyberfile.py @@ -113,7 +113,7 @@ class CyberfileFileExtractor(CyberfileExtractor): "Filesize:", ""))[:-1]), "tags" : text.split_html(extr( "Keywords:", "")), - "date" : text.parse_datetime(text.remove_html(extr( + "date" : self.parse_datetime(text.remove_html(extr( "Uploaded:", "")), "%d/%m/%Y %H:%M:%S"), "permissions": text.remove_html(extr( "Permissions:", "")).split(" & "), diff --git a/gallery_dl/extractor/dankefuerslesen.py b/gallery_dl/extractor/dankefuerslesen.py index 1c4b7d82..daed1abb 100644 --- a/gallery_dl/extractor/dankefuerslesen.py +++ b/gallery_dl/extractor/dankefuerslesen.py @@ -68,7 +68,7 @@ class DankefuerslesenChapterExtractor(DankefuerslesenBase, ChapterExtractor): "chapter_minor": minor, "group" : manga["groups"][group_id].split(" & "), "group_id" : text.parse_int(group_id), - "date" : text.parse_timestamp(data["release_date"][group_id]), + "date" : self.parse_timestamp(data["release_date"][group_id]), "lang" : util.NONE, "language" : util.NONE, } diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 64b23594..aba5401f 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -259,7 +259,7 @@ class DeviantartExtractor(Extractor): deviation["published_time"] = text.parse_int( deviation["published_time"]) - deviation["date"] = text.parse_timestamp( + deviation["date"] = self.parse_timestamp( deviation["published_time"]) if self.comments: @@ -1187,7 +1187,7 @@ class DeviantartStatusExtractor(DeviantartExtractor): deviation["username"] = deviation["author"]["username"] deviation["_username"] = deviation["username"].lower() - deviation["date"] = d = text.parse_datetime(deviation["ts"]) + deviation["date"] = d = self.parse_datetime(deviation["ts"]) deviation["published_time"] = int(dt.to_ts(d)) deviation["da_category"] = "Status" diff --git a/gallery_dl/extractor/discord.py b/gallery_dl/extractor/discord.py index 216e4866..4284233e 100644 --- a/gallery_dl/extractor/discord.py +++ b/gallery_dl/extractor/discord.py @@ -72,7 +72,7 @@ class DiscordExtractor(Extractor): "author_files": [], "message": self.extract_message_text(message), "message_id": message["id"], - "date": text.parse_datetime( + "date": self.parse_datetime( message["timestamp"], "%Y-%m-%dT%H:%M:%S.%f%z" ), "files": [] diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index 3e0424d3..8906d44f 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -62,7 +62,7 @@ class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): "author" : text.remove_html(author), "group" : (text.remove_html(group) or text.extr(group, ' alt="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '"icon-calendar"> ', '<'), "%b %d, %Y"), "tags" : text.split_html(extr( "class='tags'>", "

    1: - date = text.parse_timestamp(ts) + date = self.parse_timestamp(ts) data = { "album_id": album_id, diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index f147959a..e7b2ab9c 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -216,7 +216,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): def _items_hitomi(self): if self.config("metadata", False): data = self.metadata_from_api() - data["date"] = text.parse_timestamp(data["posted"]) + data["date"] = self.parse_timestamp(data["posted"]) else: data = {} @@ -233,7 +233,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): data = self.metadata_from_page(page) if self.config("metadata", False): data.update(self.metadata_from_api()) - data["date"] = text.parse_timestamp(data["posted"]) + data["date"] = self.parse_timestamp(data["posted"]) if self.config("tags", False): tags = collections.defaultdict(list) for tag in data["tags"]: @@ -258,7 +258,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): "_" : extr('
    "), "title='", "'")], - "date" : text.parse_datetime(extr("datetime='", "'")), + "date" : self.parse_datetime(extr("datetime='", "'")), "views" : text.parse_int(extr(">Views", "<")), "score" : text.parse_int(extr(">Vote Score", "<")), "media" : text.unescape(extr(">Media", "<").strip()), @@ -126,7 +126,7 @@ class HentaifoundryExtractor(Extractor): "title" : text.unescape(extr( "
    ", "").rpartition(">")[2]), "author" : text.unescape(extr('alt="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( ">Updated<", "").rpartition(">")[2], "%B %d, %Y"), "status" : extr("class='indent'>", "<"), } diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index f4f9d861..2510f0ae 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -35,7 +35,7 @@ class HentaihandGalleryExtractor(GalleryExtractor): "language" : info["language"]["name"], "lang" : util.language_to_code(info["language"]["name"]), "tags" : [t["slug"] for t in info["tags"]], - "date" : text.parse_datetime( + "date" : self.parse_datetime( info["uploaded_at"], "%Y-%m-%d"), } for key in ("artists", "authors", "groups", "characters", diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index 82bed807..56606b89 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -84,7 +84,7 @@ class HitomiGalleryExtractor(HitomiExtractor, GalleryExtractor): "type" : info["type"].capitalize(), "language" : language, "lang" : util.language_to_code(language), - "date" : text.parse_datetime(date, "%Y-%m-%d %H:%M:%S%z"), + "date" : self.parse_datetime(date, "%Y-%m-%d %H:%M:%S%z"), "tags" : tags, "artist" : [o["artist"] for o in iget("artists") or ()], "group" : [o["group"] for o in iget("groups") or ()], diff --git a/gallery_dl/extractor/imagechest.py b/gallery_dl/extractor/imagechest.py index 464e4897..59d8c6c5 100644 --- a/gallery_dl/extractor/imagechest.py +++ b/gallery_dl/extractor/imagechest.py @@ -53,10 +53,10 @@ class ImagechestGalleryExtractor(GalleryExtractor): def _metadata_api(self, page): post = self.api.post(self.gallery_id) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") for img in post["images"]: - img["date"] = text.parse_datetime( + img["date"] = self.parse_datetime( img["created"], "%Y-%m-%dT%H:%M:%S.%fZ") post["gallery_id"] = self.gallery_id diff --git a/gallery_dl/extractor/imgbb.py b/gallery_dl/extractor/imgbb.py index d9a63c7f..19e2a533 100644 --- a/gallery_dl/extractor/imgbb.py +++ b/gallery_dl/extractor/imgbb.py @@ -159,7 +159,7 @@ class ImgbbImageExtractor(ImgbbExtractor): "width" : text.parse_int(extr('"og:image:width" content="', '"')), "height": text.parse_int(extr('"og:image:height" content="', '"')), "album" : extr("Added to "), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '", "

    ")), "count": text.parse_int(extr( "total of images in this gallery: ", " ")), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr("created on ", " by <") .replace("th, ", " ", 1).replace("nd, ", " ", 1) .replace("st, ", " ", 1), "%B %d %Y at %H:%M"), diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index 1ac76e0b..61e090fd 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -38,7 +38,7 @@ class ImgurExtractor(Extractor): image["url"] = url = \ f"https://i.imgur.com/{image['id']}.{image['ext']}" - image["date"] = text.parse_datetime(image["created_at"]) + image["date"] = self.parse_datetime(image["created_at"]) image["_http_validate"] = self._validate text.nameext_from_url(url, image) @@ -106,7 +106,7 @@ class ImgurAlbumExtractor(ImgurExtractor): del album["media"] count = len(images) - album["date"] = text.parse_datetime(album["created_at"]) + album["date"] = self.parse_datetime(album["created_at"]) try: del album["ad_url"] diff --git a/gallery_dl/extractor/inkbunny.py b/gallery_dl/extractor/inkbunny.py index 45ae52ed..e0fff57a 100644 --- a/gallery_dl/extractor/inkbunny.py +++ b/gallery_dl/extractor/inkbunny.py @@ -35,7 +35,7 @@ class InkbunnyExtractor(Extractor): for post in self.posts(): post.update(metadata) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["create_datetime"] + "00", "%Y-%m-%d %H:%M:%S.%f%z") post["tags"] = [kw["keyword_name"] for kw in post["keywords"]] post["ratings"] = [r["name"] for r in post["ratings"]] @@ -52,7 +52,7 @@ class InkbunnyExtractor(Extractor): for post["num"], file in enumerate(files, 1): post.update(file) post["deleted"] = (file["deleted"] == "t") - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( file["create_datetime"] + "00", "%Y-%m-%d %H:%M:%S.%f%z") text.nameext_from_url(file["file_name"], post) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index 71964e99..d128f28a 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -173,7 +173,7 @@ class InstagramExtractor(Extractor): post_url = f"{self.root}/stories/highlights/{reel_id}/" data = { "user" : post.get("user"), - "expires": text.parse_timestamp(expires), + "expires": self.parse_timestamp(expires), "post_id": reel_id, "post_shortcode": shortcode_from_id(reel_id), "post_url": post_url, @@ -224,7 +224,7 @@ class InstagramExtractor(Extractor): data["owner_id"] = owner["pk"] data["username"] = owner.get("username") data["fullname"] = owner.get("full_name") - data["post_date"] = data["date"] = text.parse_timestamp( + data["post_date"] = data["date"] = self.parse_timestamp( post.get("taken_at") or post.get("created_at") or post.get("seen")) data["_files"] = files = [] for num, item in enumerate(items, 1): @@ -269,7 +269,7 @@ class InstagramExtractor(Extractor): media = { "num" : num, - "date" : text.parse_timestamp(item.get("taken_at") or + "date" : self.parse_timestamp(item.get("taken_at") or media.get("taken_at") or post.get("taken_at")), "media_id" : item["pk"], @@ -288,7 +288,7 @@ class InstagramExtractor(Extractor): if "reshared_story_media_author" in item: media["author"] = item["reshared_story_media_author"] if "expiring_at" in item: - media["expires"] = text.parse_timestamp(post["expiring_at"]) + media["expires"] = self.parse_timestamp(post["expiring_at"]) self._extract_tagged_users(item, media) files.append(media) @@ -331,7 +331,7 @@ class InstagramExtractor(Extractor): "post_id" : post["id"], "post_shortcode": post["shortcode"], "post_url" : f"{self.root}/p/{post['shortcode']}/", - "post_date" : text.parse_timestamp(post["taken_at_timestamp"]), + "post_date" : self.parse_timestamp(post["taken_at_timestamp"]), "description": text.parse_unicode_escapes("\n".join( edge["node"]["text"] for edge in post["edge_media_to_caption"]["edges"] @@ -623,7 +623,7 @@ class InstagramStoriesTrayExtractor(InstagramExtractor): def items(self): base = f"{self.root}/stories/id:" for story in self.api.reels_tray(): - story["date"] = text.parse_timestamp(story["latest_reel_media"]) + story["date"] = self.parse_timestamp(story["latest_reel_media"]) story["_extractor"] = InstagramStoriesExtractor yield Message.Queue, f"{base}{story['id']}/", story diff --git a/gallery_dl/extractor/issuu.py b/gallery_dl/extractor/issuu.py index 06c5caa5..c463492d 100644 --- a/gallery_dl/extractor/issuu.py +++ b/gallery_dl/extractor/issuu.py @@ -36,7 +36,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor): '{"":' + data.replace('\\"', '"'))) doc = data["initialDocumentData"]["document"] - doc["date"] = text.parse_datetime( + doc["date"] = self.parse_datetime( doc["originalPublishDateInISOString"], "%Y-%m-%dT%H:%M:%S.%fZ") self.count = text.parse_int(doc["pageCount"]) diff --git a/gallery_dl/extractor/itaku.py b/gallery_dl/extractor/itaku.py index 19ffc506..77ebecb4 100644 --- a/gallery_dl/extractor/itaku.py +++ b/gallery_dl/extractor/itaku.py @@ -32,7 +32,7 @@ class ItakuExtractor(Extractor): def items(self): if images := self.images(): for image in images: - image["date"] = text.parse_datetime( + image["date"] = self.parse_datetime( image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") for category, tags in image.pop("categorized_tags").items(): image[f"tags_{category.lower()}"] = [ @@ -60,14 +60,14 @@ class ItakuExtractor(Extractor): for post in posts: images = post.pop("gallery_images") or () post["count"] = len(images) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") post["tags"] = [t["name"] for t in post["tags"]] yield Message.Directory, post for post["num"], image in enumerate(images, 1): post["file"] = image - image["date"] = text.parse_datetime( + image["date"] = self.parse_datetime( image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") url = image["image"] diff --git a/gallery_dl/extractor/iwara.py b/gallery_dl/extractor/iwara.py index 8af2f42a..e0496347 100644 --- a/gallery_dl/extractor/iwara.py +++ b/gallery_dl/extractor/iwara.py @@ -122,9 +122,9 @@ class IwaraExtractor(Extractor): info["file_id"] = file_info.get("id") info["filename"] = filename info["extension"] = extension - info["date"] = text.parse_datetime( + info["date"] = self.parse_datetime( file_info.get("createdAt"), "%Y-%m-%dT%H:%M:%S.%fZ") - info["date_updated"] = text.parse_datetime( + info["date_updated"] = self.parse_datetime( file_info.get("updatedAt"), "%Y-%m-%dT%H:%M:%S.%fZ") info["mime"] = file_info.get("mime") info["size"] = file_info.get("size") @@ -144,7 +144,7 @@ class IwaraExtractor(Extractor): "status" : user.get("status"), "role" : user.get("role"), "premium": user.get("premium"), - "date" : text.parse_datetime( + "date" : self.parse_datetime( user.get("createdAt"), "%Y-%m-%dT%H:%M:%S.000Z"), "description": profile.get("body"), } diff --git a/gallery_dl/extractor/kabeuchi.py b/gallery_dl/extractor/kabeuchi.py index c259c473..49dba3ac 100644 --- a/gallery_dl/extractor/kabeuchi.py +++ b/gallery_dl/extractor/kabeuchi.py @@ -32,7 +32,7 @@ class KabeuchiUserExtractor(Extractor): if post.get("is_ad") or not post["image1"]: continue - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%d %H:%M:%S") yield Message.Directory, post diff --git a/gallery_dl/extractor/kemono.py b/gallery_dl/extractor/kemono.py index b4a8abca..441f92a5 100644 --- a/gallery_dl/extractor/kemono.py +++ b/gallery_dl/extractor/kemono.py @@ -238,7 +238,7 @@ class KemonoExtractor(Extractor): def _parse_datetime(self, date_string): if len(date_string) > 19: date_string = date_string[:19] - return text.parse_datetime(date_string, "%Y-%m-%dT%H:%M:%S") + return self.parse_datetime(date_string, "%Y-%m-%dT%H:%M:%S") def _revisions(self, posts): return itertools.chain.from_iterable( diff --git a/gallery_dl/extractor/lensdump.py b/gallery_dl/extractor/lensdump.py index b0198d54..1f81407d 100644 --- a/gallery_dl/extractor/lensdump.py +++ b/gallery_dl/extractor/lensdump.py @@ -119,7 +119,7 @@ class LensdumpImageExtractor(LensdumpBase, Extractor): 'property="image:width" content="', '"')), "height": text.parse_int(extr( 'property="image:height" content="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '", "<")), "size": text.parse_bytes(extr("", "")), - "date": text.parse_datetime( + "date": self.parse_datetime( extr("", "").strip(), "%Y-%m-%d %H:%M"), }) diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index 16eb6505..aed84b46 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -68,7 +68,7 @@ class MangadexExtractor(Extractor): "chapter" : text.parse_int(chnum), "chapter_minor": f"{sep}{minor}", "chapter_id": chapter["id"], - "date" : text.parse_datetime(cattributes["publishAt"]), + "date" : self.parse_datetime(cattributes["publishAt"]), "group" : [group["attributes"]["name"] for group in relationships["scanlation_group"]], "lang" : lang, @@ -109,8 +109,8 @@ class MangadexCoversExtractor(MangadexExtractor): "cover" : cattributes["fileName"], "lang" : cattributes.get("locale"), "volume" : text.parse_int(cattributes["volume"]), - "date" : text.parse_datetime(cattributes["createdAt"]), - "date_updated": text.parse_datetime(cattributes["updatedAt"]), + "date" : self.parse_datetime(cattributes["createdAt"]), + "date_updated": self.parse_datetime(cattributes["updatedAt"]), } @@ -454,7 +454,7 @@ def _manga_info(self, uuid): "manga_id": manga["id"], "manga_titles": [t.popitem()[1] for t in mattr.get("altTitles") or ()], - "manga_date" : text.parse_datetime(mattr.get("createdAt")), + "manga_date" : self.parse_datetime(mattr.get("createdAt")), "description" : (mattr["description"].get("en") or next(iter(mattr["description"].values()), "")), "demographic": mattr.get("publicationDemographic"), diff --git a/gallery_dl/extractor/mangafox.py b/gallery_dl/extractor/mangafox.py index 76f4b7e8..a5f31e3b 100644 --- a/gallery_dl/extractor/mangafox.py +++ b/gallery_dl/extractor/mangafox.py @@ -99,7 +99,7 @@ class MangafoxMangaExtractor(MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor" : minor or "", "chapter_string": cstr, - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('right">', ''), "%b %d, %Y"), } chapter.update(data) diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index a6948e3b..75ef0ab1 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -50,9 +50,9 @@ class ManganeloChapterExtractor(ManganeloExtractor, ChapterExtractor): extr = text.extract_from(page) data = { - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '"datePublished": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), - "date_updated": text.parse_datetime(extr( + "date_updated": self.parse_datetime(extr( '"dateModified": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), "manga_id" : text.parse_int(extr("comic_id =", ";")), "chapter_id" : text.parse_int(extr("chapter_id =", ";")), @@ -99,7 +99,7 @@ class ManganeloMangaExtractor(ManganeloExtractor, MangaExtractor): manga = text.unescape(extr("

    ", "<")) author = text.remove_html(extr("
  • Author(s) :", "")) status = extr("
  • Status :", "<").strip() - update = text.parse_datetime(extr( + update = self.parse_datetime(extr( "
  • Last updated :", "<").strip(), "%b-%d-%Y %I:%M:%S %p") tags = text.split_html(extr(">Genres :", "
  • "))[::2] @@ -121,7 +121,7 @@ class ManganeloMangaExtractor(ManganeloExtractor, MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor": (sep and ".") + minor, "title" : title.partition(": ")[2], - "date" : text.parse_datetime(date, "%b-%d-%Y %H:%M"), + "date" : self.parse_datetime(date, "%b-%d-%Y %H:%M"), "lang" : "en", "language": "English", })) diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index 19aee33e..b63426e9 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -101,7 +101,7 @@ class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor): "language" : util.code_to_language(lang), "source" : chapter["srcTitle"], "source_id" : chapter["sourceId"], - "date" : text.parse_timestamp(chapter["dateCreate"] // 1000), + "date" : self.parse_timestamp(chapter["dateCreate"] // 1000), } def images(self, _): @@ -138,7 +138,7 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor): "language" : util.code_to_language(lang), "source" : chapter["srcTitle"], "source_id" : chapter["sourceId"], - "date" : text.parse_timestamp( + "date" : self.parse_timestamp( chapter["dateCreate"] // 1000), "_extractor": MangaparkChapterExtractor, } diff --git a/gallery_dl/extractor/mangataro.py b/gallery_dl/extractor/mangataro.py index f4cc0582..77c07743 100644 --- a/gallery_dl/extractor/mangataro.py +++ b/gallery_dl/extractor/mangataro.py @@ -40,9 +40,9 @@ class MangataroChapterExtractor(MangataroBase, ChapterExtractor): "chapter_minor": str(round(minor, 5))[1:] if minor else "", "chapter_id" : text.parse_int(chapter_id), "chapter_url" : comic["url"], - "date" : text.parse_datetime( + "date" : self.parse_datetime( comic["datePublished"], "%Y-%m-%dT%H:%M:%S%z"), - "date_updated" : text.parse_datetime( + "date_updated" : self.parse_datetime( comic["dateModified"], "%Y-%m-%dT%H:%M:%S%z"), } diff --git a/gallery_dl/extractor/mangoxo.py b/gallery_dl/extractor/mangoxo.py index beb13ce5..df9f3c6d 100644 --- a/gallery_dl/extractor/mangoxo.py +++ b/gallery_dl/extractor/mangoxo.py @@ -119,7 +119,7 @@ class MangoxoAlbumExtractor(MangoxoExtractor): "album": { "id": self.album_id, "name": text.unescape(title), - "date": text.parse_datetime(date.strip(), "%Y.%m.%d %H:%M"), + "date": self.parse_datetime(date.strip(), "%Y.%m.%d %H:%M"), "description": text.unescape(descr), }, "count": text.parse_int(count), diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 1bab63aa..03d2ce44 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -64,7 +64,7 @@ class MastodonExtractor(BaseExtractor): status["count"] = len(attachments) status["tags"] = [tag["name"] for tag in status["tags"]] - status["date"] = text.parse_datetime( + status["date"] = self.parse_datetime( status["created_at"][:19], "%Y-%m-%dT%H:%M:%S") yield Message.Directory, status @@ -319,7 +319,7 @@ class MastodonAPI(): if code == 404: raise exception.NotFoundError() if code == 429: - self.extractor.wait(until=text.parse_datetime( + self.extractor.wait(until=self.parse_datetime( response.headers["x-ratelimit-reset"], "%Y-%m-%dT%H:%M:%S.%fZ", )) diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 42eaeef7..f58560c8 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -48,12 +48,12 @@ class MisskeyExtractor(BaseExtractor): note["instance"] = self.instance note["instance_remote"] = note["user"]["host"] note["count"] = len(files) - note["date"] = text.parse_datetime( + note["date"] = self.parse_datetime( note["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") yield Message.Directory, note for note["num"], file in enumerate(files, 1): - file["date"] = text.parse_datetime( + file["date"] = self.parse_datetime( file["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") note["file"] = file url = file["url"] diff --git a/gallery_dl/extractor/naverchzzk.py b/gallery_dl/extractor/naverchzzk.py index de4ee7ac..55fbf75e 100644 --- a/gallery_dl/extractor/naverchzzk.py +++ b/gallery_dl/extractor/naverchzzk.py @@ -31,16 +31,16 @@ class NaverChzzkExtractor(Extractor): data["uid"] = data["objectId"] data["user"] = comment["user"] data["count"] = len(files) - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( data["createdDate"], "%Y%m%d%H%M%S") yield Message.Directory, data for data["num"], file in enumerate(files, 1): if extra := file.get("extraJson"): file.update(util.json_loads(extra)) - file["date"] = text.parse_datetime( + file["date"] = self.parse_datetime( file["createdDate"], "%Y-%m-%dT%H:%M:%S.%f%z") - file["date_updated"] = text.parse_datetime( + file["date_updated"] = self.parse_datetime( file["updatedDate"], "%Y-%m-%dT%H:%M:%S.%f%z") data["file"] = file url = file["attachValue"] diff --git a/gallery_dl/extractor/nekohouse.py b/gallery_dl/extractor/nekohouse.py index e6b0461a..77298b0f 100644 --- a/gallery_dl/extractor/nekohouse.py +++ b/gallery_dl/extractor/nekohouse.py @@ -59,7 +59,7 @@ class NekohousePostExtractor(NekohouseExtractor): 'class="scrape__user-name', '")[2].strip()), "title" : text.unescape(extr( 'class="scrape__title', '")[2]), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( 'datetime="', '"')[:19], "%Y-%m-%d %H:%M:%S"), "content": text.unescape(extr( 'class="scrape__content">', "").strip()), diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index ffb4cadb..13a85624 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -218,7 +218,7 @@ class NewgroundsExtractor(Extractor): "description": text.unescape(extr(':description" content="', '"')), "type" : "art", "_type" : "i", - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( 'itemprop="datePublished" content="', '"')), "rating" : extr('class="rated-', '"'), "url" : full('src="', '"'), @@ -268,7 +268,7 @@ class NewgroundsExtractor(Extractor): "description": text.unescape(extr(':description" content="', '"')), "type" : "audio", "_type" : "a", - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( 'itemprop="datePublished" content="', '"')), "url" : extr('{"url":"', '"').replace("\\/", "/"), "index" : text.parse_int(index), @@ -287,7 +287,7 @@ class NewgroundsExtractor(Extractor): src = src.replace("\\/", "/") formats = () type = extr(',"description":"', '"') - date = text.parse_datetime(extr( + date = self.parse_datetime(extr( 'itemprop="datePublished" content="', '"')) if type: type = type.rpartition(" ")[2].lower() @@ -302,7 +302,7 @@ class NewgroundsExtractor(Extractor): sources = self.request_json(url, headers=headers)["sources"] formats = self._video_formats(sources) src = next(formats, "") - date = text.parse_timestamp(src.rpartition("?")[2]) + date = self.parse_timestamp(src.rpartition("?")[2]) type = "movie" return { diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index c6df835f..69f519ba 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -82,7 +82,7 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor): "title" : keywords[0].strip(), "description": text.unescape(extr( '"description": "', '"').replace("&", "&")), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '"datePublished": "', '"'), "%a %b %d %H:%M:%S %Y", 9), "artist_id" : text.parse_int(extr('/members.php?id=', '"')), "artist_name": keywords[1], @@ -101,7 +101,7 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor): "artist_id" : text.parse_int(extr('members.php?id=', '"')), "artist_name": keywords[1], "tags" : keywords[2:-1], - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "itemprop='datePublished' content=", "<").rpartition(">")[2], "%Y-%m-%d %H:%M:%S", 9), } diff --git a/gallery_dl/extractor/nitter.py b/gallery_dl/extractor/nitter.py index 69d82999..259d8b65 100644 --- a/gallery_dl/extractor/nitter.py +++ b/gallery_dl/extractor/nitter.py @@ -114,7 +114,7 @@ class NitterExtractor(BaseExtractor): return { "author" : author, "user" : self.user_obj or author, - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('title="', '"'), "%b %d, %Y · %I:%M %p %Z"), "tweet_id": link.rpartition("/")[2].partition("#")[0], "content": extr('class="tweet-content', "")[2], @@ -142,7 +142,7 @@ class NitterExtractor(BaseExtractor): return { "author" : author, "user" : self.user_obj or author, - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('title="', '"'), "%b %d, %Y · %I:%M %p %Z"), "tweet_id": link.rpartition("/")[2].partition("#")[0], "content" : extr('class="quote-text', "")[2], @@ -173,7 +173,7 @@ class NitterExtractor(BaseExtractor): "nick" : extr('title="', '"'), "name" : extr('title="@', '"'), "description" : extr('

    ', '<'), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('class="profile-joindate">Source Link<", ""), "href='", "'")), @@ -133,7 +133,7 @@ class PahealTagExtractor(PahealExtractor): "duration" : text.parse_float(duration[:-1]), "tags" : text.unescape(tags), "size" : text.parse_bytes(size[:-1]), - "date" : text.parse_datetime(date, "%B %d, %Y; %H:%M"), + "date" : self.parse_datetime(date, "%B %d, %Y; %H:%M"), "filename" : f"{pid} - {tags}", "extension": ext, } diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 552b3856..b4f0cdc4 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -177,7 +177,7 @@ class PatreonExtractor(Extractor): post, included, "attachments") attr["attachments_media"] = self._files( post, included, "attachments_media") - attr["date"] = text.parse_datetime( + attr["date"] = self.parse_datetime( attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") try: @@ -226,7 +226,7 @@ class PatreonExtractor(Extractor): user = response.json()["data"] attr = user["attributes"] attr["id"] = user["id"] - attr["date"] = text.parse_datetime( + attr["date"] = self.parse_datetime( attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z") return attr @@ -236,7 +236,7 @@ class PatreonExtractor(Extractor): coll = data["data"] attr = coll["attributes"] attr["id"] = coll["id"] - attr["date"] = text.parse_datetime( + attr["date"] = self.parse_datetime( attr["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") return attr diff --git a/gallery_dl/extractor/pexels.py b/gallery_dl/extractor/pexels.py index f95d4099..ca482cc6 100644 --- a/gallery_dl/extractor/pexels.py +++ b/gallery_dl/extractor/pexels.py @@ -35,7 +35,7 @@ class PexelsExtractor(Extractor): post["type"] = attr["type"] post.update(metadata) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"][:-5], "%Y-%m-%dT%H:%M:%S") if "image" in post: diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index 8891dc06..9407acd2 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -36,7 +36,7 @@ class PhilomenaExtractor(BooruExtractor): return url def _prepare(self, post): - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"][:19], "%Y-%m-%dT%H:%M:%S") diff --git a/gallery_dl/extractor/photovogue.py b/gallery_dl/extractor/photovogue.py index e6043044..6b38a983 100644 --- a/gallery_dl/extractor/photovogue.py +++ b/gallery_dl/extractor/photovogue.py @@ -29,7 +29,7 @@ class PhotovogueUserExtractor(Extractor): for photo in self.photos(): url = photo["gallery_image"] photo["title"] = photo["title"].strip() - photo["date"] = text.parse_datetime( + photo["date"] = self.parse_datetime( photo["date"], "%Y-%m-%dT%H:%M:%S.%f%z") yield Message.Directory, photo diff --git a/gallery_dl/extractor/picarto.py b/gallery_dl/extractor/picarto.py index 62ac38ac..a5168aa6 100644 --- a/gallery_dl/extractor/picarto.py +++ b/gallery_dl/extractor/picarto.py @@ -29,7 +29,7 @@ class PicartoGalleryExtractor(Extractor): def items(self): for post in self.posts(): - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%d %H:%M:%S") variations = post.pop("variations", ()) yield Message.Directory, post diff --git a/gallery_dl/extractor/piczel.py b/gallery_dl/extractor/piczel.py index 968776b4..becdb327 100644 --- a/gallery_dl/extractor/piczel.py +++ b/gallery_dl/extractor/piczel.py @@ -26,7 +26,7 @@ class PiczelExtractor(Extractor): def items(self): for post in self.posts(): post["tags"] = [t["title"] for t in post["tags"] if t["title"]] - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") if post["multi"]: diff --git a/gallery_dl/extractor/pillowfort.py b/gallery_dl/extractor/pillowfort.py index 05bc8e7f..ed535180 100644 --- a/gallery_dl/extractor/pillowfort.py +++ b/gallery_dl/extractor/pillowfort.py @@ -48,7 +48,7 @@ class PillowfortExtractor(Extractor): for url in inline(post["content"]): files.append({"url": url}) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") post["post_id"] = post.pop("id") post["count"] = len(files) @@ -76,7 +76,7 @@ class PillowfortExtractor(Extractor): if "id" not in file: post["id"] = post["hash"] if "created_at" in file: - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( file["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") yield msgtype, url, post diff --git a/gallery_dl/extractor/pixeldrain.py b/gallery_dl/extractor/pixeldrain.py index 73f4b1fb..78b004c7 100644 --- a/gallery_dl/extractor/pixeldrain.py +++ b/gallery_dl/extractor/pixeldrain.py @@ -24,8 +24,8 @@ class PixeldrainExtractor(Extractor): if api_key := self.config("api-key"): self.session.auth = util.HTTPBasicAuth("", api_key) - def parse_datetime(self, date_string): - return text.parse_datetime( + def _parse_datetime(self, date_string): + return self.parse_datetime( date_string, "%Y-%m-%dT%H:%M:%S.%fZ") @@ -45,7 +45,7 @@ class PixeldrainFileExtractor(PixeldrainExtractor): file = self.request_json(url + "/info") file["url"] = url + "?download" - file["date"] = self.parse_datetime(file["date_upload"]) + file["date"] = self.parse_datetime_iso(file["date_upload"]) text.nameext_from_url(file["name"], file) yield Message.Directory, file @@ -72,7 +72,7 @@ class PixeldrainAlbumExtractor(PixeldrainExtractor): files = album["files"] album["count"] = album["file_count"] - album["date"] = self.parse_datetime(album["date_created"]) + album["date"] = self.parse_datetime_iso(album["date_created"]) if self.file_index: idx = text.parse_int(self.file_index) @@ -91,7 +91,7 @@ class PixeldrainAlbumExtractor(PixeldrainExtractor): file["album"] = album file["num"] = num file["url"] = url = f"{self.root}/api/file/{file['id']}?download" - file["date"] = self.parse_datetime(file["date_upload"]) + file["date"] = self.parse_datetime_iso(file["date_upload"]) text.nameext_from_url(file["name"], file) yield Message.Url, url, file @@ -112,7 +112,7 @@ class PixeldrainFolderExtractor(PixeldrainExtractor): "mime_type" : data["file_type"], "size" : data["file_size"], "hash_sha256": data["sha256_sum"], - "date" : self.parse_datetime(data["created"]), + "date" : self.parse_datetime_iso(data["created"]), } def items(self): diff --git a/gallery_dl/extractor/pornhub.py b/gallery_dl/extractor/pornhub.py index 12113972..38905351 100644 --- a/gallery_dl/extractor/pornhub.py +++ b/gallery_dl/extractor/pornhub.py @@ -150,7 +150,7 @@ class PornhubGifExtractor(PornhubExtractor): "tags" : extr("data-context-tag='", "'").split(","), "title": extr('"name": "', '"'), "url" : extr('"contentUrl": "', '"'), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('"uploadDate": "', '"'), "%Y-%m-%d"), "viewkey" : extr('From this video: ' '')) - date = text.parse_datetime(extr( + date = self.parse_datetime(extr( '')) username = extr( '') diff --git a/gallery_dl/extractor/rawkuma.py b/gallery_dl/extractor/rawkuma.py index 242486d7..b9a12a0a 100644 --- a/gallery_dl/extractor/rawkuma.py +++ b/gallery_dl/extractor/rawkuma.py @@ -42,7 +42,7 @@ class RawkumaChapterExtractor(RawkumaBase, ChapterExtractor): "chapter_minor": sep + minor, "chapter_id" : text.parse_int(item["cid"]), "title" : text.unescape(title), - "date" : text.parse_datetime( + "date" : self.parse_datetime( date, "%Y-%m-%dWIB%H:%M:%S%z"), "thumbnail" : item.get("t"), "lang" : "ja", diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index 483a5ba0..74b700fb 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -97,7 +97,7 @@ class ReactorExtractor(BaseExtractor): return num = 0 - date = text.parse_datetime(data["datePublished"]) + date = self.parse_datetime(data["datePublished"]) user = data["author"]["name"] description = text.unescape(data["description"]) title, _, tags = text.unescape(data["headline"]).partition(" / ") diff --git a/gallery_dl/extractor/realbooru.py b/gallery_dl/extractor/realbooru.py index cf45578c..d1cf0244 100644 --- a/gallery_dl/extractor/realbooru.py +++ b/gallery_dl/extractor/realbooru.py @@ -48,7 +48,7 @@ class RealbooruExtractor(booru.BooruExtractor): return num def _prepare(self, post): - post["date"] = text.parse_datetime(post["created_at"], "%b, %d %Y") + post["date"] = self.parse_datetime(post["created_at"], "%b, %d %Y") def _pagination(self, params, begin, end): url = self.root + "/index.php" diff --git a/gallery_dl/extractor/redbust.py b/gallery_dl/extractor/redbust.py index d00ed526..3312817f 100644 --- a/gallery_dl/extractor/redbust.py +++ b/gallery_dl/extractor/redbust.py @@ -90,7 +90,7 @@ class RedbustGalleryExtractor(GalleryExtractor, RedbustExtractor): "categories" : text.split_html(extr( '

  • ', "
  • "))[::2], "title" : text.unescape(extr('class="post-title">', "<")), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('class="post-byline">', "<").strip(), "%B %d, %Y"), "views" : text.parse_int(extr("", "v").replace(",", "")), "tags" : text.split_html(extr( diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index e20d80e4..158f66f1 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -57,7 +57,7 @@ class RedditExtractor(Extractor): if submission: submission["comment"] = None - submission["date"] = text.parse_timestamp( + submission["date"] = self.parse_timestamp( submission["created_utc"]) yield Message.Directory, submission visited.add(submission["id"]) @@ -124,7 +124,7 @@ class RedditExtractor(Extractor): data = submission.copy() data["comment"] = comment - comment["date"] = text.parse_timestamp( + comment["date"] = self.parse_timestamp( comment["created_utc"]) if media: diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index 4098c54b..304ef5c8 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -51,7 +51,7 @@ class RedgifsExtractor(Extractor): gif.update(metadata) gif["count"] = cnt - gif["date"] = text.parse_timestamp(gif.get("createDate")) + gif["date"] = self.parse_timestamp(gif.get("createDate")) yield Message.Directory, gif for num, gif in enumerate(gifs, enum): diff --git a/gallery_dl/extractor/rule34vault.py b/gallery_dl/extractor/rule34vault.py index 14d5aefa..b5681207 100644 --- a/gallery_dl/extractor/rule34vault.py +++ b/gallery_dl/extractor/rule34vault.py @@ -36,7 +36,7 @@ class Rule34vaultExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") if "tags" in post: post["tags"] = [t["value"] for t in post["tags"]] diff --git a/gallery_dl/extractor/rule34xyz.py b/gallery_dl/extractor/rule34xyz.py index 05915ba7..bd60d01c 100644 --- a/gallery_dl/extractor/rule34xyz.py +++ b/gallery_dl/extractor/rule34xyz.py @@ -68,7 +68,7 @@ class Rule34xyzExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: diff --git a/gallery_dl/extractor/s3ndpics.py b/gallery_dl/extractor/s3ndpics.py index 215f160e..687dbe38 100644 --- a/gallery_dl/extractor/s3ndpics.py +++ b/gallery_dl/extractor/s3ndpics.py @@ -30,9 +30,9 @@ class S3ndpicsExtractor(Extractor): for post in self.posts(): post["id"] = post.pop("_id", None) post["user"] = post.pop("userId", None) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") - post["date_updated"] = text.parse_datetime( + post["date_updated"] = self.parse_datetime( post["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") files = post.pop("files", ()) diff --git a/gallery_dl/extractor/saint.py b/gallery_dl/extractor/saint.py index 07d490a4..9618b7d2 100644 --- a/gallery_dl/extractor/saint.py +++ b/gallery_dl/extractor/saint.py @@ -36,7 +36,7 @@ class SaintAlbumExtractor(LolisafeAlbumExtractor): break files.append({ "id2" : id2, - "date" : text.parse_timestamp(extr("", ".")), + "date" : self.parse_timestamp(extr("", ".")), "id" : extr("/embed/", '"'), "size" : text.parse_int(extr('data="', '"')), "file" : text.unescape(extr( @@ -73,7 +73,7 @@ class SaintMediaExtractor(SaintAlbumExtractor): file = { "id" : album_id, "id2" : extr("/thumbs/", "-"), - "date" : text.parse_timestamp(extr("", ".")), + "date" : self.parse_timestamp(extr("", ".")), "file" : text.unescape(extr('', '') diff --git a/gallery_dl/extractor/schalenetwork.py b/gallery_dl/extractor/schalenetwork.py index a4ef3b09..3f66e00d 100644 --- a/gallery_dl/extractor/schalenetwork.py +++ b/gallery_dl/extractor/schalenetwork.py @@ -126,7 +126,7 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor): data = self.request_json(url, headers=headers) try: - data["date"] = text.parse_timestamp(data["created_at"] // 1000) + data["date"] = self.parse_timestamp(data["created_at"] // 1000) data["count"] = len(data["thumbnails"]["entries"]) del data["thumbnails"] except Exception: diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index 73197319..5f0ab7f7 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -213,7 +213,7 @@ class SeigaImageExtractor(SeigaExtractor): data["description"] = text.remove_html(data["description"]) data["image_id"] = text.parse_int(self.image_id) - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( data["date"] + ":00+0900", "%Y年%m月%d日 %H:%M:%S%z") return (data, data) diff --git a/gallery_dl/extractor/simpcity.py b/gallery_dl/extractor/simpcity.py index d8227faf..90088ab0 100644 --- a/gallery_dl/extractor/simpcity.py +++ b/gallery_dl/extractor/simpcity.py @@ -98,7 +98,7 @@ class SimpcityExtractor(Extractor): "id" : url_t[url_t.rfind(".")+1:-1], "url" : url_t, "title": schema["headline"], - "date" : text.parse_datetime(schema["datePublished"]), + "date" : self.parse_datetime(schema["datePublished"]), "views": stats[0]["userInteractionCount"], "posts": stats[1]["userInteractionCount"], "tags" : (schema["keywords"].split(", ") @@ -119,7 +119,7 @@ class SimpcityExtractor(Extractor): "author": extr('data-author="', '"'), "id": extr('data-content="post-', '"'), "author_url": extr('itemprop="url" content="', '"'), - "date": text.parse_datetime(extr('datetime="', '"')), + "date": self.parse_datetime(extr('datetime="', '"')), "content": extr('
    ', '
    Characters
    ', '
    ')), "tags" : split(extr('box-title">Tags', '')), "artist" : split(extr('box-title">Artists', '')), - "date" : text.parse_datetime(text.remove_html( + "date" : self.parse_datetime(text.remove_html( extr('Uploaded', '')), "%d.%m.%Y"), } data["lang"] = util.language_to_code(data["language"]) @@ -152,7 +152,7 @@ class SimplyhentaiVideoExtractor(Extractor): "episode": text.parse_int(episode), "tags": text.split_html(tags)[::2], "type": "video", - "date": text.parse_datetime(text.remove_html( + "date": self.parse_datetime(text.remove_html( date), "%B %d, %Y %H:%M"), }) diff --git a/gallery_dl/extractor/sizebooru.py b/gallery_dl/extractor/sizebooru.py index cad4b23a..00002b84 100644 --- a/gallery_dl/extractor/sizebooru.py +++ b/gallery_dl/extractor/sizebooru.py @@ -45,9 +45,9 @@ class SizebooruExtractor(BooruExtractor): post.update({ "id" : text.parse_int(post_id), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr("Posted Date: ", "<"), "%m/%d/%Y"), - "date_approved": text.parse_datetime( + "date_approved": self.parse_datetime( extr("Approved Date: ", "<"), "%m/%d/%Y"), "approver" : text.remove_html(extr("Approved By:", "Posted By:", "', "

"), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "
  • Posted: ", "<"), "%Y-%m-%d"), } diff --git a/gallery_dl/extractor/tiktok.py b/gallery_dl/extractor/tiktok.py index f4508066..c80cdd33 100644 --- a/gallery_dl/extractor/tiktok.py +++ b/gallery_dl/extractor/tiktok.py @@ -43,7 +43,7 @@ class TiktokExtractor(Extractor): post = video_detail["itemInfo"]["itemStruct"] post["user"] = (a := post.get("author")) and a["uniqueId"] or "" - post["date"] = text.parse_timestamp(post["createTime"]) + post["date"] = self.parse_timestamp(post["createTime"]) original_title = title = post["desc"] yield Message.Directory, post diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 7add79ac..0963cd63 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -51,7 +51,7 @@ class ToyhouseExtractor(Extractor): extr = text.extract_from(post) return { "url": extr(needle, '"'), - "date": text.parse_datetime(extr( + "date": self.parse_datetime(extr( '\n
    ', '<'), "%d %b %Y, %I:%M:%S %p"), "artists": [ diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index 8732c604..1ccdafb1 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -65,7 +65,7 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): "title_jp" : title_jp, "thumbnail" : extr('"og:image" content="', '"'), "uploader" : text.remove_html(extr('id="Uploader">', '
    ')), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr('id="Uploaded">', '').strip(), "%Y %B %d"), "rating" : text.parse_float(extr( 'id="Rating">', '').partition(" ")[0]), diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index beaa5c77..f68808b8 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -99,7 +99,7 @@ class TumblrExtractor(Extractor): if "trail" in post: del post["trail"] - post["date"] = text.parse_timestamp(post["timestamp"]) + post["date"] = self.parse_timestamp(post["timestamp"]) posts = [] if "photos" in post: # type "photo" or "link" diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py index 45836a95..e53cfaa2 100644 --- a/gallery_dl/extractor/tungsten.py +++ b/gallery_dl/extractor/tungsten.py @@ -23,7 +23,7 @@ class TungstenExtractor(Extractor): def items(self): for post in self.posts(): url = post["original_url"] - post["date"] = text.parse_datetime(post["created_at"]) + post["date"] = self.parse_datetime(post["created_at"]) post["filename"] = url[url.rfind("/")+1:] post["extension"] = "webp" yield Message.Directory, post diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index 4f9fe845..ec862fe2 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -37,7 +37,7 @@ class TwibooruExtractor(BooruExtractor): return post["view_url"] def _prepare(self, post): - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ") if "name" in post: @@ -146,7 +146,7 @@ class TwibooruAPI(): return response.json() if response.status_code == 429: - until = text.parse_datetime( + until = self.parse_datetime( response.headers["X-RL-Reset"], "%Y-%m-%d %H:%M:%S %Z") # wait an extra minute, just to be safe self.extractor.wait(until=until, adjust=60.0) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index bf125a60..7f0d21f3 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -363,11 +363,11 @@ class TwitterExtractor(Extractor): tweet_id = int(legacy["id_str"]) if tweet_id >= 300000000000000: - date = text.parse_timestamp( + date = self.parse_timestamp( ((tweet_id >> 22) + 1288834974657) // 1000) else: try: - date = text.parse_datetime( + date = self.parse_datetime( legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") except Exception: date = util.NONE @@ -455,7 +455,7 @@ class TwitterExtractor(Extractor): tdata, legacy["extended_entities"]["media"][0]) if tdata["retweet_id"]: tdata["content"] = f"RT @{author['name']}: {tdata['content']}" - tdata["date_original"] = text.parse_timestamp( + tdata["date_original"] = self.parse_timestamp( ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) return tdata @@ -492,7 +492,7 @@ class TwitterExtractor(Extractor): "id": text.parse_int(cid), "name": com.get("name"), "description": com.get("description"), - "date": text.parse_timestamp(com.get("created_at", 0) // 1000), + "date": self.parse_timestamp(com.get("created_at", 0) // 1000), "nsfw": com.get("is_nsfw"), "role": com.get("role"), "member_count": com.get("member_count"), @@ -529,7 +529,7 @@ class TwitterExtractor(Extractor): "name" : core.get("screen_name"), "nick" : core.get("name"), "location" : user["location"]["location"], - "date" : text.parse_datetime( + "date" : self.parse_datetime( core["created_at"], "%a %b %d %H:%M:%S %z %Y"), "verified" : user["verification"]["verified"], "protected" : user["privacy"]["protected"], @@ -898,7 +898,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) - tdata["date_bookmarked"] = text.parse_timestamp( + tdata["date_bookmarked"] = self.parse_timestamp( (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index cf6631fc..9d9702af 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -41,7 +41,7 @@ class UnsplashExtractor(Extractor): if metadata: photo.update(metadata) photo["extension"] = "jpg" - photo["date"] = text.parse_datetime(photo["created_at"]) + photo["date"] = self.parse_datetime(photo["created_at"]) if "tags" in photo: photo["tags"] = [t["title"] for t in photo["tags"]] diff --git a/gallery_dl/extractor/urlgalleries.py b/gallery_dl/extractor/urlgalleries.py index 4369ac63..53e348e4 100644 --- a/gallery_dl/extractor/urlgalleries.py +++ b/gallery_dl/extractor/urlgalleries.py @@ -52,7 +52,7 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor): "blog" : text.unescape(extr(' title="', '"')), "_rprt": extr(' title="', '"'), # report button "title": text.unescape(extr(' title="', '"').strip()), - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr(" images in gallery | ", "<"), "%B %d, %Y"), } diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index e0107f36..e0834b62 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -47,7 +47,7 @@ class VanillarockPostExtractor(VanillarockExtractor): "count": len(imgs), "title": text.unescape(name), "path" : self.path.strip("/"), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( '
    ', '
    '), "%Y-%m-%d %H:%M"), "tags" : text.split_html(extr( '
    ', '
    '))[::2], diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 22d4b9ab..2ef24a8a 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -72,7 +72,7 @@ class VkExtractor(Extractor): photo["width"] = photo["height"] = 0 photo["id"] = photo["id"].rpartition("_")[2] - photo["date"] = text.parse_timestamp(text.extr( + photo["date"] = self.parse_timestamp(text.extr( photo["date"], 'data-date="', '"')) photo["description"] = text.unescape(text.extr( photo.get("desc", ""), ">", "<")) diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index df09fce2..19464c22 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -62,7 +62,7 @@ class VscoExtractor(Extractor): "grid" : img["grid_name"], "meta" : img.get("image_meta") or {}, "tags" : [tag["text"] for tag in img.get("tags") or ()], - "date" : text.parse_timestamp(img["upload_date"] // 1000), + "date" : self.parse_timestamp(img["upload_date"] // 1000), "video" : img["is_video"], "width" : img["width"], "height": img["height"], diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index f0f27e0c..861d0c99 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -43,7 +43,7 @@ class WallhavenExtractor(Extractor): wp["url"] = wp.pop("path") if "tags" in wp: wp["tags"] = [t["name"] for t in wp["tags"]] - wp["date"] = text.parse_datetime( + wp["date"] = self.parse_datetime( wp.pop("created_at"), "%Y-%m-%d %H:%M:%S") wp["width"] = wp.pop("dimension_x") wp["height"] = wp.pop("dimension_y") diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index 8ae2a49a..5463448f 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -42,7 +42,7 @@ class WarosuThreadExtractor(Extractor): if "image" in post: for key in ("w", "h", "no", "time", "tim"): post[key] = text.parse_int(post[key]) - dt = text.parse_timestamp(post["time"]) + dt = self.parse_timestamp(post["time"]) # avoid zero-padding 'day' with %d post["now"] = dt.strftime(f"%a, %b {dt.day}, %Y %H:%M:%S") post.update(data) diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index a69f3a85..48c26f77 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -24,7 +24,7 @@ class WeasylExtractor(Extractor): # Some submissions don't have content and can be skipped if "submission" in data["media"]: data["url"] = data["media"]["submission"][0]["url"] - data["date"] = text.parse_datetime( + data["date"] = self.parse_datetime( data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") text.nameext_from_url(data["url"], data) return True @@ -42,7 +42,7 @@ class WeasylExtractor(Extractor): f"{self.root}/api/journals/{journalid}/view") data["extension"] = "html" data["html"] = "text:" + data["content"] - data["date"] = text.parse_datetime(data["posted_at"]) + data["date"] = self.parse_datetime(data["posted_at"]) return data def submissions(self, owner_login, folderid=None): diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index cc41b039..10b5aa51 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -40,7 +40,7 @@ class WebmshareVideoExtractor(Extractor): 'property="og:video:width" content="', '"')), "height": text.parse_int(extr( 'property="og:video:height" content="', '"')), - "date" : text.parse_datetime(extr( + "date" : self.parse_datetime(extr( "Added ", "<"), "%B %d, %Y"), "views": text.parse_int(extr('glyphicon-eye-open">', '<')), "id" : self.video_id, diff --git a/gallery_dl/extractor/weebcentral.py b/gallery_dl/extractor/weebcentral.py index 03cbf293..cddceb76 100644 --- a/gallery_dl/extractor/weebcentral.py +++ b/gallery_dl/extractor/weebcentral.py @@ -127,7 +127,7 @@ class WeebcentralMangaExtractor(WeebcentralBase, MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor": sep + minor, "chapter_type" : type, - "date" : text.parse_datetime( + "date" : self.parse_datetime( extr(' datetime="', '"')[:-5], "%Y-%m-%dT%H:%M:%S"), } chapter.update(data) diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 3c0f0776..023f92e0 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -98,7 +98,7 @@ class WeiboExtractor(Extractor): files = [] self._extract_status(status, files) - status["date"] = text.parse_datetime( + status["date"] = self.parse_datetime( status["created_at"], "%a %b %d %H:%M:%S %z %Y") status["count"] = len(files) yield Message.Directory, status diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 31dc9cd1..7c168e9e 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -34,7 +34,7 @@ class WikifeetGalleryExtractor(GalleryExtractor): "celeb" : self.celeb, "type" : self.type, "birthplace": text.unescape(extr('"bplace":"', '"')), - "birthday" : text.parse_datetime(text.unescape( + "birthday" : self.parse_datetime(text.unescape( extr('"bdate":"', '"'))[:10], "%Y-%m-%d"), "shoesize" : text.unescape(extr('"ssize":', ',')), "rating" : text.parse_float(extr('"score":', ',')), diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index ba020d50..83220889 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -75,7 +75,7 @@ class WikimediaExtractor(BaseExtractor): for m in image["commonmetadata"] or ()} text.nameext_from_url(image["canonicaltitle"].partition(":")[2], image) - image["date"] = text.parse_datetime( + image["date"] = self.parse_datetime( image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") def items(self): diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index 6c971754..f5261d66 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -67,7 +67,7 @@ class XhamsterGalleryExtractor(XhamsterExtractor): { "id" : text.parse_int(gallery["id"]), "tags" : [t["label"] for t in info["categoriesTags"]], - "date" : text.parse_timestamp(model["created"]), + "date" : self.parse_timestamp(model["created"]), "views" : text.parse_int(model["views"]), "likes" : text.parse_int(model["rating"]["likes"]), "dislikes" : text.parse_int(model["rating"]["dislikes"]), diff --git a/gallery_dl/extractor/yiffverse.py b/gallery_dl/extractor/yiffverse.py index 1595b4d6..33640fc1 100644 --- a/gallery_dl/extractor/yiffverse.py +++ b/gallery_dl/extractor/yiffverse.py @@ -55,7 +55,7 @@ class YiffverseExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = text.parse_datetime( + post["date"] = self.parse_datetime( post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index 7bff83b4..602f994b 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -76,7 +76,7 @@ class ZerochanExtractor(BooruExtractor): data = { "id" : text.parse_int(entry_id), "file_url": jsonld["contentUrl"], - "date" : text.parse_datetime(jsonld["datePublished"]), + "date" : self.parse_datetime(jsonld["datePublished"]), "width" : text.parse_int(jsonld["width"][:-3]), "height" : text.parse_int(jsonld["height"][:-3]), "size" : text.parse_bytes(jsonld["contentSize"][:-1]), diff --git a/gallery_dl/text.py b/gallery_dl/text.py index 98bba484..e1a27ef5 100644 --- a/gallery_dl/text.py +++ b/gallery_dl/text.py @@ -8,10 +8,7 @@ """Collection of functions that work on strings/text""" -import sys import html -import time -import datetime import urllib.parse import re as re_module @@ -322,46 +319,6 @@ def build_query(params): ]) -if sys.hexversion < 0x30c0000: - # Python <= 3.11 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - return datetime.datetime.utcfromtimestamp(int(ts)) - except Exception: - return default -else: - # Python >= 3.12 - def parse_timestamp(ts, default=None): - """Create a datetime object from a Unix timestamp""" - try: - Y, m, d, H, M, S, _, _, _ = time.gmtime(int(ts)) - return datetime.datetime(Y, m, d, H, M, S) - except Exception: - return default - - -def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0): - """Create a datetime object by parsing 'date_string'""" - try: - d = datetime.datetime.strptime(date_string, format) - o = d.utcoffset() - if o is not None: - # convert to naive UTC - d = d.replace(tzinfo=None, microsecond=0) - o - else: - if d.microsecond: - d = d.replace(microsecond=0) - if utcoffset: - # apply manual UTC offset - d += datetime.timedelta(0, utcoffset * -3600) - return d - except (TypeError, IndexError, KeyError): - return None - except (ValueError, OverflowError): - return date_string - - urljoin = urllib.parse.urljoin quote = urllib.parse.quote From 6c71b279b690980bce43cd41d7a610f419c0554f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 17 Oct 2025 22:49:41 +0200 Subject: [PATCH 09/13] [dt] update 'parse_datetime' calls with one argument --- gallery_dl/dt.py | 4 ++-- gallery_dl/extractor/4archive.py | 5 ++--- gallery_dl/extractor/bellazon.py | 6 +++--- gallery_dl/extractor/blogger.py | 4 ++-- gallery_dl/extractor/cien.py | 2 +- gallery_dl/extractor/deviantart.py | 2 +- gallery_dl/extractor/discord.py | 4 +--- gallery_dl/extractor/fanbox.py | 2 +- gallery_dl/extractor/hatenablog.py | 2 +- gallery_dl/extractor/hentaifoundry.py | 2 +- gallery_dl/extractor/imgur.py | 4 ++-- gallery_dl/extractor/livedoor.py | 2 +- gallery_dl/extractor/mangadex.py | 8 ++++---- gallery_dl/extractor/mastodon.py | 9 +++------ gallery_dl/extractor/newgrounds.py | 8 ++++---- gallery_dl/extractor/nijie.py | 13 +++++++------ gallery_dl/extractor/postmill.py | 2 +- gallery_dl/extractor/reactor.py | 2 +- gallery_dl/extractor/sankakucomplex.py | 2 +- gallery_dl/extractor/simpcity.py | 4 ++-- gallery_dl/extractor/tapas.py | 2 +- gallery_dl/extractor/telegraph.py | 5 ++--- gallery_dl/extractor/tungsten.py | 2 +- gallery_dl/extractor/unsplash.py | 2 +- gallery_dl/extractor/weasyl.py | 5 ++--- gallery_dl/extractor/zerochan.py | 2 +- 26 files changed, 49 insertions(+), 56 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index 9f1cd09e..3f803171 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -74,7 +74,7 @@ if sys.hexversion < 0x30c0000: except Exception: return NONE - def parse_compat(dt_string, format): + def parse_compat(dt_string, format="%Y-%m-%dT%H:%M:%S%z"): """Parse 'dt_string' as ISO 8601 value using 'format'""" return parse(dt_string, format) @@ -90,7 +90,7 @@ else: except Exception: return NONE - def parse_compat(dt_string, format): + def parse_compat(dt_string, format=None): """Parse 'dt_string' as ISO 8601 value""" return parse_iso(dt_string) diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 18948907..a8bb5d9c 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -61,10 +61,9 @@ class _4archiveThreadExtractor(Extractor): extr = text.extract_from(post) data = { "name": extr('class="name">', ""), - "date": self.parse_datetime( + "date": self.parse_datetime_iso( (extr('class="dateTime">', "<") or - extr('class="dateTime postNum" >', "<")).strip(), - "%Y-%m-%d %H:%M:%S"), + extr('class="dateTime postNum" >', "<")).strip()), "no" : text.parse_int(extr(">Post No.", "<")), } if 'class="file"' in post: diff --git a/gallery_dl/extractor/bellazon.py b/gallery_dl/extractor/bellazon.py index 0e557cb6..44592b7a 100644 --- a/gallery_dl/extractor/bellazon.py +++ b/gallery_dl/extractor/bellazon.py @@ -141,8 +141,8 @@ class BellazonExtractor(Extractor): "title": schema["headline"], "views": stats[0]["userInteractionCount"], "posts": stats[1]["userInteractionCount"], - "date" : self.parse_datetime(schema["datePublished"]), - "date_updated": self.parse_datetime(schema["dateModified"]), + "date" : self.parse_datetime_iso(schema["datePublished"]), + "date_updated": self.parse_datetime_iso(schema["dateModified"]), "description" : text.unescape(schema["text"]).strip(), "section" : path[-2], "author" : author["name"], @@ -162,7 +162,7 @@ class BellazonExtractor(Extractor): post = { "id": extr('id="elComment_', '"'), "author_url": extr(" href='", "'"), - "date": self.parse_datetime(extr("datetime='", "'")), + "date": self.parse_datetime_iso(extr("datetime='", "'")), "content": extr("", "\n\t\t"), } diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index d2395daa..33d0ed39 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -40,7 +40,7 @@ class BloggerExtractor(BaseExtractor): blog = self.api.blog_by_url("http://" + self.blog) blog["pages"] = blog["pages"]["totalItems"] blog["posts"] = blog["posts"]["totalItems"] - blog["date"] = self.parse_datetime(blog["published"]) + blog["date"] = self.parse_datetime_iso(blog["published"]) del blog["selfLink"] findall_image = util.re( @@ -65,7 +65,7 @@ class BloggerExtractor(BaseExtractor): post["author"] = post["author"]["displayName"] post["replies"] = post["replies"]["totalItems"] post["content"] = text.remove_html(content) - post["date"] = self.parse_datetime(post["published"]) + post["date"] = self.parse_datetime_iso(post["published"]) del post["selfLink"] del post["blog"] diff --git a/gallery_dl/extractor/cien.py b/gallery_dl/extractor/cien.py index 2679b4e6..ff3474b8 100644 --- a/gallery_dl/extractor/cien.py +++ b/gallery_dl/extractor/cien.py @@ -61,7 +61,7 @@ class CienArticleExtractor(CienExtractor): post["post_url"] = url post["post_id"] = text.parse_int(post_id) post["count"] = len(files) - post["date"] = self.parse_datetime(post["datePublished"]) + post["date"] = self.parse_datetime_iso(post["datePublished"]) try: post["author"]["id"] = text.parse_int(author_id) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index aba5401f..79c88a48 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1187,7 +1187,7 @@ class DeviantartStatusExtractor(DeviantartExtractor): deviation["username"] = deviation["author"]["username"] deviation["_username"] = deviation["username"].lower() - deviation["date"] = d = self.parse_datetime(deviation["ts"]) + deviation["date"] = d = self.parse_datetime_iso(deviation["ts"]) deviation["published_time"] = int(dt.to_ts(d)) deviation["da_category"] = "Status" diff --git a/gallery_dl/extractor/discord.py b/gallery_dl/extractor/discord.py index 4284233e..b5af8760 100644 --- a/gallery_dl/extractor/discord.py +++ b/gallery_dl/extractor/discord.py @@ -72,9 +72,7 @@ class DiscordExtractor(Extractor): "author_files": [], "message": self.extract_message_text(message), "message_id": message["id"], - "date": self.parse_datetime( - message["timestamp"], "%Y-%m-%dT%H:%M:%S.%f%z" - ), + "date": self.parse_datetime_iso(message["timestamp"]), "files": [] }) diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py index f2fbe188..18747b0a 100644 --- a/gallery_dl/extractor/fanbox.py +++ b/gallery_dl/extractor/fanbox.py @@ -128,7 +128,7 @@ class FanboxExtractor(Extractor): if file.get("extension", "").lower() in exts ] - post["date"] = self.parse_datetime(post["publishedDatetime"]) + post["date"] = self.parse_datetime_iso(post["publishedDatetime"]) post["text"] = content_body.get("text") if content_body else None post["isCoverImage"] = False diff --git a/gallery_dl/extractor/hatenablog.py b/gallery_dl/extractor/hatenablog.py index adb31aa1..a6d2d329 100644 --- a/gallery_dl/extractor/hatenablog.py +++ b/gallery_dl/extractor/hatenablog.py @@ -34,7 +34,7 @@ class HatenablogExtractor(Extractor): def _handle_article(self, article: str): extr = text.extract_from(article) - date = self.parse_datetime(extr('
    ', '') diff --git a/gallery_dl/extractor/simpcity.py b/gallery_dl/extractor/simpcity.py index 90088ab0..3a6bf38c 100644 --- a/gallery_dl/extractor/simpcity.py +++ b/gallery_dl/extractor/simpcity.py @@ -98,7 +98,7 @@ class SimpcityExtractor(Extractor): "id" : url_t[url_t.rfind(".")+1:-1], "url" : url_t, "title": schema["headline"], - "date" : self.parse_datetime(schema["datePublished"]), + "date" : self.parse_datetime_iso(schema["datePublished"]), "views": stats[0]["userInteractionCount"], "posts": stats[1]["userInteractionCount"], "tags" : (schema["keywords"].split(", ") @@ -119,7 +119,7 @@ class SimpcityExtractor(Extractor): "author": extr('data-author="', '"'), "id": extr('data-content="post-', '"'), "author_url": extr('itemprop="url" content="', '"'), - "date": self.parse_datetime(extr('datetime="', '"')), + "date": self.parse_datetime_iso(extr('datetime="', '"')), "content": extr('
    ', '
    Date: Sun, 19 Oct 2025 18:26:32 +0200 Subject: [PATCH 10/13] [dt] replace 'parse_compat' with compat workaround --- gallery_dl/dt.py | 26 ++++++-------------------- gallery_dl/extractor/newgrounds.py | 6 +++--- test/test_dt.py | 17 +++++++---------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index 3f803171..a09e028e 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -38,19 +38,10 @@ def convert(value): """Convert 'value' to a naive UTC datetime object""" if not value: return NONE - if isinstance(value, datetime): - return value - - if isinstance(value, str): - try: - if value[-1] == "Z": - # compat for Python < 3.11 - value = value[:-1] - return normalize(datetime.fromisoformat(value)) - except Exception: - pass - + return normalize(value) + if isinstance(value, str) and (dt := parse_iso(value)) is not NONE: + return dt return parse_ts(value) @@ -70,14 +61,13 @@ if sys.hexversion < 0x30c0000: if dt_string[-1] == "Z": # compat for Python < 3.11 dt_string = dt_string[:-1] + elif dt_string[-5] in "+-": + # compat for Python < 3.11 + dt_string = f"{dt_string[:-2]}:{dt_string[-2:]}" return normalize(datetime.fromisoformat(dt_string)) except Exception: return NONE - def parse_compat(dt_string, format="%Y-%m-%dT%H:%M:%S%z"): - """Parse 'dt_string' as ISO 8601 value using 'format'""" - return parse(dt_string, format) - from_ts = datetime.utcfromtimestamp now = datetime.utcnow @@ -90,10 +80,6 @@ else: except Exception: return NONE - def parse_compat(dt_string, format=None): - """Parse 'dt_string' as ISO 8601 value""" - return parse_iso(dt_string) - def from_ts(ts=None): """Convert Unix timestamp to naive UTC datetime""" Y, m, d, H, M, S, _, _, _ = time.gmtime(ts) diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 58e07130..768a115f 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -218,7 +218,7 @@ class NewgroundsExtractor(Extractor): "description": text.unescape(extr(':description" content="', '"')), "type" : "art", "_type" : "i", - "date" : dt.parse_compat(extr( + "date" : dt.parse_iso(extr( 'itemprop="datePublished" content="', '"')), "rating" : extr('class="rated-', '"'), "url" : full('src="', '"'), @@ -268,7 +268,7 @@ class NewgroundsExtractor(Extractor): "description": text.unescape(extr(':description" content="', '"')), "type" : "audio", "_type" : "a", - "date" : dt.parse_compat(extr( + "date" : dt.parse_iso(extr( 'itemprop="datePublished" content="', '"')), "url" : extr('{"url":"', '"').replace("\\/", "/"), "index" : text.parse_int(index), @@ -287,7 +287,7 @@ class NewgroundsExtractor(Extractor): src = src.replace("\\/", "/") formats = () type = extr(',"description":"', '"') - date = dt.parse_compat(extr( + date = dt.parse_iso(extr( 'itemprop="datePublished" content="', '"')) if type: type = type.rpartition(" ")[2].lower() diff --git a/test/test_dt.py b/test/test_dt.py index 1c0aa16b..dc00d4b4 100644 --- a/test/test_dt.py +++ b/test/test_dt.py @@ -43,6 +43,8 @@ class TestDatetime(unittest.TestCase): _assert("2009-12-31T19:00:00.123456-05:00", d) _assert("2010-01-01T00:00:00Z" , d) _assert("2010-01-01T00:00:00.123456Z" , d) + _assert("2009-12-31T19:00:00-0500" , d) + _assert("2009-12-31T19:00:00.123456-0500" , d) _assert(0 , dt.NONE) _assert("" , dt.NONE) @@ -63,11 +65,8 @@ class TestDatetime(unittest.TestCase): self.assertEqual(result, expected, msg=repr(value)) d = datetime.datetime(2010, 1, 1) - _assert("2009-12-31T19:00:00-05" , d) - _assert("2009-12-31T19:00:00-0500" , d) _assert("2009-12-31T19:00:00.123456-05" , d) - _assert("2009-12-31T19:00:00.123456-0500" , d) def test_to_timestamp(self, f=dt.to_ts): self.assertEqual(f(dt.EPOCH), 0.0) @@ -146,20 +145,18 @@ class TestDatetime(unittest.TestCase): f("1970.01.01"), dt.NONE, ) - - for value in ((), [], {}, None, 1, 2.3): - self.assertEqual(f(value), dt.NONE) - - def test_parse_compat(self, f=dt.parse_compat): self.assertEqual( - f("1970-01-01T00:00:00+0000", "%Y-%m-%dT%H:%M:%S%z"), + f("1970-01-01T00:00:00+0000"), dt.EPOCH, ) self.assertEqual( - f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), + f("2019-05-07T21:25:02.753+0900"), datetime.datetime(2019, 5, 7, 12, 25, 2), ) + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value), dt.NONE) + def test_none(self): self.assertFalse(dt.NONE) self.assertIsInstance(dt.NONE, dt.datetime) From 5802107cdf3073f4e4dfbfcb417cf5d9329025a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 19 Oct 2025 18:51:42 +0200 Subject: [PATCH 11/13] [dt] update representation of invalid datetime objects --- gallery_dl/dt.py | 8 +++++++- test/test_dt.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index a09e028e..74e65fe3 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -18,8 +18,14 @@ class NullDatetime(datetime): def __bool__(self): return False + def __str__(self): + return "[Invalid DateTime]" -NONE = NullDatetime(101, 1, 1) + def __format__(self, format_spec): + return f"[Invalid DateTime {format_spec}]" + + +NONE = NullDatetime(1, 1, 1) EPOCH = datetime(1970, 1, 1) SECOND = timedelta(0, 1) diff --git a/test/test_dt.py b/test/test_dt.py index dc00d4b4..02e3ac23 100644 --- a/test/test_dt.py +++ b/test/test_dt.py @@ -160,7 +160,7 @@ class TestDatetime(unittest.TestCase): def test_none(self): self.assertFalse(dt.NONE) self.assertIsInstance(dt.NONE, dt.datetime) - self.assertEqual(str(dt.NONE), "0101-01-01 00:00:00") + self.assertEqual(str(dt.NONE), "[Invalid DateTime]") if __name__ == "__main__": From c38856bd3f5b367d7b7b81850382fa9f87b1dbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 19 Oct 2025 19:31:31 +0200 Subject: [PATCH 12/13] [dt] use 'parse_datetime_iso()' for ISO formats --- gallery_dl/extractor/8muses.py | 3 +-- gallery_dl/extractor/ao3.py | 8 ++++---- gallery_dl/extractor/arcalive.py | 3 +-- gallery_dl/extractor/artstation.py | 3 +-- gallery_dl/extractor/batoto.py | 3 +-- gallery_dl/extractor/bluesky.py | 3 +-- gallery_dl/extractor/booth.py | 3 +-- gallery_dl/extractor/chevereto.py | 6 ++---- gallery_dl/extractor/civitai.py | 18 ++++++------------ gallery_dl/extractor/comick.py | 6 ++---- gallery_dl/extractor/dynastyscans.py | 6 ++---- gallery_dl/extractor/e621.py | 3 +-- gallery_dl/extractor/exhentai.py | 4 ++-- gallery_dl/extractor/furry34.py | 3 +-- gallery_dl/extractor/gelbooru_v01.py | 3 +-- gallery_dl/extractor/girlswithmuscle.py | 5 ++--- gallery_dl/extractor/hentaihand.py | 3 +-- gallery_dl/extractor/hitomi.py | 2 +- gallery_dl/extractor/imagechest.py | 6 ++---- gallery_dl/extractor/imgbb.py | 3 +-- gallery_dl/extractor/inkbunny.py | 8 ++++---- gallery_dl/extractor/issuu.py | 4 ++-- gallery_dl/extractor/itaku.py | 10 ++++------ gallery_dl/extractor/iwara.py | 11 +++++------ gallery_dl/extractor/kabeuchi.py | 3 +-- gallery_dl/extractor/kemono.py | 2 +- gallery_dl/extractor/lensdump.py | 3 +-- gallery_dl/extractor/madokami.py | 3 +-- gallery_dl/extractor/manganelo.py | 8 ++++---- gallery_dl/extractor/mangataro.py | 6 ++---- gallery_dl/extractor/misskey.py | 6 ++---- gallery_dl/extractor/naverchzzk.py | 8 ++++---- gallery_dl/extractor/nekohouse.py | 4 ++-- gallery_dl/extractor/nozomi.py | 7 +++---- gallery_dl/extractor/paheal.py | 3 +-- gallery_dl/extractor/patreon.py | 9 +++------ gallery_dl/extractor/pexels.py | 3 +-- gallery_dl/extractor/philomena.py | 3 +-- gallery_dl/extractor/photovogue.py | 3 +-- gallery_dl/extractor/picarto.py | 3 +-- gallery_dl/extractor/piczel.py | 3 +-- gallery_dl/extractor/pillowfort.py | 6 ++---- gallery_dl/extractor/pixeldrain.py | 4 ---- gallery_dl/extractor/pornhub.py | 3 +-- gallery_dl/extractor/rule34vault.py | 3 +-- gallery_dl/extractor/rule34xyz.py | 3 +-- gallery_dl/extractor/s3ndpics.py | 6 ++---- gallery_dl/extractor/slideshare.py | 4 ++-- gallery_dl/extractor/szurubooru.py | 3 +-- gallery_dl/extractor/thehentaiworld.py | 3 +-- gallery_dl/extractor/twibooru.py | 7 +++---- gallery_dl/extractor/vanillarock.py | 4 ++-- gallery_dl/extractor/wallhaven.py | 3 +-- gallery_dl/extractor/weebcentral.py | 4 ++-- gallery_dl/extractor/wikifeet.py | 4 ++-- gallery_dl/extractor/wikimedia.py | 3 +-- gallery_dl/extractor/yiffverse.py | 3 +-- 57 files changed, 102 insertions(+), 165 deletions(-) diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index dfc34224..7cb6eee9 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -85,8 +85,7 @@ class _8musesAlbumExtractor(Extractor): "parent" : text.parse_int(album["parentId"]), "views" : text.parse_int(album["numberViews"]), "likes" : text.parse_int(album["numberLikes"]), - "date" : self.parse_datetime( - album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), + "date" : self.parse_datetime_iso(album["updatedAt"]), } def _unobfuscate(self, data): diff --git a/gallery_dl/extractor/ao3.py b/gallery_dl/extractor/ao3.py index 49a8fa75..7ee528a4 100644 --- a/gallery_dl/extractor/ao3.py +++ b/gallery_dl/extractor/ao3.py @@ -182,10 +182,10 @@ class Ao3WorkExtractor(Ao3Extractor): extr('
    ', "
    ")), "lang" : extr('
    ', "
    "), - "date" : self.parse_datetime( - extr('
    ', "<"), "%Y-%m-%d"), - "date_completed": self.parse_datetime( - extr('>Completed:
    ', "<"), "%Y-%m-%d"), + "date" : self.parse_datetime_iso(extr( + '
    ', "<")), + "date_completed": self.parse_datetime_iso(extr( + '>Completed:
    ', "<")), "date_updated" : self.parse_timestamp( path.rpartition("updated_at=")[2]), "words" : text.parse_int( diff --git a/gallery_dl/extractor/arcalive.py b/gallery_dl/extractor/arcalive.py index 94e49988..51f54aa8 100644 --- a/gallery_dl/extractor/arcalive.py +++ b/gallery_dl/extractor/arcalive.py @@ -49,8 +49,7 @@ class ArcalivePostExtractor(ArcaliveExtractor): files = self._extract_files(post) post["count"] = len(files) - post["date"] = self.parse_datetime( - post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") + post["date"] = self.parse_datetime_iso(post["createdAt"][:19]) post["post_url"] = post_url = \ f"{self.root}/b/{post['boardSlug']}/{post['id']}" post["_http_headers"] = {"Referer": post_url + "?p=1"} diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index 9a14f5e8..c332ab21 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -126,8 +126,7 @@ class ArtstationExtractor(Extractor): data["title"] = text.unescape(data["title"]) data["description"] = text.unescape(text.remove_html( data["description"])) - data["date"] = self.parse_datetime( - data["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + data["date"] = self.parse_datetime_iso(data["created_at"]) assets = data["assets"] del data["assets"] diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index 7495751b..0d35540c 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -167,8 +167,7 @@ class BatotoMangaExtractor(BatotoBase, MangaExtractor): data["chapter"] = text.parse_int(chapter) data["chapter_minor"] = sep + minor - data["date"] = self.parse_datetime( - extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") + data["date"] = self.parse_datetime_iso(extr('time="', '"')) url = f"{self.root}/title/{href}" results.append((url, data.copy())) diff --git a/gallery_dl/extractor/bluesky.py b/gallery_dl/extractor/bluesky.py index f1fd2eee..d314a28b 100644 --- a/gallery_dl/extractor/bluesky.py +++ b/gallery_dl/extractor/bluesky.py @@ -135,8 +135,7 @@ class BlueskyExtractor(Extractor): post["instance"] = self.instance post["post_id"] = self._pid(post) - post["date"] = self.parse_datetime( - post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") + post["date"] = self.parse_datetime_iso(post["createdAt"][:19]) def _extract_files(self, post): if "embed" not in post: diff --git a/gallery_dl/extractor/booth.py b/gallery_dl/extractor/booth.py index 17679da0..0be05c42 100644 --- a/gallery_dl/extractor/booth.py +++ b/gallery_dl/extractor/booth.py @@ -70,8 +70,7 @@ class BoothItemExtractor(BoothExtractor): url + ".json", headers=headers, interval=False) item["booth_category"] = item.pop("category", None) - item["date"] = self.parse_datetime( - item["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + item["date"] = self.parse_datetime_iso(item["published_at"]) item["tags"] = [t["name"] for t in item["tags"]] shop = item["shop"] diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index 465da8dd..e58a4d7b 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -79,8 +79,7 @@ class CheveretoImageExtractor(CheveretoExtractor): "url" : url, "album": text.remove_html(extr( "Added to ").rpartition(">")[2]), - "date" : self.parse_datetime(extr( - '', "—"), "album": text.remove_html(extr( "Added to ").rpartition(">")[2]), - "date" : self.parse_datetime(extr( - '
    ', '<'), "uploader" : extr('
    ', '
    '), - "date" : self.parse_datetime(extr( - '>Posted:', ''), "%Y-%m-%d %H:%M"), + "date" : self.parse_datetime_iso(extr( + '>Posted:', '')), "parent" : extr( '>Parent:
    "), - "date" : self.parse_datetime(extr( - ' 19: date_string = date_string[:19] - return self.parse_datetime(date_string, "%Y-%m-%dT%H:%M:%S") + return self.parse_datetime_iso(date_string) def _revisions(self, posts): return itertools.chain.from_iterable( diff --git a/gallery_dl/extractor/lensdump.py b/gallery_dl/extractor/lensdump.py index 1f81407d..5435c99f 100644 --- a/gallery_dl/extractor/lensdump.py +++ b/gallery_dl/extractor/lensdump.py @@ -119,8 +119,7 @@ class LensdumpImageExtractor(LensdumpBase, Extractor): 'property="image:width" content="', '"')), "height": text.parse_int(extr( 'property="image:height" content="', '"')), - "date" : self.parse_datetime(extr( - '", "<")), "size": text.parse_bytes(extr("", "")), - "date": self.parse_datetime( - extr("", "").strip(), "%Y-%m-%d %H:%M"), + "date": self.parse_datetime_iso(extr("", "").strip()), }) if self.config("chapter-reverse"): diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index 75ef0ab1..3d88b2d0 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -50,10 +50,10 @@ class ManganeloChapterExtractor(ManganeloExtractor, ChapterExtractor): extr = text.extract_from(page) data = { - "date" : self.parse_datetime(extr( - '"datePublished": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), - "date_updated": self.parse_datetime(extr( - '"dateModified": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), + "date" : self.parse_datetime_iso(extr( + '"datePublished": "', '"')[:19]), + "date_updated": self.parse_datetime_iso(extr( + '"dateModified": "', '"')[:19]), "manga_id" : text.parse_int(extr("comic_id =", ";")), "chapter_id" : text.parse_int(extr("chapter_id =", ";")), "manga" : extr("comic_name =", ";").strip('" '), diff --git a/gallery_dl/extractor/mangataro.py b/gallery_dl/extractor/mangataro.py index 77c07743..029bc2e9 100644 --- a/gallery_dl/extractor/mangataro.py +++ b/gallery_dl/extractor/mangataro.py @@ -40,10 +40,8 @@ class MangataroChapterExtractor(MangataroBase, ChapterExtractor): "chapter_minor": str(round(minor, 5))[1:] if minor else "", "chapter_id" : text.parse_int(chapter_id), "chapter_url" : comic["url"], - "date" : self.parse_datetime( - comic["datePublished"], "%Y-%m-%dT%H:%M:%S%z"), - "date_updated" : self.parse_datetime( - comic["dateModified"], "%Y-%m-%dT%H:%M:%S%z"), + "date" : self.parse_datetime_iso(comic["datePublished"]), + "date_updated" : self.parse_datetime_iso(comic["dateModified"]), } def images(self, page): diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index f58560c8..628f7e3f 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -48,13 +48,11 @@ class MisskeyExtractor(BaseExtractor): note["instance"] = self.instance note["instance_remote"] = note["user"]["host"] note["count"] = len(files) - note["date"] = self.parse_datetime( - note["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") + note["date"] = self.parse_datetime_iso(note["createdAt"]) yield Message.Directory, note for note["num"], file in enumerate(files, 1): - file["date"] = self.parse_datetime( - file["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") + file["date"] = self.parse_datetime_iso(file["createdAt"]) note["file"] = file url = file["url"] yield Message.Url, url, text.nameext_from_url(url, note) diff --git a/gallery_dl/extractor/naverchzzk.py b/gallery_dl/extractor/naverchzzk.py index 55fbf75e..e1dc1362 100644 --- a/gallery_dl/extractor/naverchzzk.py +++ b/gallery_dl/extractor/naverchzzk.py @@ -38,10 +38,10 @@ class NaverChzzkExtractor(Extractor): for data["num"], file in enumerate(files, 1): if extra := file.get("extraJson"): file.update(util.json_loads(extra)) - file["date"] = self.parse_datetime( - file["createdDate"], "%Y-%m-%dT%H:%M:%S.%f%z") - file["date_updated"] = self.parse_datetime( - file["updatedDate"], "%Y-%m-%dT%H:%M:%S.%f%z") + file["date"] = self.parse_datetime_iso( + file["createdDate"]) + file["date_updated"] = self.parse_datetime_iso( + file["updatedDate"]) data["file"] = file url = file["attachValue"] yield Message.Url, url, text.nameext_from_url(url, data) diff --git a/gallery_dl/extractor/nekohouse.py b/gallery_dl/extractor/nekohouse.py index 77298b0f..f865cbe1 100644 --- a/gallery_dl/extractor/nekohouse.py +++ b/gallery_dl/extractor/nekohouse.py @@ -59,8 +59,8 @@ class NekohousePostExtractor(NekohouseExtractor): 'class="scrape__user-name', '")[2].strip()), "title" : text.unescape(extr( 'class="scrape__title', '")[2]), - "date" : self.parse_datetime(extr( - 'datetime="', '"')[:19], "%Y-%m-%d %H:%M:%S"), + "date" : self.parse_datetime_iso(extr( + 'datetime="', '"')[:19]), "content": text.unescape(extr( 'class="scrape__content">', "
    ").strip()), } diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index b4ef1741..6a78a47a 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -9,7 +9,7 @@ """Extractors for https://nozomi.la/""" from .common import Extractor, Message -from .. import text +from .. import text, dt def decode_nozomi(n): @@ -49,10 +49,9 @@ class NozomiExtractor(Extractor): post["character"] = self._list(post.get("character")) try: - post["date"] = self.parse_datetime( - post["date"] + ":00", "%Y-%m-%d %H:%M:%S%z") + post["date"] = dt.parse_iso(post["date"] + ":00") except Exception: - post["date"] = None + post["date"] = dt.NONE post.update(data) diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py index cf73db83..c87a69e7 100644 --- a/gallery_dl/extractor/paheal.py +++ b/gallery_dl/extractor/paheal.py @@ -53,8 +53,7 @@ class PahealExtractor(Extractor): extr("Source Link<", ""), "href='", "'")), } diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index b4f0cdc4..760a7e1d 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -177,8 +177,7 @@ class PatreonExtractor(Extractor): post, included, "attachments") attr["attachments_media"] = self._files( post, included, "attachments_media") - attr["date"] = self.parse_datetime( - attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + attr["date"] = self.parse_datetime_iso(attr["published_at"]) try: attr["campaign"] = (included["campaign"][ @@ -226,8 +225,7 @@ class PatreonExtractor(Extractor): user = response.json()["data"] attr = user["attributes"] attr["id"] = user["id"] - attr["date"] = self.parse_datetime( - attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z") + attr["date"] = self.parse_datetime_iso(attr["created"]) return attr def _collection(self, collection_id): @@ -236,8 +234,7 @@ class PatreonExtractor(Extractor): coll = data["data"] attr = coll["attributes"] attr["id"] = coll["id"] - attr["date"] = self.parse_datetime( - attr["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + attr["date"] = self.parse_datetime_iso(attr["created_at"]) return attr def _filename(self, url): diff --git a/gallery_dl/extractor/pexels.py b/gallery_dl/extractor/pexels.py index ca482cc6..34dbcbd0 100644 --- a/gallery_dl/extractor/pexels.py +++ b/gallery_dl/extractor/pexels.py @@ -35,8 +35,7 @@ class PexelsExtractor(Extractor): post["type"] = attr["type"] post.update(metadata) - post["date"] = self.parse_datetime( - post["created_at"][:-5], "%Y-%m-%dT%H:%M:%S") + post["date"] = self.parse_datetime_iso(post["created_at"][:-5]) if "image" in post: url, _, query = post["image"]["download_link"].partition("?") diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index 9407acd2..7665250a 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -36,8 +36,7 @@ class PhilomenaExtractor(BooruExtractor): return url def _prepare(self, post): - post["date"] = self.parse_datetime( - post["created_at"][:19], "%Y-%m-%dT%H:%M:%S") + post["date"] = self.parse_datetime_iso(post["created_at"][:19]) BASE_PATTERN = PhilomenaExtractor.update({ diff --git a/gallery_dl/extractor/photovogue.py b/gallery_dl/extractor/photovogue.py index 6b38a983..16d9a7f5 100644 --- a/gallery_dl/extractor/photovogue.py +++ b/gallery_dl/extractor/photovogue.py @@ -29,8 +29,7 @@ class PhotovogueUserExtractor(Extractor): for photo in self.photos(): url = photo["gallery_image"] photo["title"] = photo["title"].strip() - photo["date"] = self.parse_datetime( - photo["date"], "%Y-%m-%dT%H:%M:%S.%f%z") + photo["date"] = self.parse_datetime_iso(photo["date"]) yield Message.Directory, photo yield Message.Url, url, text.nameext_from_url(url, photo) diff --git a/gallery_dl/extractor/picarto.py b/gallery_dl/extractor/picarto.py index a5168aa6..3c49452b 100644 --- a/gallery_dl/extractor/picarto.py +++ b/gallery_dl/extractor/picarto.py @@ -29,8 +29,7 @@ class PicartoGalleryExtractor(Extractor): def items(self): for post in self.posts(): - post["date"] = self.parse_datetime( - post["created_at"], "%Y-%m-%d %H:%M:%S") + post["date"] = self.parse_datetime_iso(post["created_at"]) variations = post.pop("variations", ()) yield Message.Directory, post diff --git a/gallery_dl/extractor/piczel.py b/gallery_dl/extractor/piczel.py index becdb327..3f43a499 100644 --- a/gallery_dl/extractor/piczel.py +++ b/gallery_dl/extractor/piczel.py @@ -26,8 +26,7 @@ class PiczelExtractor(Extractor): def items(self): for post in self.posts(): post["tags"] = [t["title"] for t in post["tags"] if t["title"]] - post["date"] = self.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = self.parse_datetime_iso(post["created_at"]) if post["multi"]: images = post["images"] diff --git a/gallery_dl/extractor/pillowfort.py b/gallery_dl/extractor/pillowfort.py index ed535180..637b0b9a 100644 --- a/gallery_dl/extractor/pillowfort.py +++ b/gallery_dl/extractor/pillowfort.py @@ -48,8 +48,7 @@ class PillowfortExtractor(Extractor): for url in inline(post["content"]): files.append({"url": url}) - post["date"] = self.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = self.parse_datetime_iso(post["created_at"]) post["post_id"] = post.pop("id") post["count"] = len(files) yield Message.Directory, post @@ -76,8 +75,7 @@ class PillowfortExtractor(Extractor): if "id" not in file: post["id"] = post["hash"] if "created_at" in file: - post["date"] = self.parse_datetime( - file["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = self.parse_datetime_iso(file["created_at"]) yield msgtype, url, post diff --git a/gallery_dl/extractor/pixeldrain.py b/gallery_dl/extractor/pixeldrain.py index 78b004c7..6df855e6 100644 --- a/gallery_dl/extractor/pixeldrain.py +++ b/gallery_dl/extractor/pixeldrain.py @@ -24,10 +24,6 @@ class PixeldrainExtractor(Extractor): if api_key := self.config("api-key"): self.session.auth = util.HTTPBasicAuth("", api_key) - def _parse_datetime(self, date_string): - return self.parse_datetime( - date_string, "%Y-%m-%dT%H:%M:%S.%fZ") - class PixeldrainFileExtractor(PixeldrainExtractor): """Extractor for pixeldrain files""" diff --git a/gallery_dl/extractor/pornhub.py b/gallery_dl/extractor/pornhub.py index 38905351..fbe14dd5 100644 --- a/gallery_dl/extractor/pornhub.py +++ b/gallery_dl/extractor/pornhub.py @@ -150,8 +150,7 @@ class PornhubGifExtractor(PornhubExtractor): "tags" : extr("data-context-tag='", "'").split(","), "title": extr('"name": "', '"'), "url" : extr('"contentUrl": "', '"'), - "date" : self.parse_datetime( - extr('"uploadDate": "', '"'), "%Y-%m-%d"), + "date" : self.parse_datetime_iso(extr('"uploadDate": "', '"')), "viewkey" : extr('From this video: ' '', '<'), diff --git a/gallery_dl/extractor/rule34vault.py b/gallery_dl/extractor/rule34vault.py index b5681207..74d79d3f 100644 --- a/gallery_dl/extractor/rule34vault.py +++ b/gallery_dl/extractor/rule34vault.py @@ -36,8 +36,7 @@ class Rule34vaultExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = self.parse_datetime( - post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created"]) if "tags" in post: post["tags"] = [t["value"] for t in post["tags"]] diff --git a/gallery_dl/extractor/rule34xyz.py b/gallery_dl/extractor/rule34xyz.py index bd60d01c..8174313a 100644 --- a/gallery_dl/extractor/rule34xyz.py +++ b/gallery_dl/extractor/rule34xyz.py @@ -68,8 +68,7 @@ class Rule34xyzExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = self.parse_datetime( - post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created"]) post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: post["tags"] = [t["value"] for t in post["tags"]] diff --git a/gallery_dl/extractor/s3ndpics.py b/gallery_dl/extractor/s3ndpics.py index 687dbe38..87d91a6f 100644 --- a/gallery_dl/extractor/s3ndpics.py +++ b/gallery_dl/extractor/s3ndpics.py @@ -30,10 +30,8 @@ class S3ndpicsExtractor(Extractor): for post in self.posts(): post["id"] = post.pop("_id", None) post["user"] = post.pop("userId", None) - post["date"] = self.parse_datetime( - post["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") - post["date_updated"] = self.parse_datetime( - post["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["createdAt"]) + post["date_updated"] = self.parse_datetime_iso(post["updatedAt"]) files = post.pop("files", ()) post["count"] = len(files) diff --git a/gallery_dl/extractor/slideshare.py b/gallery_dl/extractor/slideshare.py index b104e873..1bb70ed9 100644 --- a/gallery_dl/extractor/slideshare.py +++ b/gallery_dl/extractor/slideshare.py @@ -39,8 +39,8 @@ class SlidesharePresentationExtractor(GalleryExtractor): "description" : slideshow["description"].strip(), "views" : slideshow["views"], "likes" : slideshow["likes"], - "date" : self.parse_datetime( - slideshow["createdAt"], "%Y-%m-%d %H:%M:%S %Z"), + "date" : self.parse_datetime_iso( + slideshow["createdAt"][:19]), } def images(self, page): diff --git a/gallery_dl/extractor/szurubooru.py b/gallery_dl/extractor/szurubooru.py index 197012f0..2bf4a61e 100644 --- a/gallery_dl/extractor/szurubooru.py +++ b/gallery_dl/extractor/szurubooru.py @@ -57,8 +57,7 @@ class SzurubooruExtractor(booru.BooruExtractor): return url def _prepare(self, post): - post["date"] = self.parse_datetime( - post["creationTime"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["creationTime"]) tags = [] tags_categories = collections.defaultdict(list) diff --git a/gallery_dl/extractor/thehentaiworld.py b/gallery_dl/extractor/thehentaiworld.py index aed22836..47cc4329 100644 --- a/gallery_dl/extractor/thehentaiworld.py +++ b/gallery_dl/extractor/thehentaiworld.py @@ -56,8 +56,7 @@ class ThehentaiworldExtractor(Extractor): "id" : text.parse_int(extr(" postid-", " ")), "slug" : extr(" post-", '"'), "tags" : extr('id="tagsHead">', ""), - "date" : self.parse_datetime(extr( - "
  • Posted: ", "<"), "%Y-%m-%d"), + "date" : self.parse_datetime_iso(extr("
  • Posted: ", "<")), } if (c := url[27]) == "v": diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index ec862fe2..4558e212 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -37,8 +37,7 @@ class TwibooruExtractor(BooruExtractor): return post["view_url"] def _prepare(self, post): - post["date"] = self.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created_at"]) if "name" in post: name, sep, rest = post["name"].rpartition(".") @@ -146,8 +145,8 @@ class TwibooruAPI(): return response.json() if response.status_code == 429: - until = self.parse_datetime( - response.headers["X-RL-Reset"], "%Y-%m-%d %H:%M:%S %Z") + until = self.parse_datetime_iso( + response.headers["X-RL-Reset"][:19]) # wait an extra minute, just to be safe self.extractor.wait(until=until, adjust=60.0) continue diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index e0834b62..a6a10ff3 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -47,8 +47,8 @@ class VanillarockPostExtractor(VanillarockExtractor): "count": len(imgs), "title": text.unescape(name), "path" : self.path.strip("/"), - "date" : self.parse_datetime(extr( - '
    ', '
    '), "%Y-%m-%d %H:%M"), + "date" : self.parse_datetime_iso(extr( + '
    ', '
    ')), "tags" : text.split_html(extr( '
    ', '
    '))[::2], } diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index 861d0c99..623de8bf 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -43,8 +43,7 @@ class WallhavenExtractor(Extractor): wp["url"] = wp.pop("path") if "tags" in wp: wp["tags"] = [t["name"] for t in wp["tags"]] - wp["date"] = self.parse_datetime( - wp.pop("created_at"), "%Y-%m-%d %H:%M:%S") + wp["date"] = self.parse_datetime_iso(wp.pop("created_at")) wp["width"] = wp.pop("dimension_x") wp["height"] = wp.pop("dimension_y") wp["wh_category"] = wp["category"] diff --git a/gallery_dl/extractor/weebcentral.py b/gallery_dl/extractor/weebcentral.py index cddceb76..c5496e32 100644 --- a/gallery_dl/extractor/weebcentral.py +++ b/gallery_dl/extractor/weebcentral.py @@ -127,8 +127,8 @@ class WeebcentralMangaExtractor(WeebcentralBase, MangaExtractor): "chapter" : text.parse_int(chapter), "chapter_minor": sep + minor, "chapter_type" : type, - "date" : self.parse_datetime( - extr(' datetime="', '"')[:-5], "%Y-%m-%dT%H:%M:%S"), + "date" : self.parse_datetime_iso(extr( + ' datetime="', '"')[:-5]), } chapter.update(data) results.append((base + chapter_id, chapter)) diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 7c168e9e..a07fd840 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -34,8 +34,8 @@ class WikifeetGalleryExtractor(GalleryExtractor): "celeb" : self.celeb, "type" : self.type, "birthplace": text.unescape(extr('"bplace":"', '"')), - "birthday" : self.parse_datetime(text.unescape( - extr('"bdate":"', '"'))[:10], "%Y-%m-%d"), + "birthday" : self.parse_datetime_iso(text.unescape(extr( + '"bdate":"', '"'))[:10]), "shoesize" : text.unescape(extr('"ssize":', ',')), "rating" : text.parse_float(extr('"score":', ',')), "celebrity" : text.unescape(extr('"cname":"', '"')), diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index 83220889..040e4f6a 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -75,8 +75,7 @@ class WikimediaExtractor(BaseExtractor): for m in image["commonmetadata"] or ()} text.nameext_from_url(image["canonicaltitle"].partition(":")[2], image) - image["date"] = self.parse_datetime( - image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + image["date"] = self.parse_datetime_iso(image["timestamp"]) def items(self): for info in self._pagination(self.params): diff --git a/gallery_dl/extractor/yiffverse.py b/gallery_dl/extractor/yiffverse.py index 33640fc1..f1073ed6 100644 --- a/gallery_dl/extractor/yiffverse.py +++ b/gallery_dl/extractor/yiffverse.py @@ -55,8 +55,7 @@ class YiffverseExtractor(BooruExtractor): def _prepare(self, post): post.pop("files", None) - post["date"] = self.parse_datetime( - post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = self.parse_datetime_iso(post["created"]) post["filename"], _, post["format"] = post["filename"].rpartition(".") if "tags" in post: post["tags"] = [t["value"] for t in post["tags"]] From ed8fe07f92146464ccbdf7f5326ff5325af9ce92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 20 Oct 2025 09:19:47 +0200 Subject: [PATCH 13/13] [tests/formatter] update results --- gallery_dl/dt.py | 2 +- test/test_formatter.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gallery_dl/dt.py b/gallery_dl/dt.py index 74e65fe3..b37ebf38 100644 --- a/gallery_dl/dt.py +++ b/gallery_dl/dt.py @@ -22,7 +22,7 @@ class NullDatetime(datetime): return "[Invalid DateTime]" def __format__(self, format_spec): - return f"[Invalid DateTime {format_spec}]" + return "[Invalid DateTime]" NONE = NullDatetime(1, 1, 1) diff --git a/test/test_formatter.py b/test/test_formatter.py index e4444346..f08ae49c 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -15,7 +15,7 @@ import datetime import tempfile sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import formatter, text, util, config # noqa E402 +from gallery_dl import formatter, text, dt, util, config # noqa E402 try: import jinja2 @@ -154,7 +154,7 @@ class TestFormatter(unittest.TestCase): self._run_test("{t}" , self.kwdict["t"] , None, int) self._run_test("{t}" , self.kwdict["t"] , None, util.identity) self._run_test("{dt}", self.kwdict["dt"], None, util.identity) - self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime) + self._run_test("{ds}", self.kwdict["dt"], None, dt.parse_iso) self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], None, util.identity) @@ -271,8 +271,8 @@ class TestFormatter(unittest.TestCase): def test_specifier_datetime(self): self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00") - self._run_test("{ds:D%Y}", "0101-01-01 00:00:00") - self._run_test("{l2:D%Y}", "0101-01-01 00:00:00") + self._run_test("{ds:D%Y}", "[Invalid DateTime]") + self._run_test("{l2:D%Y}", "[Invalid DateTime]") def test_specifier_offset(self): self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00")