github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[dt] replace 'util' datetime functions

Browse Source
This commit is contained in:
Mike Fährmann
2025-10-15 21:30:38 +02:00
parent d0f5d4e0d3
commit 0eb3c8a994
13 changed files with 62 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gallery_dl/extractor/4archive.py
View File

@@ -7,7 +7,7 @@
"""Extractors for https://4archive.org/"""
from .common import Extractor, Message
from .. import text, util
from .. import text, dt
class _4archiveThreadExtractor(Extractor):
@@ -37,7 +37,7 @@ class _4archiveThreadExtractor(Extractor):
for post in posts:
post.update(data)
post["time"] = int(util.datetime_to_timestamp(post["date"]))
post["time"] = int(dt.to_ts(post["date"]))
yield Message.Directory, post
if "url" in post:
yield Message.Url, post["url"], text.nameext_from_url(

7
gallery_dl/extractor/8chan.py
View File

@@ -9,9 +9,8 @@
"""Extractors for https://8chan.moe/"""
from .common import Extractor, Message
from .. import text, util
from .. import text, dt
from ..cache import memcache
from datetime import timedelta
import itertools
BASE_PATTERN = r"(?:https?://)?8chan\.(moe|se|cc)"
@@ -44,7 +43,7 @@ class _8chanExtractor(Extractor):
def cookies_prepare(self):
# fetch captcha cookies
# (necessary to download without getting interrupted)
now = util.datetime_utcnow()
now = dt.now()
url = self.root + "/captcha.js"
params = {"d": now.strftime("%a %b %d %Y %H:%M:%S GMT+0000 (UTC)")}
self.request(url, params=params).content
@@ -57,7 +56,7 @@ class _8chanExtractor(Extractor):
if cookie.domain.endswith(domain):
cookie.expires = None
if cookie.name == "captchaexpiration":
cookie.value = (now + timedelta(30, 300)).strftime(
cookie.value = (now + dt.timedelta(30, 300)).strftime(
"%a, %d %b %Y %H:%M:%S GMT")
return self.cookies

4
gallery_dl/extractor/common.py
View File

@@ -23,7 +23,7 @@ from datetime import datetime
from xml.etree import ElementTree
from requests.adapters import HTTPAdapter
from .message import Message
from .. import config, output, text, util, cache, exception
from .. import config, output, text, util, dt, cache, exception
urllib3 = requests.packages.urllib3
@@ -315,7 +315,7 @@ class Extractor():
elif until:
if isinstance(until, datetime):
# convert to UTC timestamp
until = util.datetime_to_timestamp(until)
until = dt.to_ts(until)
else:
until = float(until)
seconds = until - now

6
gallery_dl/extractor/deviantart.py
View File

@@ -9,7 +9,7 @@
"""Extractors for https://www.deviantart.com/"""
from .common import Extractor, Message, Dispatch
from .. import text, util, exception
from .. import text, util, dt, exception
from ..cache import cache, memcache
import collections
import mimetypes
@@ -1187,8 +1187,8 @@ class DeviantartStatusExtractor(DeviantartExtractor):
deviation["username"] = deviation["author"]["username"]
deviation["_username"] = deviation["username"].lower()
deviation["date"] = dt = text.parse_datetime(deviation["ts"])
deviation["published_time"] = int(util.datetime_to_timestamp(dt))
deviation["date"] = d = text.parse_datetime(deviation["ts"])
deviation["published_time"] = int(dt.to_ts(d))
deviation["da_category"] = "Status"
deviation["category_path"] = "status"

17
gallery_dl/extractor/motherless.py
View File

@@ -9,9 +9,8 @@
"""Extractors for https://motherless.com/"""
from .common import Extractor, Message
from .. import text, util, exception
from .. import text, dt, exception
from ..cache import memcache
from datetime import timedelta
BASE_PATTERN = r"(?:https?://)?motherless\.com"
@@ -115,14 +114,14 @@ class MotherlessExtractor(Extractor):
return data
def _parse_datetime(self, dt):
if " ago" not in dt:
return text.parse_datetime(dt, "%d %b %Y")
def _parse_datetime(self, dt_string):
if " ago" not in dt_string:
return dt.parse(dt_string, "%d %b %Y")
value = text.parse_int(dt[:-5])
delta = timedelta(0, value*3600) if dt[-5] == "h" else timedelta(value)
return (util.datetime_utcnow() - delta).replace(
hour=0, minute=0, second=0)
value = text.parse_int(dt_string[:-5])
delta = (dt.timedelta(0, value*3600) if dt_string[-5] == "h" else
dt.timedelta(value))
return (dt.now() - delta).replace(hour=0, minute=0, second=0)
@memcache(keyarg=2)
def _extract_gallery_title(self, page, gallery_id):

5
gallery_dl/extractor/patreon.py
View File

@@ -9,7 +9,7 @@
"""Extractors for https://www.patreon.com/"""
from .common import Extractor, Message
from .. import text, util, exception
from .. import text, util, dt, exception
from ..cache import memcache
import collections
import itertools
@@ -445,8 +445,7 @@ class PatreonUserExtractor(PatreonExtractor):
def posts(self):
if date_max := self._get_date_min_max(None, None)[1]:
self._cursor = cursor = \
util.datetime_from_timestamp(date_max).isoformat()
self._cursor = cursor = dt.from_ts(date_max).isoformat()
self._init_cursor = lambda: cursor
url = self._build_url("stream", (

27
gallery_dl/extractor/pixiv.py
View File

@@ -9,9 +9,8 @@
"""Extractors for https://www.pixiv.net/"""
from .common import Extractor, Message, Dispatch
from .. import text, util, exception
from .. import text, util, dt, exception
from ..cache import cache, memcache
from datetime import datetime, timedelta
import itertools
import hashlib
@@ -96,7 +95,7 @@ class PixivExtractor(Extractor):
if transform_tags:
transform_tags(work)
work["num"] = 0
work["date"] = text.parse_datetime(work["create_date"])
work["date"] = dt.parse_iso(work["create_date"])
work["rating"] = ratings.get(work["x_restrict"])
work["suffix"] = ""
work.update(metadata)
@@ -350,10 +349,10 @@ class PixivExtractor(Extractor):
if fmt in urls:
yield urls[fmt]
def _date_from_url(self, url, offset=timedelta(hours=9)):
def _date_from_url(self, url, offset=dt.timedelta(hours=9)):
try:
_, _, _, _, _, y, m, d, H, M, S, _ = url.split("/")
return datetime(
return dt.datetime(
int(y), int(m), int(d), int(H), int(M), int(S)) - offset
except Exception:
return None
@@ -712,8 +711,7 @@ class PixivRankingExtractor(PixivExtractor):
self.log.warning("invalid date '%s'", date)
date = None
if not date:
now = util.datetime_utcnow()
date = (now - timedelta(days=1)).strftime("%Y-%m-%d")
date = (dt.now() - dt.timedelta(days=1)).strftime("%Y-%m-%d")
self.date = date
self.type = type = query.get("content")
@@ -888,8 +886,7 @@ class PixivSketchExtractor(Extractor):
for post in self.posts():
media = post["media"]
post["post_id"] = post["id"]
post["date"] = text.parse_datetime(
post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
post["date"] = dt.parse_iso(post["created_at"])
util.delete_items(post, ("id", "media", "_links"))
yield Message.Directory, post
@@ -969,7 +966,7 @@ class PixivNovelExtractor(PixivExtractor):
if transform_tags:
transform_tags(novel)
novel["num"] = 0
novel["date"] = text.parse_datetime(novel["create_date"])
novel["date"] = dt.parse_iso(novel["create_date"])
novel["rating"] = ratings.get(novel["x_restrict"])
novel["suffix"] = ""
@@ -1151,7 +1148,7 @@ class PixivAppAPI():
"get_secure_url": "1",
}
time = util.datetime_utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00")
time = dt.now().strftime("%Y-%m-%dT%H:%M:%S+00:00")
headers = {
"X-Client-Time": time,
"X-Client-Hash": hashlib.md5(
@@ -1326,11 +1323,11 @@ class PixivAppAPI():
sort = params["sort"]
if sort == "date_desc":
date_key = "end_date"
date_off = timedelta(days=1)
date_off = dt.timedelta(days=1)
date_cmp = lambda lhs, rhs: lhs >= rhs # noqa E731
elif sort == "date_asc":
date_key = "start_date"
date_off = timedelta(days=-1)
date_off = dt.timedelta(days=-1)
date_cmp = lambda lhs, rhs: lhs <= rhs # noqa E731
else:
date_key = None
@@ -1357,8 +1354,8 @@ class PixivAppAPI():
if date_key and text.parse_int(params.get("offset")) >= 5000:
date_last = data["illusts"][-1]["create_date"]
date_val = (text.parse_datetime(
date_last) + date_off).strftime("%Y-%m-%d")
date_val = (dt.parse_iso(date_last) + date_off).strftime(
"%Y-%m-%d")
self.log.info("Reached 'offset' >= 5000; "
"Updating '%s' to '%s'", date_key, date_val)
params[date_key] = date_val