From 4089bcedddb5900781df230b2ddc94bc44cd189e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Fri, 30 Sep 2022 19:55:48 +0200
Subject: [PATCH] [sankaku] implement 'refresh' option (#2958)

---
 docs/configuration.rst          | 10 ++++++++++
 docs/gallery-dl.conf            | 11 ++++++-----
 gallery_dl/extractor/sankaku.py | 34 ++++++++++++++++++++++++++++++++-
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index f806298b..84f7bd2c 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -2172,6 +2172,16 @@ Description
     restrict it to only one possible format.
 
 
+extractor.sankaku.refresh
+-------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Refresh download URLs before they expire.
+
+
 extractor.sankakucomplex.embeds
 -------------------------------
 Type
diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
index 497fa4a8..1c565ece 100644
--- a/docs/gallery-dl.conf
+++ b/docs/gallery-dl.conf
@@ -245,16 +245,17 @@
         {
             "format": ["hd", "sd", "gif"]
         },
+        "sankaku":
+        {
+            "username": null,
+            "password": null,
+            "refresh": false
+        },
         "sankakucomplex":
         {
             "embeds": false,
             "videos": true
         },
-        "sankaku":
-        {
-            "username": null,
-            "password": null
-        },
         "skeb":
         {
             "article": false,
diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py
index 34fa0e35..0d79f9d7 100644
--- a/gallery_dl/extractor/sankaku.py
+++ b/gallery_dl/extractor/sankaku.py
@@ -281,9 +281,41 @@ class SankakuAPI():
         params["lang"] = "en"
         params["limit"] = str(self.extractor.per_page)
 
+        refresh = self.extractor.config("refresh", False)
+        if refresh:
+            offset = expires = 0
+            from time import time
+
         while True:
             data = self._call(endpoint, params)
-            yield from data["data"]
+
+            if refresh:
+                posts = data["data"]
+                if offset:
+                    posts = util.advance(posts, offset)
+
+                for post in posts:
+                    if not expires:
+                        url = post["file_url"]
+                        if url:
+                            expires = text.parse_int(
+                                text.extract(url, "e=", "&")[0]) - 60
+
+                    if 0 < expires <= time():
+                        self.extractor.log.debug("Refreshing download URLs")
+                        expires = None
+                        break
+
+                    offset += 1
+                    yield post
+
+                if expires is None:
+                    expires = 0
+                    continue
+                offset = expires = 0
+
+            else:
+                yield from data["data"]
 
             params["next"] = data["meta"]["next"]
             if not params["next"]: