From 343981ac1c37afba1f4de80edf68dd1dba9f4d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 19:59:40 +0100 Subject: [PATCH 01/11] [common] add 'utils()' method --- gallery_dl/extractor/common.py | 12 ++++++++++++ gallery_dl/extractor/utils/__init__.py | 0 2 files changed, 12 insertions(+) create mode 100644 gallery_dl/extractor/utils/__init__.py diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 52f4ae04..83a9a65a 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -354,6 +354,17 @@ class Extractor(): seconds, reason) time.sleep(seconds) + def utils(self, module="", name=None): + module = (self.__class__.category if not module else + module[1:] if module[0] == "/" else + f"{self.__class__.category}_{module}") + if module in CACHE_UTILS: + res = CACHE_UTILS[module] + else: + res = CACHE_UTILS[module] = __import__( + "utils." + module, globals(), None, module, 1) + return res if name is None else getattr(res, name, None) + def input(self, prompt, echo=True): self._check_input_allowed(prompt) @@ -1130,6 +1141,7 @@ def _browser_useragent(browser): CACHE_ADAPTERS = {} CACHE_COOKIES = {} +CACHE_UTILS = {} CATEGORY_MAP = () diff --git a/gallery_dl/extractor/utils/__init__.py b/gallery_dl/extractor/utils/__init__.py new file mode 100644 index 00000000..e69de29b From 7692d31a5773635eccb55c1a3b3c7b25f92b2866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 20:09:26 +0100 Subject: [PATCH 02/11] [twitter] move transaction_id.py --- gallery_dl/extractor/twitter.py | 5 ++--- .../utils/twitter_transaction_id.py} | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) rename gallery_dl/{transaction_id.py => extractor/utils/twitter_transaction_id.py} (98%) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 31b431b8..553dbaf4 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2025 Mike Fährmann +# Copyright 2016-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -1720,8 +1720,7 @@ class TwitterAPI(): def _client_transaction(self): self.log.info("Initializing client transaction keys") - from .. import transaction_id - ct = transaction_id.ClientTransaction() + ct = self.extractor.utils("transaction_id").ClientTransaction() ct.initialize(self.extractor) # update 'x-csrf-token' header (#7467) diff --git a/gallery_dl/transaction_id.py b/gallery_dl/extractor/utils/twitter_transaction_id.py similarity index 98% rename from gallery_dl/transaction_id.py rename to gallery_dl/extractor/utils/twitter_transaction_id.py index f8769d9a..aae3a5ca 100644 --- a/gallery_dl/transaction_id.py +++ b/gallery_dl/extractor/utils/twitter_transaction_id.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2025 Mike Fährmann +# Copyright 2025-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -22,8 +22,8 @@ import random import hashlib import binascii import itertools -from . import text, util -from .cache import cache +from ... import text, util +from ...cache import cache class ClientTransaction(): From 20ef39be456280aa3a9ba15fd331d1875eb5bda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 20:28:36 +0100 Subject: [PATCH 03/11] [tsumino] export 'jsurl' code --- gallery_dl/extractor/tsumino.py | 129 +-------------------------- gallery_dl/extractor/utils/jsurl.py | 133 ++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 127 deletions(-) create mode 100644 gallery_dl/extractor/utils/jsurl.py diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index 2ac7e42f..9ccfd6d1 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2025 Mike Fährmann +# Copyright 2019-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -153,7 +153,7 @@ class TsuminoSearchExtractor(TsuminoBase, Extractor): try: if query[0] == "?": return self._parse_simple(query) - return self._parse_jsurl(query) + return self.utils("/jsurl").parse(query) except Exception as exc: raise exception.AbortExtraction( f"Invalid search query '{query}' ({exc})") @@ -177,128 +177,3 @@ class TsuminoSearchExtractor(TsuminoBase, Extractor): "Tags[0][Text]": text.unquote(value).replace("+", " "), "Tags[0][Exclude]": "false", } - - def _parse_jsurl(self, data): - """Parse search query in JSURL format - - Nested lists and dicts are handled in a special way to deal - with the way Tsumino expects its parameters -> expand(...) - - Example: ~(name~'John*20Doe~age~42~children~(~'Mary~'Bill)) - Ref: https://github.com/Sage/jsurl - """ - i = 0 - imax = len(data) - - def eat(expected): - nonlocal i - - if data[i] != expected: - raise ValueError( - f"bad JSURL syntax: expected '{expected}', got {data[i]}") - i += 1 - - def decode(): - nonlocal i - - beg = i - result = "" - - while i < imax: - ch = data[i] - - if ch not in "~)*!": - i += 1 - - elif ch == "*": - if beg < i: - result += data[beg:i] - if data[i + 1] == "*": - result += chr(int(data[i+2:i+6], 16)) - i += 6 - else: - result += chr(int(data[i+1:i+3], 16)) - i += 3 - beg = i - - elif ch == "!": - if beg < i: - result += data[beg:i] - result += "$" - i += 1 - beg = i - - else: - break - - return result + data[beg:i] - - def parse_one(): - nonlocal i - - eat('~') - result = "" - ch = data[i] - - if ch == "(": - i += 1 - - if data[i] == "~": - result = [] - if data[i+1] == ")": - i += 1 - else: - result.append(parse_one()) - while data[i] == "~": - result.append(parse_one()) - - else: - result = {} - - if data[i] != ")": - while True: - key = decode() - value = parse_one() - for ekey, evalue in expand(key, value): - result[ekey] = evalue - if data[i] != "~": - break - i += 1 - eat(")") - - elif ch == "'": - i += 1 - result = decode() - - else: - beg = i - i += 1 - - while i < imax and data[i] not in "~)": - i += 1 - - sub = data[beg:i] - if ch in "0123456789-": - fval = float(sub) - ival = int(fval) - result = ival if ival == fval else fval - else: - if sub not in ("true", "false", "null"): - raise ValueError("bad value keyword: " + sub) - result = sub - - return result - - def expand(key, value): - if isinstance(value, list): - for index, cvalue in enumerate(value): - ckey = f"{key}[{index}]" - yield from expand(ckey, cvalue) - elif isinstance(value, dict): - for ckey, cvalue in value.items(): - ckey = f"{key}[{ckey}]" - yield from expand(ckey, cvalue) - else: - yield key, value - - return parse_one() diff --git a/gallery_dl/extractor/utils/jsurl.py b/gallery_dl/extractor/utils/jsurl.py new file mode 100644 index 00000000..248dda11 --- /dev/null +++ b/gallery_dl/extractor/utils/jsurl.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +def parse(data): + """Parse JSURL data + + Nested lists and dicts are handled in a special way to deal + with the way Tsumino expects its parameters -> expand(...) + + Example: ~(name~'John*20Doe~age~42~children~(~'Mary~'Bill)) + Ref: https://github.com/Sage/jsurl + """ + i = 0 + imax = len(data) + + def eat(expected): + nonlocal i + + if data[i] != expected: + raise ValueError( + f"bad JSURL syntax: expected '{expected}', got {data[i]}") + i += 1 + + def decode(): + nonlocal i + + beg = i + result = "" + + while i < imax: + ch = data[i] + + if ch not in "~)*!": + i += 1 + + elif ch == "*": + if beg < i: + result += data[beg:i] + if data[i + 1] == "*": + result += chr(int(data[i+2:i+6], 16)) + i += 6 + else: + result += chr(int(data[i+1:i+3], 16)) + i += 3 + beg = i + + elif ch == "!": + if beg < i: + result += data[beg:i] + result += "$" + i += 1 + beg = i + + else: + break + + return result + data[beg:i] + + def parse_one(): + nonlocal i + + eat('~') + result = "" + ch = data[i] + + if ch == "(": + i += 1 + + if data[i] == "~": + result = [] + if data[i+1] == ")": + i += 1 + else: + result.append(parse_one()) + while data[i] == "~": + result.append(parse_one()) + + else: + result = {} + + if data[i] != ")": + while True: + key = decode() + value = parse_one() + for ekey, evalue in expand(key, value): + result[ekey] = evalue + if data[i] != "~": + break + i += 1 + eat(")") + + elif ch == "'": + i += 1 + result = decode() + + else: + beg = i + i += 1 + + while i < imax and data[i] not in "~)": + i += 1 + + sub = data[beg:i] + if ch in "0123456789-": + fval = float(sub) + ival = int(fval) + result = ival if ival == fval else fval + else: + if sub not in ("true", "false", "null"): + raise ValueError("bad value keyword: " + sub) + result = sub + + return result + + def expand(key, value): + if isinstance(value, list): + for index, cvalue in enumerate(value): + ckey = f"{key}[{index}]" + yield from expand(ckey, cvalue) + elif isinstance(value, dict): + for ckey, cvalue in value.items(): + ckey = f"{key}[{ckey}]" + yield from expand(ckey, cvalue) + else: + yield key, value + + return parse_one() From 3d114dbc67b70e6349dda9200f4335bfa02c8342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 21:05:10 +0100 Subject: [PATCH 04/11] [deviantart] export 'tiptap' functions --- gallery_dl/extractor/deviantart.py | 279 ++---------------- .../extractor/utils/deviantart_tiptap.py | 256 ++++++++++++++++ 2 files changed, 278 insertions(+), 257 deletions(-) create mode 100644 gallery_dl/extractor/utils/deviantart_tiptap.py diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index aec14cb6..22a74a02 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -402,7 +402,7 @@ class DeviantartExtractor(Extractor): if html["type"] == "tiptap": try: - return self._tiptap_to_html(markup) + return self.utils("tiptap").to_html(markup) except Exception as exc: self.log.traceback(exc) self.log.error("%s: '%s: %s'", deviation["index"], @@ -411,242 +411,6 @@ class DeviantartExtractor(Extractor): self.log.warning("%s: Unsupported '%s' markup.", deviation["index"], html["type"]) - def _tiptap_to_html(self, markup): - html = [] - - html.append('
') - data = util.json_loads(markup) - for block in data["document"]["content"]: - self._tiptap_process_content(html, block) - html.append("
") - - return "".join(html) - - def _tiptap_process_content(self, html, content): - type = content["type"] - - if type == "paragraph": - if children := content.get("content"): - html.append('

') - else: - html.append('margin-inline-start:0px">') - - for block in children: - self._tiptap_process_content(html, block) - html.append("

") - else: - html.append('


') - - elif type == "text": - self._tiptap_process_text(html, content) - - elif type == "heading": - attrs = content["attrs"] - level = str(attrs.get("level") or "3") - - html.append("') - html.append('') - self._tiptap_process_children(html, content) - html.append("") - - elif type in ("listItem", "bulletList", "orderedList", "blockquote"): - c = type[1] - tag = ( - "li" if c == "i" else - "ul" if c == "u" else - "ol" if c == "r" else - "blockquote" - ) - html.append("<" + tag + ">") - self._tiptap_process_children(html, content) - html.append("") - - elif type == "anchor": - attrs = content["attrs"] - html.append('') - - elif type == "hardBreak": - html.append("

") - - elif type == "horizontalRule": - html.append("
") - - elif type == "da-deviation": - self._tiptap_process_deviation(html, content) - - elif type == "da-mention": - user = content["attrs"]["user"]["username"] - html.append('@') - html.append(user) - html.append('') - - elif type == "da-gif": - attrs = content["attrs"] - width = str(attrs.get("width") or "") - height = str(attrs.get("height") or "") - url = text.escape(attrs.get("url") or "") - - html.append('
') - - elif type == "da-video": - src = text.escape(content["attrs"].get("src") or "") - html.append('
' - '
') - - else: - self.log.warning("Unsupported content type '%s'", type) - - def _tiptap_process_text(self, html, content): - if marks := content.get("marks"): - close = [] - for mark in marks: - type = mark["type"] - if type == "link": - attrs = mark.get("attrs") or {} - html.append('') - close.append("") - elif type == "bold": - html.append("") - close.append("") - elif type == "italic": - html.append("") - close.append("") - elif type == "underline": - html.append("") - close.append("") - elif type == "strike": - html.append("") - close.append("") - elif type == "textStyle" and len(mark) <= 1: - pass - else: - self.log.warning("Unsupported text marker '%s'", type) - close.reverse() - html.append(text.escape(content["text"])) - html.extend(close) - else: - html.append(text.escape(content["text"])) - - def _tiptap_process_children(self, html, content): - if children := content.get("content"): - for block in children: - self._tiptap_process_content(html, block) - - def _tiptap_process_indentation(self, html, attrs): - itype = ("text-indent" if attrs.get("indentType") == "line" else - "margin-inline-start") - isize = str((attrs.get("indentation") or 0) * 24) - html.append(itype + ":" + isize + "px") - - def _tiptap_process_deviation(self, html, content): - dev = content["attrs"]["deviation"] - media = dev.get("media") or () - - html.append('
') - html.append('
') - - if "baseUri" in media: - url, formats = self._eclipse_media(media) - full = formats["fullview"] - - html.append('') - - html.append('') - html.append(text.escape(dev[') - html.append("") - - elif "textContent" in dev: - html.append('
') - - html.append('') - - html.append('''\ -
\ -
Literature

\ -''') - html.append(text.escape(dev["title"])) - html.append('

') - html.append(text.escape(dev["textContent"]["excerpt"])) - html.append('
') - - html.append('
') - def _extract_content(self, deviation): content = deviation["content"] @@ -827,25 +591,6 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\ self.log.info("Unwatching %s", username) self.api.user_friends_unwatch(username) - def _eclipse_media(self, media, format="preview"): - url = [media["baseUri"]] - - formats = { - fmt["t"]: fmt - for fmt in media["types"] - } - - if tokens := media.get("token") or (): - if len(tokens) <= 1: - fmt = formats[format] - if "c" in fmt: - url.append(fmt["c"].replace( - "", media["prettyName"])) - url.append("?token=") - url.append(tokens[-1]) - - return "".join(url), formats - def _eclipse_to_oauth(self, eclipse_api, deviations): for obj in deviations: deviation = obj["deviation"] if "deviation" in obj else obj @@ -1303,7 +1048,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor): yield deviation for index, post in enumerate(additional_media): - uri = self._eclipse_media(post["media"], "fullview")[0] + uri = eclipse_media(post["media"], "fullview")[0] deviation["content"]["src"] = uri deviation["num"] += 1 deviation["index_file"] = post["fileId"] @@ -2245,3 +1990,23 @@ by {username}, {date} {content} """ + + +def eclipse_media(media, format="preview"): + url = [media["baseUri"]] + + formats = { + fmt["t"]: fmt + for fmt in media["types"] + } + + if tokens := media.get("token") or (): + if len(tokens) <= 1: + fmt = formats[format] + if "c" in fmt: + url.append(fmt["c"].replace( + "", media["prettyName"])) + url.append("?token=") + url.append(tokens[-1]) + + return "".join(url), formats diff --git a/gallery_dl/extractor/utils/deviantart_tiptap.py b/gallery_dl/extractor/utils/deviantart_tiptap.py new file mode 100644 index 00000000..36dfc6af --- /dev/null +++ b/gallery_dl/extractor/utils/deviantart_tiptap.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from ... import text, util +from .. deviantart import eclipse_media + + +def to_html(markup): + html = [] + + html.append('
') + data = util.json_loads(markup) + for block in data["document"]["content"]: + process_content(html, block) + html.append("
") + + return "".join(html) + + +def process_content(html, content): + type = content["type"] + + if type == "paragraph": + if children := content.get("content"): + html.append('

') + else: + html.append('margin-inline-start:0px">') + + for block in children: + process_content(html, block) + html.append("

") + else: + html.append('


') + + elif type == "text": + process_text(html, content) + + elif type == "heading": + attrs = content["attrs"] + level = str(attrs.get("level") or "3") + + html.append("') + html.append('') + process_children(html, content) + html.append("") + + elif type in ("listItem", "bulletList", "orderedList", "blockquote"): + c = type[1] + tag = ( + "li" if c == "i" else + "ul" if c == "u" else + "ol" if c == "r" else + "blockquote" + ) + html.append("<" + tag + ">") + process_children(html, content) + html.append("") + + elif type == "anchor": + attrs = content["attrs"] + html.append('') + + elif type == "hardBreak": + html.append("

") + + elif type == "horizontalRule": + html.append("
") + + elif type == "da-deviation": + process_deviation(html, content) + + elif type == "da-mention": + user = content["attrs"]["user"]["username"] + html.append('@') + html.append(user) + html.append('') + + elif type == "da-gif": + attrs = content["attrs"] + width = str(attrs.get("width") or "") + height = str(attrs.get("height") or "") + url = text.escape(attrs.get("url") or "") + + html.append('
') + + elif type == "da-video": + src = text.escape(content["attrs"].get("src") or "") + html.append('
' + '
') + + else: + import logging + logging.getLogger("tiptap").warning( + "Unsupported content type '%s'", type) + + +def process_text(html, content): + if marks := content.get("marks"): + close = [] + for mark in marks: + type = mark["type"] + if type == "link": + attrs = mark.get("attrs") or {} + html.append('') + close.append("") + elif type == "bold": + html.append("") + close.append("") + elif type == "italic": + html.append("") + close.append("") + elif type == "underline": + html.append("") + close.append("") + elif type == "strike": + html.append("") + close.append("") + elif type == "textStyle" and len(mark) <= 1: + pass + else: + import logging + logging.getLogger("tiptap").warning( + "Unsupported text marker '%s'", type) + close.reverse() + html.append(text.escape(content["text"])) + html.extend(close) + else: + html.append(text.escape(content["text"])) + + +def process_children(html, content): + if children := content.get("content"): + for block in children: + process_content(html, block) + + +def process_indentation(html, attrs): + itype = ("text-indent" if attrs.get("indentType") == "line" else + "margin-inline-start") + isize = str((attrs.get("indentation") or 0) * 24) + html.append(itype + ":" + isize + "px") + + +def process_deviation(html, content): + dev = content["attrs"]["deviation"] + media = dev.get("media") or () + + html.append('
') + html.append('
') + + if "baseUri" in media: + url, formats = eclipse_media(media) + full = formats["fullview"] + + html.append('') + + html.append('') + html.append(text.escape(dev[') + html.append("") + + elif "textContent" in dev: + html.append('
') + + html.append('') + + html.append('''\ +
\ +
Literature

\ +''') + html.append(text.escape(dev["title"])) + html.append('

') + html.append(text.escape(dev["textContent"]["excerpt"])) + html.append('
') + + html.append('
') From 1c2e2d5d08744d16b49f15591d6bb8bb70771a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 21:22:43 +0100 Subject: [PATCH 05/11] [deviantart] export journal templates --- gallery_dl/extractor/deviantart.py | 131 ++---------------- .../extractor/utils/deviantart_journal.py | 116 ++++++++++++++++ 2 files changed, 126 insertions(+), 121 deletions(-) create mode 100644 gallery_dl/extractor/utils/deviantart_journal.py diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 22a74a02..f4abc7a6 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -292,7 +292,8 @@ class DeviantartExtractor(Extractor): url = deviation["url"] thumbs = deviation.get("thumbs") or deviation.get("files") html = journal["html"] - shadow = SHADOW_TEMPLATE.format_map(thumbs[0]) if thumbs else "" + tmpl = self.utils("journal") + shadow = tmpl.SHADOW.format_map(thumbs[0]) if thumbs else "" if not html: self.log.warning("%s: Empty journal content", deviation["index"]) @@ -308,14 +309,14 @@ class DeviantartExtractor(Extractor): if html.find('
', 0, 250) != -1: needle = '
' - header = HEADER_CUSTOM_TEMPLATE.format( + header = tmpl.HEADER_CUSTOM.format( title=title, url=url, date=deviation["date"], ) else: needle = '
' username = deviation["author"]["username"] urlname = deviation.get("username") or username.lower() - header = HEADER_TEMPLATE.format( + header = tmpl.HEADER.format( title=title, url=url, userurl=f"{self.root}/{urlname}/", @@ -326,9 +327,9 @@ class DeviantartExtractor(Extractor): if needle in html: html = html.replace(needle, header, 1) else: - html = JOURNAL_TEMPLATE_HTML_EXTRA.format(header, html) + html = tmpl.HTML_EXTRA.format(header, html) - html = JOURNAL_TEMPLATE_HTML.format( + html = tmpl.HTML.format( title=title, html=html, shadow=shadow, css=css, cls=cls) deviation["extension"] = "htm" @@ -345,7 +346,7 @@ class DeviantartExtractor(Extractor): text.unescape(text.remove_html(txt)) for txt in (head or tail).split("
") ) - txt = JOURNAL_TEMPLATE_TEXT.format( + txt = self.utils("journal").TEXT.format( title=deviation["title"], username=deviation["author"]["username"], date=deviation["date"], @@ -1869,6 +1870,9 @@ def _login_impl(extr, username, password): } +_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz" + + def id_from_base36(base36): return util.bdecode(base36, _ALPHABET) @@ -1877,121 +1881,6 @@ def base36_from_id(deviation_id): return util.bencode(int(deviation_id), _ALPHABET) -_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz" - - -############################################################################### -# Journal Formats ############################################################# - -SHADOW_TEMPLATE = """ - - - -

-""" - -HEADER_TEMPLATE = """
-
-

{title}

- -
-""" - -HEADER_CUSTOM_TEMPLATE = """
-

- - {title} -

-Journal Entry: {date} -""" - -JOURNAL_TEMPLATE_HTML = """text: - - - - {title} - - - - - - - - - - - - - -
-
-
-
-
- {shadow} -
-
-
- {html} -
-
-
-
-
-
-
-
- - -""" - -JOURNAL_TEMPLATE_HTML_EXTRA = """\ -
\ -
- - {} -
-
-
-
- {}
-
-
- - -
-
""" - -JOURNAL_TEMPLATE_TEXT = """text:{title} -by {username}, {date} - -{content} -""" - - def eclipse_media(media, format="preview"): url = [media["baseUri"]] diff --git a/gallery_dl/extractor/utils/deviantart_journal.py b/gallery_dl/extractor/utils/deviantart_journal.py new file mode 100644 index 00000000..6c4ecf35 --- /dev/null +++ b/gallery_dl/extractor/utils/deviantart_journal.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +SHADOW = """ + + + +

+""" + +HEADER = """
+
+

{title}

+ +
+""" + +HEADER_CUSTOM = """
+

+ + {title} +

+Journal Entry: {date} +""" + +HTML = """text: + + + + {title} + + + + + + + + + + + + + +
+
+
+
+
+ {shadow} +
+
+
+ {html} +
+
+
+
+
+
+
+
+ + +""" + +HTML_EXTRA = """\ +
\ +
+ + {} +
+
+
+
+ {}
+
+
+ + +
+
""" + +TEXT = """text:{title} +by {username}, {date} + +{content} +""" From 51d9fd2f4d50114e743c53fb32307eec7f2b2a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 20:00:30 +0100 Subject: [PATCH 06/11] [behance] export GraphQL queries --- gallery_dl/extractor/behance.py | 416 +----------------- gallery_dl/extractor/utils/behance_graphql.py | 416 ++++++++++++++++++ 2 files changed, 418 insertions(+), 414 deletions(-) create mode 100644 gallery_dl/extractor/utils/behance_graphql.py diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index bb0562d8..eb3c250c 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2025 Mike Fährmann +# Copyright 2018-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -42,7 +42,7 @@ class BehanceExtractor(Extractor): "X-Requested-With": "XMLHttpRequest", } data = { - "query" : GRAPHQL_QUERIES[endpoint], + "query" : self.utils("graphql", endpoint), "variables": variables, } @@ -284,415 +284,3 @@ class BehanceCollectionExtractor(BehanceExtractor): if not items["pageInfo"]["hasNextPage"]: return variables["afterItem"] = items["pageInfo"]["endCursor"] - - -GRAPHQL_QUERIES = { - "GetProfileProjects": """\ -query GetProfileProjects($username: String, $after: String) { - user(username: $username) { - profileProjects(first: 12, after: $after) { - pageInfo { - endCursor - hasNextPage - } - nodes { - __typename - adminFlags { - mature_lock - privacy_lock - dmca_lock - flagged_lock - privacy_violation_lock - trademark_lock - spam_lock - eu_ip_lock - } - colors { - r - g - b - } - covers { - size_202 { - url - } - size_404 { - url - } - size_808 { - url - } - } - features { - url - name - featuredOn - ribbon { - image - image2x - image3x - } - } - fields { - id - label - slug - url - } - hasMatureContent - id - isFeatured - isHiddenFromWorkTab - isMatureReviewSubmitted - isOwner - isFounder - isPinnedToSubscriptionOverview - isPrivate - linkedAssets { - ...sourceLinkFields - } - linkedAssetsCount - sourceFiles { - ...sourceFileFields - } - matureAccess - modifiedOn - name - owners { - ...OwnerFields - images { - size_50 { - url - } - } - } - premium - publishedOn - stats { - appreciations { - all - } - views { - all - } - comments { - all - } - } - slug - tools { - id - title - category - categoryLabel - categoryId - approved - url - backgroundColor - } - url - } - } - } -} - -fragment sourceFileFields on SourceFile { - __typename - sourceFileId - projectId - userId - title - assetId - renditionUrl - mimeType - size - category - licenseType - unitAmount - currency - tier - hidden - extension - hasUserPurchased -} - -fragment sourceLinkFields on LinkedAsset { - __typename - name - premium - url - category - licenseType -} - -fragment OwnerFields on User { - displayName - hasPremiumAccess - id - isFollowing - isProfileOwner - location - locationUrl - url - username - availabilityInfo { - availabilityTimeline - isAvailableFullTime - isAvailableFreelance - } -} -""", - - "GetMoodboardItemsAndRecommendations": """\ -query GetMoodboardItemsAndRecommendations( - $id: Int! - $firstItem: Int! - $afterItem: String - $shouldGetRecommendations: Boolean! - $shouldGetItems: Boolean! - $shouldGetMoodboardFields: Boolean! -) { - viewer @include(if: $shouldGetMoodboardFields) { - isOptedOutOfRecommendations - isAdmin - } - moodboard(id: $id) { - ...moodboardFields @include(if: $shouldGetMoodboardFields) - - items(first: $firstItem, after: $afterItem) @include(if: $shouldGetItems) { - pageInfo { - endCursor - hasNextPage - } - nodes { - ...nodesFields - } - } - - recommendedItems(first: 80) @include(if: $shouldGetRecommendations) { - nodes { - ...nodesFields - fetchSource - } - } - } -} - -fragment moodboardFields on Moodboard { - id - label - privacy - followerCount - isFollowing - projectCount - url - isOwner - owners { - ...OwnerFields - images { - size_50 { - url - } - size_100 { - url - } - size_115 { - url - } - size_230 { - url - } - size_138 { - url - } - size_276 { - url - } - } - } -} - -fragment projectFields on Project { - __typename - id - isOwner - publishedOn - matureAccess - hasMatureContent - modifiedOn - name - url - isPrivate - slug - license { - license - description - id - label - url - text - images - } - fields { - label - } - colors { - r - g - b - } - owners { - ...OwnerFields - images { - size_50 { - url - } - size_100 { - url - } - size_115 { - url - } - size_230 { - url - } - size_138 { - url - } - size_276 { - url - } - } - } - covers { - size_original { - url - } - size_max_808 { - url - } - size_808 { - url - } - size_404 { - url - } - size_202 { - url - } - size_230 { - url - } - size_115 { - url - } - } - stats { - views { - all - } - appreciations { - all - } - comments { - all - } - } -} - -fragment exifDataValueFields on exifDataValue { - id - label - value - searchValue -} - -fragment nodesFields on MoodboardItem { - id - entityType - width - height - flexWidth - flexHeight - images { - size - url - } - - entity { - ... on Project { - ...projectFields - } - - ... on ImageModule { - project { - ...projectFields - } - - colors { - r - g - b - } - - exifData { - lens { - ...exifDataValueFields - } - software { - ...exifDataValueFields - } - makeAndModel { - ...exifDataValueFields - } - focalLength { - ...exifDataValueFields - } - iso { - ...exifDataValueFields - } - location { - ...exifDataValueFields - } - flash { - ...exifDataValueFields - } - exposureMode { - ...exifDataValueFields - } - shutterSpeed { - ...exifDataValueFields - } - aperture { - ...exifDataValueFields - } - } - } - - ... on MediaCollectionComponent { - project { - ...projectFields - } - } - } -} - -fragment OwnerFields on User { - displayName - hasPremiumAccess - id - isFollowing - isProfileOwner - location - locationUrl - url - username - availabilityInfo { - availabilityTimeline - isAvailableFullTime - isAvailableFreelance - } -} -""", - -} diff --git a/gallery_dl/extractor/utils/behance_graphql.py b/gallery_dl/extractor/utils/behance_graphql.py new file mode 100644 index 00000000..39aacffd --- /dev/null +++ b/gallery_dl/extractor/utils/behance_graphql.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +GetProfileProjects = """\ +query GetProfileProjects($username: String, $after: String) { + user(username: $username) { + profileProjects(first: 12, after: $after) { + pageInfo { + endCursor + hasNextPage + } + nodes { + __typename + adminFlags { + mature_lock + privacy_lock + dmca_lock + flagged_lock + privacy_violation_lock + trademark_lock + spam_lock + eu_ip_lock + } + colors { + r + g + b + } + covers { + size_202 { + url + } + size_404 { + url + } + size_808 { + url + } + } + features { + url + name + featuredOn + ribbon { + image + image2x + image3x + } + } + fields { + id + label + slug + url + } + hasMatureContent + id + isFeatured + isHiddenFromWorkTab + isMatureReviewSubmitted + isOwner + isFounder + isPinnedToSubscriptionOverview + isPrivate + linkedAssets { + ...sourceLinkFields + } + linkedAssetsCount + sourceFiles { + ...sourceFileFields + } + matureAccess + modifiedOn + name + owners { + ...OwnerFields + images { + size_50 { + url + } + } + } + premium + publishedOn + stats { + appreciations { + all + } + views { + all + } + comments { + all + } + } + slug + tools { + id + title + category + categoryLabel + categoryId + approved + url + backgroundColor + } + url + } + } + } +} + +fragment sourceFileFields on SourceFile { + __typename + sourceFileId + projectId + userId + title + assetId + renditionUrl + mimeType + size + category + licenseType + unitAmount + currency + tier + hidden + extension + hasUserPurchased +} + +fragment sourceLinkFields on LinkedAsset { + __typename + name + premium + url + category + licenseType +} + +fragment OwnerFields on User { + displayName + hasPremiumAccess + id + isFollowing + isProfileOwner + location + locationUrl + url + username + availabilityInfo { + availabilityTimeline + isAvailableFullTime + isAvailableFreelance + } +} +""" + +GetMoodboardItemsAndRecommendations = """\ +query GetMoodboardItemsAndRecommendations( + $id: Int! + $firstItem: Int! + $afterItem: String + $shouldGetRecommendations: Boolean! + $shouldGetItems: Boolean! + $shouldGetMoodboardFields: Boolean! +) { + viewer @include(if: $shouldGetMoodboardFields) { + isOptedOutOfRecommendations + isAdmin + } + moodboard(id: $id) { + ...moodboardFields @include(if: $shouldGetMoodboardFields) + + items(first: $firstItem, after: $afterItem) @include(if: $shouldGetItems) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ...nodesFields + } + } + + recommendedItems(first: 80) @include(if: $shouldGetRecommendations) { + nodes { + ...nodesFields + fetchSource + } + } + } +} + +fragment moodboardFields on Moodboard { + id + label + privacy + followerCount + isFollowing + projectCount + url + isOwner + owners { + ...OwnerFields + images { + size_50 { + url + } + size_100 { + url + } + size_115 { + url + } + size_230 { + url + } + size_138 { + url + } + size_276 { + url + } + } + } +} + +fragment projectFields on Project { + __typename + id + isOwner + publishedOn + matureAccess + hasMatureContent + modifiedOn + name + url + isPrivate + slug + license { + license + description + id + label + url + text + images + } + fields { + label + } + colors { + r + g + b + } + owners { + ...OwnerFields + images { + size_50 { + url + } + size_100 { + url + } + size_115 { + url + } + size_230 { + url + } + size_138 { + url + } + size_276 { + url + } + } + } + covers { + size_original { + url + } + size_max_808 { + url + } + size_808 { + url + } + size_404 { + url + } + size_202 { + url + } + size_230 { + url + } + size_115 { + url + } + } + stats { + views { + all + } + appreciations { + all + } + comments { + all + } + } +} + +fragment exifDataValueFields on exifDataValue { + id + label + value + searchValue +} + +fragment nodesFields on MoodboardItem { + id + entityType + width + height + flexWidth + flexHeight + images { + size + url + } + + entity { + ... on Project { + ...projectFields + } + + ... on ImageModule { + project { + ...projectFields + } + + colors { + r + g + b + } + + exifData { + lens { + ...exifDataValueFields + } + software { + ...exifDataValueFields + } + makeAndModel { + ...exifDataValueFields + } + focalLength { + ...exifDataValueFields + } + iso { + ...exifDataValueFields + } + location { + ...exifDataValueFields + } + flash { + ...exifDataValueFields + } + exposureMode { + ...exifDataValueFields + } + shutterSpeed { + ...exifDataValueFields + } + aperture { + ...exifDataValueFields + } + } + } + + ... on MediaCollectionComponent { + project { + ...projectFields + } + } + } +} + +fragment OwnerFields on User { + displayName + hasPremiumAccess + id + isFollowing + isProfileOwner + location + locationUrl + url + username + availabilityInfo { + availabilityTimeline + isAvailableFullTime + isAvailableFreelance + } +} +""" From 40a4ff935a68ea6e2e1862d3f3517b06e9fdda72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 21:40:23 +0100 Subject: [PATCH 07/11] [500px] export GraphQL queries --- gallery_dl/extractor/500px.py | 513 +------------------- gallery_dl/extractor/utils/500px_graphql.py | 512 +++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 515 insertions(+), 512 deletions(-) create mode 100644 gallery_dl/extractor/utils/500px_graphql.py diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index 748f23c4..53ae828e 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2025 Mike Fährmann +# Copyright 2019-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -83,7 +83,7 @@ class _500pxExtractor(Extractor): data = { "operationName": opname, "variables" : util.json_dumps(variables), - "query" : QUERIES[opname], + "query" : self.utils("graphql", opname), } return self.request_json( url, method="POST", headers=headers, json=data)["data"] @@ -212,512 +212,3 @@ class _500pxImageExtractor(_500pxExtractor): def photos(self): edges = ({"node": {"legacyId": self.photo_id}},) return self._extend(edges) - - -QUERIES = { - - "OtherPhotosQuery": """\ -query OtherPhotosQuery($username: String!, $pageSize: Int) { - user: userByUsername(username: $username) { - ...OtherPhotosPaginationContainer_user_RlXb8 - id - } -} - -fragment OtherPhotosPaginationContainer_user_RlXb8 on User { - photos(first: $pageSize, privacy: PROFILE, sort: ID_DESC) { - edges { - node { - id - legacyId - canonicalPath - width - height - name - isLikedByMe - notSafeForWork - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - followedByUsers { - isFollowedByMe - } - } - images(sizes: [33, 35]) { - size - url - jpegUrl - webpUrl - id - } - __typename - } - cursor - } - totalCount - pageInfo { - endCursor - hasNextPage - } - } -} -""", - - "OtherPhotosPaginationContainerQuery": """\ -query OtherPhotosPaginationContainerQuery($username: String!, $pageSize: Int, $cursor: String) { - userByUsername(username: $username) { - ...OtherPhotosPaginationContainer_user_3e6UuE - id - } -} - -fragment OtherPhotosPaginationContainer_user_3e6UuE on User { - photos(first: $pageSize, after: $cursor, privacy: PROFILE, sort: ID_DESC) { - edges { - node { - id - legacyId - canonicalPath - width - height - name - isLikedByMe - notSafeForWork - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - followedByUsers { - isFollowedByMe - } - } - images(sizes: [33, 35]) { - size - url - jpegUrl - webpUrl - id - } - __typename - } - cursor - } - totalCount - pageInfo { - endCursor - hasNextPage - } - } -} -""", - - "ProfileRendererQuery": """\ -query ProfileRendererQuery($username: String!) { - profile: userByUsername(username: $username) { - id - legacyId - userType: type - username - firstName - displayName - registeredAt - canonicalPath - avatar { - ...ProfileAvatar_avatar - id - } - userProfile { - firstname - lastname - state - country - city - about - id - } - socialMedia { - website - twitter - instagram - facebook - id - } - coverPhotoUrl - followedByUsers { - totalCount - isFollowedByMe - } - followingUsers { - totalCount - } - membership { - expiryDate - membershipTier: tier - photoUploadQuota - refreshPhotoUploadQuotaAt - paymentStatus - id - } - profileTabs { - tabs { - name - visible - } - } - ...EditCover_cover - photoStats { - likeCount - viewCount - } - photos(privacy: PROFILE) { - totalCount - } - licensingPhotos(status: ACCEPTED) { - totalCount - } - portfolio { - id - status - userDisabled - } - } -} - -fragment EditCover_cover on User { - coverPhotoUrl -} - -fragment ProfileAvatar_avatar on UserAvatar { - images(sizes: [MEDIUM, LARGE]) { - size - url - id - } -} -""", - - "GalleriesDetailQueryRendererQuery": """\ -query GalleriesDetailQueryRendererQuery($galleryOwnerLegacyId: ID!, $ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $gallerySize: Int) { - galleries(galleryOwnerLegacyId: $galleryOwnerLegacyId, first: $gallerySize) { - edges { - node { - legacyId - description - name - privacy - canonicalPath - notSafeForWork - buttonName - externalUrl - cover { - images(sizes: [35, 33]) { - size - webpUrl - jpegUrl - id - } - id - } - photos { - totalCount - } - id - } - } - } - gallery: galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) { - ...GalleriesDetailPaginationContainer_gallery_RlXb8 - id - } -} - -fragment GalleriesDetailPaginationContainer_gallery_RlXb8 on Gallery { - id - legacyId - name - privacy - notSafeForWork - ownPhotosOnly - canonicalPath - publicSlug - lastPublishedAt - photosAddedSinceLastPublished - reportStatus - creator { - legacyId - id - } - cover { - images(sizes: [33, 32, 36, 2048]) { - url - size - webpUrl - id - } - id - } - description - externalUrl - buttonName - photos(first: $pageSize) { - totalCount - edges { - cursor - node { - id - legacyId - canonicalPath - name - description - category - uploadedAt - location - width - height - isLikedByMe - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - avatar { - images(sizes: SMALL) { - url - id - } - id - } - followedByUsers { - totalCount - isFollowedByMe - } - } - images(sizes: [33, 32]) { - size - url - webpUrl - id - } - __typename - } - } - pageInfo { - endCursor - hasNextPage - } - } -} -""", - - "GalleriesDetailPaginationContainerQuery": """\ -query GalleriesDetailPaginationContainerQuery($ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $cursor: String) { - galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) { - ...GalleriesDetailPaginationContainer_gallery_3e6UuE - id - } -} - -fragment GalleriesDetailPaginationContainer_gallery_3e6UuE on Gallery { - id - legacyId - name - privacy - notSafeForWork - ownPhotosOnly - canonicalPath - publicSlug - lastPublishedAt - photosAddedSinceLastPublished - reportStatus - creator { - legacyId - id - } - cover { - images(sizes: [33, 32, 36, 2048]) { - url - size - webpUrl - id - } - id - } - description - externalUrl - buttonName - photos(first: $pageSize, after: $cursor) { - totalCount - edges { - cursor - node { - id - legacyId - canonicalPath - name - description - category - uploadedAt - location - width - height - isLikedByMe - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - avatar { - images(sizes: SMALL) { - url - id - } - id - } - followedByUsers { - totalCount - isFollowedByMe - } - } - images(sizes: [33, 32]) { - size - url - webpUrl - id - } - __typename - } - } - pageInfo { - endCursor - hasNextPage - } - } -} -""", - - "LikedPhotosQueryRendererQuery": """\ -query LikedPhotosQueryRendererQuery($pageSize: Int) { - ...LikedPhotosPaginationContainer_query_RlXb8 -} - -fragment LikedPhotosPaginationContainer_query_RlXb8 on Query { - likedPhotos(first: $pageSize) { - edges { - node { - id - legacyId - canonicalPath - name - description - category - uploadedAt - location - width - height - isLikedByMe - notSafeForWork - tags - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - avatar { - images { - url - id - } - id - } - followedByUsers { - totalCount - isFollowedByMe - } - } - images(sizes: [33, 35]) { - size - url - jpegUrl - webpUrl - id - } - __typename - } - cursor - } - pageInfo { - endCursor - hasNextPage - } - } -} -""", - - "LikedPhotosPaginationContainerQuery": """\ -query LikedPhotosPaginationContainerQuery($cursor: String, $pageSize: Int) { - ...LikedPhotosPaginationContainer_query_3e6UuE -} - -fragment LikedPhotosPaginationContainer_query_3e6UuE on Query { - likedPhotos(first: $pageSize, after: $cursor) { - edges { - node { - id - legacyId - canonicalPath - name - description - category - uploadedAt - location - width - height - isLikedByMe - notSafeForWork - tags - photographer: uploader { - id - legacyId - username - displayName - canonicalPath - avatar { - images { - url - id - } - id - } - followedByUsers { - totalCount - isFollowedByMe - } - } - images(sizes: [33, 35]) { - size - url - jpegUrl - webpUrl - id - } - __typename - } - cursor - } - pageInfo { - endCursor - hasNextPage - } - } -} -""", - -} diff --git a/gallery_dl/extractor/utils/500px_graphql.py b/gallery_dl/extractor/utils/500px_graphql.py new file mode 100644 index 00000000..aaab5d66 --- /dev/null +++ b/gallery_dl/extractor/utils/500px_graphql.py @@ -0,0 +1,512 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +OtherPhotosQuery = """\ +query OtherPhotosQuery($username: String!, $pageSize: Int) { + user: userByUsername(username: $username) { + ...OtherPhotosPaginationContainer_user_RlXb8 + id + } +} + +fragment OtherPhotosPaginationContainer_user_RlXb8 on User { + photos(first: $pageSize, privacy: PROFILE, sort: ID_DESC) { + edges { + node { + id + legacyId + canonicalPath + width + height + name + isLikedByMe + notSafeForWork + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + followedByUsers { + isFollowedByMe + } + } + images(sizes: [33, 35]) { + size + url + jpegUrl + webpUrl + id + } + __typename + } + cursor + } + totalCount + pageInfo { + endCursor + hasNextPage + } + } +} +""" + +OtherPhotosPaginationContainerQuery = """\ +query OtherPhotosPaginationContainerQuery($username: String!, $pageSize: Int, $cursor: String) { + userByUsername(username: $username) { + ...OtherPhotosPaginationContainer_user_3e6UuE + id + } +} + +fragment OtherPhotosPaginationContainer_user_3e6UuE on User { + photos(first: $pageSize, after: $cursor, privacy: PROFILE, sort: ID_DESC) { + edges { + node { + id + legacyId + canonicalPath + width + height + name + isLikedByMe + notSafeForWork + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + followedByUsers { + isFollowedByMe + } + } + images(sizes: [33, 35]) { + size + url + jpegUrl + webpUrl + id + } + __typename + } + cursor + } + totalCount + pageInfo { + endCursor + hasNextPage + } + } +} +""" + +ProfileRendererQuery = """\ +query ProfileRendererQuery($username: String!) { + profile: userByUsername(username: $username) { + id + legacyId + userType: type + username + firstName + displayName + registeredAt + canonicalPath + avatar { + ...ProfileAvatar_avatar + id + } + userProfile { + firstname + lastname + state + country + city + about + id + } + socialMedia { + website + twitter + instagram + facebook + id + } + coverPhotoUrl + followedByUsers { + totalCount + isFollowedByMe + } + followingUsers { + totalCount + } + membership { + expiryDate + membershipTier: tier + photoUploadQuota + refreshPhotoUploadQuotaAt + paymentStatus + id + } + profileTabs { + tabs { + name + visible + } + } + ...EditCover_cover + photoStats { + likeCount + viewCount + } + photos(privacy: PROFILE) { + totalCount + } + licensingPhotos(status: ACCEPTED) { + totalCount + } + portfolio { + id + status + userDisabled + } + } +} + +fragment EditCover_cover on User { + coverPhotoUrl +} + +fragment ProfileAvatar_avatar on UserAvatar { + images(sizes: [MEDIUM, LARGE]) { + size + url + id + } +} +""" + +GalleriesDetailQueryRendererQuery = """\ +query GalleriesDetailQueryRendererQuery($galleryOwnerLegacyId: ID!, $ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $gallerySize: Int) { + galleries(galleryOwnerLegacyId: $galleryOwnerLegacyId, first: $gallerySize) { + edges { + node { + legacyId + description + name + privacy + canonicalPath + notSafeForWork + buttonName + externalUrl + cover { + images(sizes: [35, 33]) { + size + webpUrl + jpegUrl + id + } + id + } + photos { + totalCount + } + id + } + } + } + gallery: galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) { + ...GalleriesDetailPaginationContainer_gallery_RlXb8 + id + } +} + +fragment GalleriesDetailPaginationContainer_gallery_RlXb8 on Gallery { + id + legacyId + name + privacy + notSafeForWork + ownPhotosOnly + canonicalPath + publicSlug + lastPublishedAt + photosAddedSinceLastPublished + reportStatus + creator { + legacyId + id + } + cover { + images(sizes: [33, 32, 36, 2048]) { + url + size + webpUrl + id + } + id + } + description + externalUrl + buttonName + photos(first: $pageSize) { + totalCount + edges { + cursor + node { + id + legacyId + canonicalPath + name + description + category + uploadedAt + location + width + height + isLikedByMe + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + avatar { + images(sizes: SMALL) { + url + id + } + id + } + followedByUsers { + totalCount + isFollowedByMe + } + } + images(sizes: [33, 32]) { + size + url + webpUrl + id + } + __typename + } + } + pageInfo { + endCursor + hasNextPage + } + } +} +""" + +GalleriesDetailPaginationContainerQuery = """\ +query GalleriesDetailPaginationContainerQuery($ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $cursor: String) { + galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) { + ...GalleriesDetailPaginationContainer_gallery_3e6UuE + id + } +} + +fragment GalleriesDetailPaginationContainer_gallery_3e6UuE on Gallery { + id + legacyId + name + privacy + notSafeForWork + ownPhotosOnly + canonicalPath + publicSlug + lastPublishedAt + photosAddedSinceLastPublished + reportStatus + creator { + legacyId + id + } + cover { + images(sizes: [33, 32, 36, 2048]) { + url + size + webpUrl + id + } + id + } + description + externalUrl + buttonName + photos(first: $pageSize, after: $cursor) { + totalCount + edges { + cursor + node { + id + legacyId + canonicalPath + name + description + category + uploadedAt + location + width + height + isLikedByMe + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + avatar { + images(sizes: SMALL) { + url + id + } + id + } + followedByUsers { + totalCount + isFollowedByMe + } + } + images(sizes: [33, 32]) { + size + url + webpUrl + id + } + __typename + } + } + pageInfo { + endCursor + hasNextPage + } + } +} +""" + +LikedPhotosQueryRendererQuery = """\ +query LikedPhotosQueryRendererQuery($pageSize: Int) { + ...LikedPhotosPaginationContainer_query_RlXb8 +} + +fragment LikedPhotosPaginationContainer_query_RlXb8 on Query { + likedPhotos(first: $pageSize) { + edges { + node { + id + legacyId + canonicalPath + name + description + category + uploadedAt + location + width + height + isLikedByMe + notSafeForWork + tags + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + avatar { + images { + url + id + } + id + } + followedByUsers { + totalCount + isFollowedByMe + } + } + images(sizes: [33, 35]) { + size + url + jpegUrl + webpUrl + id + } + __typename + } + cursor + } + pageInfo { + endCursor + hasNextPage + } + } +} +""" + +LikedPhotosPaginationContainerQuery = """\ +query LikedPhotosPaginationContainerQuery($cursor: String, $pageSize: Int) { + ...LikedPhotosPaginationContainer_query_3e6UuE +} + +fragment LikedPhotosPaginationContainer_query_3e6UuE on Query { + likedPhotos(first: $pageSize, after: $cursor) { + edges { + node { + id + legacyId + canonicalPath + name + description + category + uploadedAt + location + width + height + isLikedByMe + notSafeForWork + tags + photographer: uploader { + id + legacyId + username + displayName + canonicalPath + avatar { + images { + url + id + } + id + } + followedByUsers { + totalCount + isFollowedByMe + } + } + images(sizes: [33, 35]) { + size + url + jpegUrl + webpUrl + id + } + __typename + } + cursor + } + pageInfo { + endCursor + hasNextPage + } + } +} +""" diff --git a/setup.cfg b/setup.cfg index a5e01b66..7fb6eece 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,6 @@ exclude = .git,__pycache__,build,dist,archive ignore = E203,E226,W504 per-file-ignores = setup.py: E501 - gallery_dl/extractor/500px.py: E501 + gallery_dl/extractor/utils/500px_graphql.py: E501 gallery_dl/extractor/mangapark.py: E501 test/results/*.py: E122,E241,E402,E501 From 0c2495550738861334fb4b0b419094c66ec4b8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 31 Jan 2026 22:10:57 +0100 Subject: [PATCH 08/11] [mangapark] export GraphQL queries --- gallery_dl/extractor/mangapark.py | 184 +---------------- .../extractor/utils/mangapark_graphql.py | 185 ++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 188 insertions(+), 183 deletions(-) create mode 100644 gallery_dl/extractor/utils/mangapark_graphql.py diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index cd72850f..05899e99 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2025 Mike Fährmann +# Copyright 2015-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -60,7 +60,7 @@ class MangaparkBase(): def _request_graphql(self, opname, variables): url = self.root + "/apo/" data = { - "query" : QUERIES[opname], + "query" : self.utils("graphql", opname), "variables" : variables, "operationName": opname, } @@ -177,183 +177,3 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor): raise exception.AbortExtraction( f"'{source}' does not match any available source") - - -QUERIES = { - "Get_comicChapterList": """ -query Get_comicChapterList($comicId: ID!) { - get_comicChapterList(comicId: $comicId) { - data { - id - dname - title - lang - urlPath - srcTitle - sourceId - dateCreate - } - } -} -""", - - "Get_chapterNode": """ -query Get_chapterNode($getChapterNodeId: ID!) { - get_chapterNode(id: $getChapterNodeId) { - data { - id - dname - lang - sourceId - srcTitle - dateCreate - comicNode{ - id - } - imageFile { - urlList - } - } - } -} -""", - - "Get_comicNode": """ -query Get_comicNode($getComicNodeId: ID!) { - get_comicNode(id: $getComicNodeId) { - data { - id - name - artists - authors - genres - } - } -} -""", - - "get_content_source_chapterList": """ - query get_content_source_chapterList($sourceId: Int!) { - get_content_source_chapterList( - sourceId: $sourceId - ) { - - id - data { - - - id - sourceId - - dbStatus - isNormal - isHidden - isDeleted - isFinal - - dateCreate - datePublic - dateModify - lang - volume - serial - dname - title - urlPath - - srcTitle srcColor - - count_images - - stat_count_post_child - stat_count_post_reply - stat_count_views_login - stat_count_views_guest - - userId - userNode { - - id - data { - -id -name -uniq -avatarUrl -urlPath - -verified -deleted -banned - -dateCreate -dateOnline - -stat_count_chapters_normal -stat_count_chapters_others - -is_adm is_mod is_vip is_upr - - } - - } - - disqusId - - - } - - } - } -""", - - "get_content_comic_sources": """ - query get_content_comic_sources($comicId: Int!, $dbStatuss: [String] = [], $userId: Int, $haveChapter: Boolean, $sortFor: String) { - get_content_comic_sources( - comicId: $comicId - dbStatuss: $dbStatuss - userId: $userId - haveChapter: $haveChapter - sortFor: $sortFor - ) { - -id -data{ - - id - - dbStatus - isNormal - isHidden - isDeleted - - lang name altNames authors artists - - release - genres summary{code} extraInfo{code} - - urlCover600 - urlCover300 - urlCoverOri - - srcTitle srcColor - - chapterCount - chapterNode_last { - id - data { - dateCreate datePublic dateModify - volume serial - dname title - urlPath - userNode { - id data {uniq name} - } - } - } -} - - } - } -""", -} diff --git a/gallery_dl/extractor/utils/mangapark_graphql.py b/gallery_dl/extractor/utils/mangapark_graphql.py new file mode 100644 index 00000000..271f2d53 --- /dev/null +++ b/gallery_dl/extractor/utils/mangapark_graphql.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +Get_comicChapterList = """ +query Get_comicChapterList($comicId: ID!) { + get_comicChapterList(comicId: $comicId) { + data { + id + dname + title + lang + urlPath + srcTitle + sourceId + dateCreate + } + } +} +""" + +Get_chapterNode = """ +query Get_chapterNode($getChapterNodeId: ID!) { + get_chapterNode(id: $getChapterNodeId) { + data { + id + dname + lang + sourceId + srcTitle + dateCreate + comicNode{ + id + } + imageFile { + urlList + } + } + } +} +""" + +Get_comicNode = """ +query Get_comicNode($getComicNodeId: ID!) { + get_comicNode(id: $getComicNodeId) { + data { + id + name + artists + authors + genres + } + } +} +""" + +get_content_source_chapterList = """ + query get_content_source_chapterList($sourceId: Int!) { + get_content_source_chapterList( + sourceId: $sourceId + ) { + + id + data { + + + id + sourceId + + dbStatus + isNormal + isHidden + isDeleted + isFinal + + dateCreate + datePublic + dateModify + lang + volume + serial + dname + title + urlPath + + srcTitle srcColor + + count_images + + stat_count_post_child + stat_count_post_reply + stat_count_views_login + stat_count_views_guest + + userId + userNode { + + id + data { + +id +name +uniq +avatarUrl +urlPath + +verified +deleted +banned + +dateCreate +dateOnline + +stat_count_chapters_normal +stat_count_chapters_others + +is_adm is_mod is_vip is_upr + + } + + } + + disqusId + + + } + + } + } +""" + +get_content_comic_sources = """ + query get_content_comic_sources($comicId: Int!, $dbStatuss: [String] = [], $userId: Int, $haveChapter: Boolean, $sortFor: String) { + get_content_comic_sources( + comicId: $comicId + dbStatuss: $dbStatuss + userId: $userId + haveChapter: $haveChapter + sortFor: $sortFor + ) { + +id +data{ + + id + + dbStatus + isNormal + isHidden + isDeleted + + lang name altNames authors artists + + release + genres summary{code} extraInfo{code} + + urlCover600 + urlCover300 + urlCoverOri + + srcTitle srcColor + + chapterCount + chapterNode_last { + id + data { + dateCreate datePublic dateModify + volume serial + dname title + urlPath + userNode { + id data {uniq name} + } + } + } +} + + } + } +""" diff --git a/setup.cfg b/setup.cfg index 7fb6eece..a9a1b150 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,5 +4,5 @@ ignore = E203,E226,W504 per-file-ignores = setup.py: E501 gallery_dl/extractor/utils/500px_graphql.py: E501 - gallery_dl/extractor/mangapark.py: E501 + gallery_dl/extractor/utils/mangapark_graphql.py: E501 test/results/*.py: E122,E241,E402,E501 From cc645984a4929abb62ee009f1a50e8a5f0f49e04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 21 Oct 2025 16:56:34 +0200 Subject: [PATCH 09/11] [luscious] export GraphQL queries --- gallery_dl/extractor/luscious.py | 257 +----------------- .../extractor/utils/luscious_graphql.py | 240 ++++++++++++++++ 2 files changed, 250 insertions(+), 247 deletions(-) create mode 100644 gallery_dl/extractor/utils/luscious_graphql.py diff --git a/gallery_dl/extractor/luscious.py b/gallery_dl/extractor/luscious.py index 2abd1c89..96c607a1 100644 --- a/gallery_dl/extractor/luscious.py +++ b/gallery_dl/extractor/luscious.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2025 Mike Fährmann +# Copyright 2016-2026 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,15 +18,15 @@ class LusciousExtractor(Extractor): cookies_domain = ".luscious.net" root = "https://members.luscious.net" - def _graphql(self, op, variables, query): + def _request_graphql(self, opname, variables): data = { "id" : 1, - "operationName": op, - "query" : query, + "operationName": opname, + "query" : self.utils("graphql", opname), "variables" : variables, } response = self.request( - f"{self.root}/graphql/nobatch/?operationName={op}", + f"{self.root}/graphql/nobatch/?operationName={opname}", method="POST", json=data, fatal=False, ) @@ -49,11 +49,8 @@ class LusciousAlbumExtractor(LusciousExtractor): r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)") example = "https://luscious.net/albums/TITLE_12345/" - def __init__(self, match): - LusciousExtractor.__init__(self, match) - self.album_id = match[1] - def _init(self): + self.album_id = self.groups[0] self.gif = self.config("gif", False) def items(self): @@ -83,98 +80,7 @@ class LusciousAlbumExtractor(LusciousExtractor): "id": self.album_id, } - query = """ -query AlbumGet($id: ID!) { - album { - get(id: $id) { - ... on Album { - ...AlbumStandard - } - ... on MutationError { - errors { - code - message - } - } - } - } -} - -fragment AlbumStandard on Album { - __typename - id - title - labels - description - created - modified - like_status - number_of_favorites - rating - status - marked_for_deletion - marked_for_processing - number_of_pictures - number_of_animated_pictures - slug - is_manga - url - download_url - permissions - cover { - width - height - size - url - } - created_by { - id - name - display_name - user_title - avatar { - url - size - } - url - } - content { - id - title - url - } - language { - id - title - url - } - tags { - id - category - text - url - count - } - genres { - id - title - slug - url - } - audiences { - id - title - url - url - } - last_viewed_picture { - id - position - url - } -} -""" - album = self._graphql("AlbumGet", variables, query)["album"]["get"] + album = self._request_graphql("AlbumGet", variables)["album"]["get"] if "errors" in album: raise exception.NotFoundError("album") @@ -204,66 +110,8 @@ fragment AlbumStandard on Album { }, } - query = """ -query AlbumListOwnPictures($input: PictureListInput!) { - picture { - list(input: $input) { - info { - ...FacetCollectionInfo - } - items { - ...PictureStandardWithoutAlbum - } - } - } -} - -fragment FacetCollectionInfo on FacetCollectionInfo { - page - has_next_page - has_previous_page - total_items - total_pages - items_per_page - url_complete - url_filters_only -} - -fragment PictureStandardWithoutAlbum on Picture { - __typename - id - title - created - like_status - number_of_comments - number_of_favorites - status - width - height - resolution - aspect_ratio - url_to_original - url_to_video - is_animated - position - tags { - id - category - text - url - } - permissions - url - thumbnails { - width - height - size - url - } -} -""" while True: - data = self._graphql("AlbumListOwnPictures", variables, query) + data = self._request_graphql("AlbumListOwnPictures", variables) yield from data["picture"]["list"]["items"] if not data["picture"]["list"]["info"]["has_next_page"]: @@ -278,12 +126,8 @@ class LusciousSearchExtractor(LusciousExtractor): r"/albums/list/?(?:\?([^#]+))?") example = "https://luscious.net/albums/list/?tagged=TAG" - def __init__(self, match): - LusciousExtractor.__init__(self, match) - self.query = match[1] - def items(self): - query = text.parse_query(self.query) + query = text.parse_query(self.groups[0]) display = query.pop("display", "date_newest") page = query.pop("page", None) @@ -295,89 +139,8 @@ class LusciousSearchExtractor(LusciousExtractor): }, } - query = """ -query AlbumListWithPeek($input: AlbumListInput!) { - album { - list(input: $input) { - info { - ...FacetCollectionInfo - } - items { - ...AlbumMinimal - peek_thumbnails { - width - height - size - url - } - } - } - } -} - -fragment FacetCollectionInfo on FacetCollectionInfo { - page - has_next_page - has_previous_page - total_items - total_pages - items_per_page - url_complete - url_filters_only -} - -fragment AlbumMinimal on Album { - __typename - id - title - labels - description - created - modified - number_of_favorites - number_of_pictures - slug - is_manga - url - download_url - cover { - width - height - size - url - } - content { - id - title - url - } - language { - id - title - url - } - tags { - id - category - text - url - count - } - genres { - id - title - slug - url - } - audiences { - id - title - url - } -} -""" while True: - data = self._graphql("AlbumListWithPeek", variables, query) + data = self._request_graphql("AlbumListWithPeek", variables) for album in data["album"]["list"]["items"]: album["url"] = self.root + album["url"] diff --git a/gallery_dl/extractor/utils/luscious_graphql.py b/gallery_dl/extractor/utils/luscious_graphql.py new file mode 100644 index 00000000..dd127c9d --- /dev/null +++ b/gallery_dl/extractor/utils/luscious_graphql.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +AlbumGet = """ +query AlbumGet($id: ID!) { + album { + get(id: $id) { + ... on Album { + ...AlbumStandard + } + ... on MutationError { + errors { + code + message + } + } + } + } +} + +fragment AlbumStandard on Album { + __typename + id + title + labels + description + created + modified + like_status + number_of_favorites + rating + status + marked_for_deletion + marked_for_processing + number_of_pictures + number_of_animated_pictures + slug + is_manga + url + download_url + permissions + cover { + width + height + size + url + } + created_by { + id + name + display_name + user_title + avatar { + url + size + } + url + } + content { + id + title + url + } + language { + id + title + url + } + tags { + id + category + text + url + count + } + genres { + id + title + slug + url + } + audiences { + id + title + url + url + } + last_viewed_picture { + id + position + url + } +} +""" + +AlbumListOwnPictures = """ +query AlbumListOwnPictures($input: PictureListInput!) { + picture { + list(input: $input) { + info { + ...FacetCollectionInfo + } + items { + ...PictureStandardWithoutAlbum + } + } + } +} + +fragment FacetCollectionInfo on FacetCollectionInfo { + page + has_next_page + has_previous_page + total_items + total_pages + items_per_page + url_complete + url_filters_only +} + +fragment PictureStandardWithoutAlbum on Picture { + __typename + id + title + created + like_status + number_of_comments + number_of_favorites + status + width + height + resolution + aspect_ratio + url_to_original + url_to_video + is_animated + position + tags { + id + category + text + url + } + permissions + url + thumbnails { + width + height + size + url + } +} +""" + +AlbumListWithPeek = """ +query AlbumListWithPeek($input: AlbumListInput!) { + album { + list(input: $input) { + info { + ...FacetCollectionInfo + } + items { + ...AlbumMinimal + peek_thumbnails { + width + height + size + url + } + } + } + } +} + +fragment FacetCollectionInfo on FacetCollectionInfo { + page + has_next_page + has_previous_page + total_items + total_pages + items_per_page + url_complete + url_filters_only +} + +fragment AlbumMinimal on Album { + __typename + id + title + labels + description + created + modified + number_of_favorites + number_of_pictures + slug + is_manga + url + download_url + cover { + width + height + size + url + } + content { + id + title + url + } + language { + id + title + url + } + tags { + id + category + text + url + count + } + genres { + id + title + slug + url + } + audiences { + id + title + url + } +} +""" From 73bf99612adca4580bae314aeab8bcd011165391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 21 Oct 2025 17:04:12 +0200 Subject: [PATCH 10/11] [scrolller] move GraphQL queries --- gallery_dl/extractor/scrolller.py | 143 +---------------- .../extractor/utils/scrolller_graphql.py | 144 ++++++++++++++++++ 2 files changed, 145 insertions(+), 142 deletions(-) create mode 100644 gallery_dl/extractor/utils/scrolller_graphql.py diff --git a/gallery_dl/extractor/scrolller.py b/gallery_dl/extractor/scrolller.py index 3fce2cf6..a48fdb60 100644 --- a/gallery_dl/extractor/scrolller.py +++ b/gallery_dl/extractor/scrolller.py @@ -98,7 +98,7 @@ class ScrolllerExtractor(Extractor): "Sec-Fetch-Site": "same-site", } data = { - "query" : QUERIES[opname], + "query" : self.utils("graphql", opname), "variables" : variables, "authorization": self.auth_token, } @@ -206,144 +206,3 @@ class ScrolllerPostExtractor(ScrolllerExtractor): variables = {"url": "/" + self.groups[0]} data = self._request_graphql("SubredditPostQuery", variables) return (data["getPost"],) - - -QUERIES = { - - "SubredditPostQuery": """\ -query SubredditPostQuery( - $url: String! -) { - getPost( - data: { url: $url } - ) { - __typename id url title subredditId subredditTitle subredditUrl - redditPath isNsfw hasAudio fullLengthSource gfycatSource redgifsSource - ownerAvatar username displayName favoriteCount isPaid tags - commentsCount commentsRepliesCount isFavorite - albumContent { mediaSources { url width height isOptimized } } - mediaSources { url width height isOptimized } - blurredMediaSources { url width height isOptimized } - } -} -""", - - "SubredditQuery": """\ -query SubredditQuery( - $url: String! - $iterator: String - $sortBy: GallerySortBy - $filter: GalleryFilter - $limit: Int! -) { - getSubreddit( - data: { - url: $url, - iterator: $iterator, - filter: $filter, - limit: $limit, - sortBy: $sortBy - } - ) { - __typename id url title secondaryTitle description createdAt isNsfw - subscribers isComplete itemCount videoCount pictureCount albumCount - isPaid username tags isFollowing - banner { url width height isOptimized } - children { - iterator items { - __typename id url title subredditId subredditTitle subredditUrl - redditPath isNsfw hasAudio fullLengthSource gfycatSource - redgifsSource ownerAvatar username displayName favoriteCount - isPaid tags commentsCount commentsRepliesCount isFavorite - albumContent { mediaSources { url width height isOptimized } } - mediaSources { url width height isOptimized } - blurredMediaSources { url width height isOptimized } - } - } - } -} -""", - - "SubredditChildrenQuery": """\ -query SubredditChildrenQuery( - $subredditId: Int! - $iterator: String - $filter: GalleryFilter - $sortBy: GallerySortBy - $limit: Int! - $isNsfw: Boolean -) { - getSubredditChildren( - data: { - subredditId: $subredditId, - iterator: $iterator, - filter: $filter, - sortBy: $sortBy, - limit: $limit, - isNsfw: $isNsfw - }, - ) { - iterator items { - __typename id url title subredditId subredditTitle subredditUrl - redditPath isNsfw hasAudio fullLengthSource gfycatSource - redgifsSource ownerAvatar username displayName favoriteCount isPaid - tags commentsCount commentsRepliesCount isFavorite - albumContent { mediaSources { url width height isOptimized } } - mediaSources { url width height isOptimized } - blurredMediaSources { url width height isOptimized } - } - } -} -""", - - "GetFollowingSubreddits": """\ -query GetFollowingSubreddits( - $iterator: String, - $limit: Int!, - $filter: GalleryFilter, - $isNsfw: Boolean, - $sortBy: GallerySortBy -) { - getFollowingSubreddits( - data: { - isNsfw: $isNsfw - limit: $limit - filter: $filter - iterator: $iterator - sortBy: $sortBy - } - ) { - iterator items { - __typename id url title secondaryTitle description createdAt isNsfw - subscribers isComplete itemCount videoCount pictureCount albumCount - isFollowing - } - } -} -""", - - "LoginQuery": """\ -query LoginQuery( - $username: String!, - $password: String! -) { - login( - username: $username, - password: $password - ) { - username token expiresAt isAdmin status isPremium - } -} -""", - - "ItemTypeQuery": """\ -query ItemTypeQuery( - $url: String! -) { - getItemType( - url: $url - ) -} -""", - -} diff --git a/gallery_dl/extractor/utils/scrolller_graphql.py b/gallery_dl/extractor/utils/scrolller_graphql.py new file mode 100644 index 00000000..bb7d1d7d --- /dev/null +++ b/gallery_dl/extractor/utils/scrolller_graphql.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + + +SubredditPostQuery = """\ +query SubredditPostQuery( + $url: String! +) { + getPost( + data: { url: $url } + ) { + __typename id url title subredditId subredditTitle subredditUrl + redditPath isNsfw hasAudio fullLengthSource gfycatSource redgifsSource + ownerAvatar username displayName favoriteCount isPaid tags + commentsCount commentsRepliesCount isFavorite + albumContent { mediaSources { url width height isOptimized } } + mediaSources { url width height isOptimized } + blurredMediaSources { url width height isOptimized } + } +} +""" + +SubredditQuery = """\ +query SubredditQuery( + $url: String! + $iterator: String + $sortBy: GallerySortBy + $filter: GalleryFilter + $limit: Int! +) { + getSubreddit( + data: { + url: $url, + iterator: $iterator, + filter: $filter, + limit: $limit, + sortBy: $sortBy + } + ) { + __typename id url title secondaryTitle description createdAt isNsfw + subscribers isComplete itemCount videoCount pictureCount albumCount + isPaid username tags isFollowing + banner { url width height isOptimized } + children { + iterator items { + __typename id url title subredditId subredditTitle subredditUrl + redditPath isNsfw hasAudio fullLengthSource gfycatSource + redgifsSource ownerAvatar username displayName favoriteCount + isPaid tags commentsCount commentsRepliesCount isFavorite + albumContent { mediaSources { url width height isOptimized } } + mediaSources { url width height isOptimized } + blurredMediaSources { url width height isOptimized } + } + } + } +} +""" + +SubredditChildrenQuery = """\ +query SubredditChildrenQuery( + $subredditId: Int! + $iterator: String + $filter: GalleryFilter + $sortBy: GallerySortBy + $limit: Int! + $isNsfw: Boolean +) { + getSubredditChildren( + data: { + subredditId: $subredditId, + iterator: $iterator, + filter: $filter, + sortBy: $sortBy, + limit: $limit, + isNsfw: $isNsfw + }, + ) { + iterator items { + __typename id url title subredditId subredditTitle subredditUrl + redditPath isNsfw hasAudio fullLengthSource gfycatSource + redgifsSource ownerAvatar username displayName favoriteCount isPaid + tags commentsCount commentsRepliesCount isFavorite + albumContent { mediaSources { url width height isOptimized } } + mediaSources { url width height isOptimized } + blurredMediaSources { url width height isOptimized } + } + } +} +""" + +GetFollowingSubreddits = """\ +query GetFollowingSubreddits( + $iterator: String, + $limit: Int!, + $filter: GalleryFilter, + $isNsfw: Boolean, + $sortBy: GallerySortBy +) { + getFollowingSubreddits( + data: { + isNsfw: $isNsfw + limit: $limit + filter: $filter + iterator: $iterator + sortBy: $sortBy + } + ) { + iterator items { + __typename id url title secondaryTitle description createdAt isNsfw + subscribers isComplete itemCount videoCount pictureCount albumCount + isFollowing + } + } +} +""" + +LoginQuery = """\ +query LoginQuery( + $username: String!, + $password: String! +) { + login( + username: $username, + password: $password + ) { + username token expiresAt isAdmin status isPremium + } +} +""" + +ItemTypeQuery = """\ +query ItemTypeQuery( + $url: String! +) { + getItemType( + url: $url + ) +} +""" From eed46f8dcf7028762e81314fe4feab62e3a51cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 25 Oct 2025 18:35:55 +0200 Subject: [PATCH 11/11] [build] update PyInstaller hiddenimports and py2exe modules --- scripts/hook-gallery_dl.py | 11 ++++++++++- setup.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/hook-gallery_dl.py b/scripts/hook-gallery_dl.py index ae51b068..dd97ec9b 100644 --- a/scripts/hook-gallery_dl.py +++ b/scripts/hook-gallery_dl.py @@ -1,11 +1,20 @@ # -*- coding: utf-8 -*- from gallery_dl import extractor, downloader, postprocessor +import os hiddenimports = [ - package.__name__ + "." + module + f"{package.__name__}.{module}" for package in (extractor, downloader, postprocessor) for module in package.modules ] +base = extractor.__name__ + ".utils." +path = os.path.join(extractor.__path__[0], "utils") +hiddenimports.extend( + base + file[:-3] + for file in os.listdir(path) + if not file.startswith("__") +) + hiddenimports.append("yt_dlp") diff --git a/setup.py b/setup.py index fd2bff7b..72d5648f 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ FILES = [ PACKAGES = [ "gallery_dl", "gallery_dl.extractor", + "gallery_dl.extractor.utils", "gallery_dl.downloader", "gallery_dl.postprocessor", ]