diff --git a/gallery_dl/extractor/arena.py b/gallery_dl/extractor/arena.py index b464f317..ada2fa1d 100644 --- a/gallery_dl/extractor/arena.py +++ b/gallery_dl/extractor/arena.py @@ -16,22 +16,28 @@ class ArenaChannelExtractor(GalleryExtractor): category = "arena" subcategory = "channel" root = "https://are.na" + directory_fmt = ("{category}", "{user[full_name]} ({user[id]})", + "{channel[title]} ({channel[id]})") + filename_fmt = "{num:>03}{block[id]:? //}.{extension}" + archive_fmt = "{channel[id]}/{block[id]}" pattern = r"(?:https?://)?(?:www\.)?are\.na/[^/?#]+/([^/?#]+)" example = "https://are.na/evan-collins-1522646491/cassette-futurism" def metadata(self, page): - info = self.request_json( + channel = self.request_json( f"https://api.are.na/v2/channels/{self.groups[0]}") + channel["date"] = self.parse_datetime_iso( + channel["created_at"]) + channel["date_updated"] = self.parse_datetime_iso( + channel["updated_at"]) + channel.pop("contents", None) + return { - "gallery_id" : info.get("slug") or str(info.get("id")), - "channel_id" : info.get("id"), - "channel_slug": info.get("slug"), - "title" : info.get("title") or "", - "count" : info.get("length") or 0, - "user" : info.get("user"), - "date" : self.parse_datetime_iso(info.get("created_at")), - "date_updated": self.parse_datetime_iso(info.get("updated_at")), + "count" : channel.get("length"), + "user" : channel.pop("user", None), + "owner" : channel.pop("owner", None), + "channel": channel, } def images(self, page): @@ -48,12 +54,6 @@ class ArenaChannelExtractor(GalleryExtractor): for block in contents: url = None - meta = { - "id": block.get("id"), - "block_class": block.get("class"), - "block_title": block.get("title") or block.get( - "generated_title") or "", - } # Attachments (e.g., PDFs, files) if attachment := block.get("attachment"): @@ -74,11 +74,15 @@ class ArenaChannelExtractor(GalleryExtractor): if not url: continue - # Provide source link if it exists - if src := block.get("source"): - meta["source_url"] = src.get("url") or "" + block["date"] = self.parse_datetime_iso( + block["created_at"]) + block["date_updated"] = self.parse_datetime_iso( + block["updated_at"]) - yield url, meta + yield url, { + "block" : block, + "source": block.pop("source", None), + } if len(contents) < limit: return diff --git a/test/results/arena.py b/test/results/arena.py index 54066a47..3d1abacc 100644 --- a/test/results/arena.py +++ b/test/results/arena.py @@ -8,107 +8,6 @@ from gallery_dl.extractor import arena __tests__ = ( -{ - "#url" : "https://are.na/evan-collins-1522646491/cassette-futurism", - "#class" : arena.ArenaChannelExtractor, - "#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d", - "#count" : 160, - - "block_class" : "Image", - "block_title" : str, - "channel_id" : 1102343, - "channel_slug": "cassette-futurism", - "count" : 160, - "date" : "dt:2021-05-31 20:38:28", - "date_updated": "dt:2025-10-24 15:25:40", - "gallery_id" : "cassette-futurism", - "id" : int, - "num" : range(1, 160), - "title" : "Cassette Futurism", - "user" : { - "avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", - "badge" : "premium", - "base_class" : "User", - "can_index" : True, - "channel_count" : range(250, 300), - "class" : "User", - "created_at" : "2018-04-02T05:21:30.282Z", - "first_name" : "Evan", - "follower_count" : range(4900, 6000), - "following_count": range(10, 20), - "full_name" : "Evan Collins", - "id" : 51156, - "initials" : "EC", - "is_confirmed" : True, - "is_exceeding_connections_limit": False, - "is_lifetime_premium": False, - "is_pending_confirmation": False, - "is_pending_reconfirmation": False, - "is_premium" : True, - "is_supporter" : False, - "last_name" : "Collins", - "metadata" : {"description": None}, - "profile_id" : 171860, - "slug" : "evan-collins-1522646491", - "username" : "Evan Collins", - "avatar_image" : { - "display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", - "thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", - }, - }, -}, - -{ - "#url" : "https://are.na/lachie/transparent-tech-cobxde9pu40", - "#class" : arena.ArenaChannelExtractor, - "#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+(\.\w+)?\?\d+\?bc=\d", - "#count" : 89, - - "block_class" : str, - "block_title" : str, - "channel_id" : 2599871, - "channel_slug": "transparent-tech-cobxde9pu40", - "count" : 91, - "date" : "dt:2024-01-14 02:37:22", - "date_updated": "dt:2025-10-20 20:52:09", - "gallery_id" : "transparent-tech-cobxde9pu40", - "id" : int, - "num" : int, - "?source_url" : str, - "title" : "šŸ«™ Transparent Tech", - "user" : { - "avatar" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629", - "badge" : "premium", - "base_class" : "User", - "can_index" : True, - "channel_count" : 219, - "class" : "User", - "created_at" : "2018-05-03T07:13:39.847Z", - "first_name" : "lachie", - "follower_count" : range(80, 120), - "following_count": range(40, 80), - "full_name" : "lachie šŸ”", - "id" : 55241, - "initials" : "lšŸ”", - "is_confirmed" : True, - "is_exceeding_connections_limit": False, - "is_lifetime_premium": False, - "is_pending_confirmation": False, - "is_pending_reconfirmation": False, - "is_premium" : True, - "is_supporter" : False, - "last_name" : "šŸ”", - "metadata" : {"description": None}, - "profile_id" : 188402, - "slug" : "lachie", - "username" : "lachie šŸ”", - "avatar_image" : { - "display": "https://static.avatars.are.na/55241/medium_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629", - "thumb" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629", - }, - }, -}, - { "#url" : "https://www.are.na/mikf/touhou-zr5p8idnkag", "#class" : arena.ArenaChannelExtractor, @@ -120,17 +19,101 @@ __tests__ = ( "https://d2w9rnfcy7mm78.cloudfront.net/40873379/original_289824f61eade100785db100652abd9a.jpg?1762359483?bc=0", ), - "block_class" : str, - "block_title" : str, - "channel_id" : 4422732, - "channel_slug": "touhou-zr5p8idnkag", - "count" : 6, - "date" : "dt:2025-11-05 15:37:40", - "date_updated": "dt:2025-11-10 19:52:52", - "gallery_id" : "touhou-zr5p8idnkag", - "id" : int, - "title" : '''Touhou "ę±ę–¹"''', - "user" : { + "extension": {"jpg", "png", "mp4"}, + "filename" : str, + "count" : 6, + "num" : range(1, 5), + # "source" : {None, dict}, + "block" : { + # "attachment" : {None, dict}, + "base_class" : "Block", + "class" : {"Link", "Attachment", "Image"}, + "comment_count" : 0, + "connected_at" : "iso:datetime", + "connected_by_user_id": 1127493, + "connected_by_user_slug": "mikf", + "connected_by_username": "mikf .", + "connection_id" : int, + "content" : {None, ""}, + "content_html" : {None, ""}, + "created_at" : "iso:datetime", + "date" : "type:datetime", + "date_updated" : "type:datetime", + "description" : str, + "description_html": str, + "embed" : None, + "generated_title" : str, + "id" : int, + "position" : int, + "selected" : False, + "state" : "available", + "title" : str, + "updated_at" : "iso:datetime", + "visibility" : "public", + "image" : dict, + "user" : dict, + }, + "channel" : { + "added_to_at" : "2025-11-10T19:52:52.729Z", + "base_class" : "Channel", + "can_index" : False, + "class_name" : "Channel", + "collaboration" : False, + "collaborator_count": 0, + "collaborators" : [], + "created_at" : "2025-11-05T15:37:40.626Z", + "date" : "dt:2025-11-05 15:37:40", + "date_updated" : "type:datetime", + "follower_count": 0, + "id" : 4422732, + "kind" : "default", + "length" : 6, + "manifest" : dict, + "metadata" : {"description": ""}, + "nsfw?" : False, + "open" : False, + "page" : 1, + "per" : 20, + "published" : True, + "share_link" : None, + "slug" : "touhou-zr5p8idnkag", + "status" : "closed", + "title" : '''Touhou "ę±ę–¹"''', + "updated_at" : "iso:datetime", + "user_id" : 1127493, + }, + "owner" : { + "avatar" : "", + "badge" : None, + "base_class" : "User", + "can_index" : False, + "channel_count" : 3, + "class" : "User", + "created_at" : "2025-11-05T15:35:15.242Z", + "first_name" : "mikf", + "follower_count" : 0, + "following_count": 0, + "full_name" : "mikf .", + "id" : 1127493, + "initials" : "m.", + "is_confirmed" : True, + "is_exceeding_connections_limit": False, + "is_lifetime_premium": False, + "is_pending_confirmation": False, + "is_pending_reconfirmation": False, + "is_premium" : False, + "is_supporter" : False, + "last_name" : ".", + "metadata" : {"description": None}, + "profile_id" : 4422723, + "slug" : "mikf", + "username" : "mikf .", + "avatar_image" : { + "display": "", + "thumb" : "", + }, + }, + "user" : { "avatar" : "", "badge" : None, "base_class" : "User", @@ -163,4 +146,106 @@ __tests__ = ( }, }, +{ + "#url" : "https://are.na/evan-collins-1522646491/cassette-futurism", + "#class" : arena.ArenaChannelExtractor, + "#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d", + "#count" : 160, + + "extension": str, + "filename" : str, + "count" : 160, + "num" : range(1, 160), + "source" : None, + "block" : dict, + "channel" : { + "base_class" : "Channel", + "can_index" : True, + "class_name" : "Channel", + "collaboration" : False, + "collaborator_count": 0, + "collaborators" : [], + "created_at" : "2021-05-31T20:38:28.898Z", + "date" : "dt:2021-05-31 20:38:28", + "date_updated" : "type:datetime", + "follower_count": int, + "id" : 1102343, + "kind" : "default", + "length" : 160, + "manifest" : dict, + "metadata" : {"description": "The 70s-and-80s bulky, gray, angular scifi & hardware aesthetic. Eg. Syd Mead"}, + "nsfw?" : False, + "open" : False, + "published" : True, + "share_link" : None, + "slug" : "cassette-futurism", + "status" : "closed", + "title" : "Cassette Futurism", + "updated_at" : "iso:datetime", + "user_id" : 51156, + }, + "owner" : { + "avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + "badge" : "premium", + "base_class" : "User", + "can_index" : True, + "channel_count" : range(250, 300), + "class" : "User", + "created_at" : "2018-04-02T05:21:30.282Z", + "first_name" : "Evan", + "follower_count" : range(4900, 6000), + "following_count": range(10, 20), + "full_name" : "Evan Collins", + "id" : 51156, + "initials" : "EC", + "is_confirmed" : True, + "is_exceeding_connections_limit": False, + "is_lifetime_premium": False, + "is_pending_confirmation": False, + "is_pending_reconfirmation": False, + "is_premium" : True, + "is_supporter" : False, + "last_name" : "Collins", + "metadata" : {"description": None}, + "profile_id" : 171860, + "slug" : "evan-collins-1522646491", + "username" : "Evan Collins", + "avatar_image" : { + "display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + "thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + }, + }, + "user" : { + "avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + "badge" : "premium", + "base_class" : "User", + "can_index" : True, + "channel_count" : range(250, 300), + "class" : "User", + "created_at" : "2018-04-02T05:21:30.282Z", + "first_name" : "Evan", + "follower_count" : range(4900, 6000), + "following_count": range(10, 20), + "full_name" : "Evan Collins", + "id" : 51156, + "initials" : "EC", + "is_confirmed" : True, + "is_exceeding_connections_limit": False, + "is_lifetime_premium": False, + "is_pending_confirmation": False, + "is_pending_reconfirmation": False, + "is_premium" : True, + "is_supporter" : False, + "last_name" : "Collins", + "metadata" : {"description": None}, + "profile_id" : 171860, + "slug" : "evan-collins-1522646491", + "username" : "Evan Collins", + "avatar_image" : { + "display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + "thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757", + }, + }, +}, + )