[arena] update metadata structure
- provide data in 'block', 'channel', 'user', etc dicts - convert block dates to 'datetime' objects - update directory/filename/archive formats
This commit is contained in:
Mike Fährmann
@@ -16,22 +16,28 @@ class ArenaChannelExtractor(GalleryExtractor):
|
|||||||
category = "arena"
|
category = "arena"
|
||||||
subcategory = "channel"
|
subcategory = "channel"
|
||||||
root = "https://are.na"
|
root = "https://are.na"
|
||||||
|
directory_fmt = ("{category}", "{user[full_name]} ({user[id]})",
|
||||||
|
"{channel[title]} ({channel[id]})")
|
||||||
|
filename_fmt = "{num:>03}{block[id]:? //}.{extension}"
|
||||||
|
archive_fmt = "{channel[id]}/{block[id]}"
|
||||||
pattern = r"(?:https?://)?(?:www\.)?are\.na/[^/?#]+/([^/?#]+)"
|
pattern = r"(?:https?://)?(?:www\.)?are\.na/[^/?#]+/([^/?#]+)"
|
||||||
example = "https://are.na/evan-collins-1522646491/cassette-futurism"
|
example = "https://are.na/evan-collins-1522646491/cassette-futurism"
|
||||||
|
|
||||||
def metadata(self, page):
|
def metadata(self, page):
|
||||||
info = self.request_json(
|
channel = self.request_json(
|
||||||
f"https://api.are.na/v2/channels/{self.groups[0]}")
|
f"https://api.are.na/v2/channels/{self.groups[0]}")
|
||||||
|
|
||||||
|
channel["date"] = self.parse_datetime_iso(
|
||||||
|
channel["created_at"])
|
||||||
|
channel["date_updated"] = self.parse_datetime_iso(
|
||||||
|
channel["updated_at"])
|
||||||
|
channel.pop("contents", None)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"gallery_id" : info.get("slug") or str(info.get("id")),
|
"count" : channel.get("length"),
|
||||||
"channel_id" : info.get("id"),
|
"user" : channel.pop("user", None),
|
||||||
"channel_slug": info.get("slug"),
|
"owner" : channel.pop("owner", None),
|
||||||
"title" : info.get("title") or "",
|
"channel": channel,
|
||||||
"count" : info.get("length") or 0,
|
|
||||||
"user" : info.get("user"),
|
|
||||||
"date" : self.parse_datetime_iso(info.get("created_at")),
|
|
||||||
"date_updated": self.parse_datetime_iso(info.get("updated_at")),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def images(self, page):
|
def images(self, page):
|
||||||
@@ -48,12 +54,6 @@ class ArenaChannelExtractor(GalleryExtractor):
|
|||||||
|
|
||||||
for block in contents:
|
for block in contents:
|
||||||
url = None
|
url = None
|
||||||
meta = {
|
|
||||||
"id": block.get("id"),
|
|
||||||
"block_class": block.get("class"),
|
|
||||||
"block_title": block.get("title") or block.get(
|
|
||||||
"generated_title") or "",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Attachments (e.g., PDFs, files)
|
# Attachments (e.g., PDFs, files)
|
||||||
if attachment := block.get("attachment"):
|
if attachment := block.get("attachment"):
|
||||||
@@ -74,11 +74,15 @@ class ArenaChannelExtractor(GalleryExtractor):
|
|||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Provide source link if it exists
|
block["date"] = self.parse_datetime_iso(
|
||||||
if src := block.get("source"):
|
block["created_at"])
|
||||||
meta["source_url"] = src.get("url") or ""
|
block["date_updated"] = self.parse_datetime_iso(
|
||||||
|
block["updated_at"])
|
||||||
|
|
||||||
yield url, meta
|
yield url, {
|
||||||
|
"block" : block,
|
||||||
|
"source": block.pop("source", None),
|
||||||
|
}
|
||||||
|
|
||||||
if len(contents) < limit:
|
if len(contents) < limit:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -8,107 +8,6 @@ from gallery_dl.extractor import arena
|
|||||||
|
|
||||||
|
|
||||||
__tests__ = (
|
__tests__ = (
|
||||||
{
|
|
||||||
"#url" : "https://are.na/evan-collins-1522646491/cassette-futurism",
|
|
||||||
"#class" : arena.ArenaChannelExtractor,
|
|
||||||
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d",
|
|
||||||
"#count" : 160,
|
|
||||||
|
|
||||||
"block_class" : "Image",
|
|
||||||
"block_title" : str,
|
|
||||||
"channel_id" : 1102343,
|
|
||||||
"channel_slug": "cassette-futurism",
|
|
||||||
"count" : 160,
|
|
||||||
"date" : "dt:2021-05-31 20:38:28",
|
|
||||||
"date_updated": "dt:2025-10-24 15:25:40",
|
|
||||||
"gallery_id" : "cassette-futurism",
|
|
||||||
"id" : int,
|
|
||||||
"num" : range(1, 160),
|
|
||||||
"title" : "Cassette Futurism",
|
|
||||||
"user" : {
|
|
||||||
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
|
||||||
"badge" : "premium",
|
|
||||||
"base_class" : "User",
|
|
||||||
"can_index" : True,
|
|
||||||
"channel_count" : range(250, 300),
|
|
||||||
"class" : "User",
|
|
||||||
"created_at" : "2018-04-02T05:21:30.282Z",
|
|
||||||
"first_name" : "Evan",
|
|
||||||
"follower_count" : range(4900, 6000),
|
|
||||||
"following_count": range(10, 20),
|
|
||||||
"full_name" : "Evan Collins",
|
|
||||||
"id" : 51156,
|
|
||||||
"initials" : "EC",
|
|
||||||
"is_confirmed" : True,
|
|
||||||
"is_exceeding_connections_limit": False,
|
|
||||||
"is_lifetime_premium": False,
|
|
||||||
"is_pending_confirmation": False,
|
|
||||||
"is_pending_reconfirmation": False,
|
|
||||||
"is_premium" : True,
|
|
||||||
"is_supporter" : False,
|
|
||||||
"last_name" : "Collins",
|
|
||||||
"metadata" : {"description": None},
|
|
||||||
"profile_id" : 171860,
|
|
||||||
"slug" : "evan-collins-1522646491",
|
|
||||||
"username" : "Evan Collins",
|
|
||||||
"avatar_image" : {
|
|
||||||
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
|
||||||
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"#url" : "https://are.na/lachie/transparent-tech-cobxde9pu40",
|
|
||||||
"#class" : arena.ArenaChannelExtractor,
|
|
||||||
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+(\.\w+)?\?\d+\?bc=\d",
|
|
||||||
"#count" : 89,
|
|
||||||
|
|
||||||
"block_class" : str,
|
|
||||||
"block_title" : str,
|
|
||||||
"channel_id" : 2599871,
|
|
||||||
"channel_slug": "transparent-tech-cobxde9pu40",
|
|
||||||
"count" : 91,
|
|
||||||
"date" : "dt:2024-01-14 02:37:22",
|
|
||||||
"date_updated": "dt:2025-10-20 20:52:09",
|
|
||||||
"gallery_id" : "transparent-tech-cobxde9pu40",
|
|
||||||
"id" : int,
|
|
||||||
"num" : int,
|
|
||||||
"?source_url" : str,
|
|
||||||
"title" : "🫙 Transparent Tech",
|
|
||||||
"user" : {
|
|
||||||
"avatar" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
|
|
||||||
"badge" : "premium",
|
|
||||||
"base_class" : "User",
|
|
||||||
"can_index" : True,
|
|
||||||
"channel_count" : 219,
|
|
||||||
"class" : "User",
|
|
||||||
"created_at" : "2018-05-03T07:13:39.847Z",
|
|
||||||
"first_name" : "lachie",
|
|
||||||
"follower_count" : range(80, 120),
|
|
||||||
"following_count": range(40, 80),
|
|
||||||
"full_name" : "lachie 🔐",
|
|
||||||
"id" : 55241,
|
|
||||||
"initials" : "l🔐",
|
|
||||||
"is_confirmed" : True,
|
|
||||||
"is_exceeding_connections_limit": False,
|
|
||||||
"is_lifetime_premium": False,
|
|
||||||
"is_pending_confirmation": False,
|
|
||||||
"is_pending_reconfirmation": False,
|
|
||||||
"is_premium" : True,
|
|
||||||
"is_supporter" : False,
|
|
||||||
"last_name" : "🔐",
|
|
||||||
"metadata" : {"description": None},
|
|
||||||
"profile_id" : 188402,
|
|
||||||
"slug" : "lachie",
|
|
||||||
"username" : "lachie 🔐",
|
|
||||||
"avatar_image" : {
|
|
||||||
"display": "https://static.avatars.are.na/55241/medium_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
|
|
||||||
"thumb" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
{
|
||||||
"#url" : "https://www.are.na/mikf/touhou-zr5p8idnkag",
|
"#url" : "https://www.are.na/mikf/touhou-zr5p8idnkag",
|
||||||
"#class" : arena.ArenaChannelExtractor,
|
"#class" : arena.ArenaChannelExtractor,
|
||||||
@@ -120,17 +19,101 @@ __tests__ = (
|
|||||||
"https://d2w9rnfcy7mm78.cloudfront.net/40873379/original_289824f61eade100785db100652abd9a.jpg?1762359483?bc=0",
|
"https://d2w9rnfcy7mm78.cloudfront.net/40873379/original_289824f61eade100785db100652abd9a.jpg?1762359483?bc=0",
|
||||||
),
|
),
|
||||||
|
|
||||||
"block_class" : str,
|
"extension": {"jpg", "png", "mp4"},
|
||||||
"block_title" : str,
|
"filename" : str,
|
||||||
"channel_id" : 4422732,
|
"count" : 6,
|
||||||
"channel_slug": "touhou-zr5p8idnkag",
|
"num" : range(1, 5),
|
||||||
"count" : 6,
|
# "source" : {None, dict},
|
||||||
"date" : "dt:2025-11-05 15:37:40",
|
"block" : {
|
||||||
"date_updated": "dt:2025-11-10 19:52:52",
|
# "attachment" : {None, dict},
|
||||||
"gallery_id" : "touhou-zr5p8idnkag",
|
"base_class" : "Block",
|
||||||
"id" : int,
|
"class" : {"Link", "Attachment", "Image"},
|
||||||
"title" : '''Touhou "東方"''',
|
"comment_count" : 0,
|
||||||
"user" : {
|
"connected_at" : "iso:datetime",
|
||||||
|
"connected_by_user_id": 1127493,
|
||||||
|
"connected_by_user_slug": "mikf",
|
||||||
|
"connected_by_username": "mikf .",
|
||||||
|
"connection_id" : int,
|
||||||
|
"content" : {None, ""},
|
||||||
|
"content_html" : {None, ""},
|
||||||
|
"created_at" : "iso:datetime",
|
||||||
|
"date" : "type:datetime",
|
||||||
|
"date_updated" : "type:datetime",
|
||||||
|
"description" : str,
|
||||||
|
"description_html": str,
|
||||||
|
"embed" : None,
|
||||||
|
"generated_title" : str,
|
||||||
|
"id" : int,
|
||||||
|
"position" : int,
|
||||||
|
"selected" : False,
|
||||||
|
"state" : "available",
|
||||||
|
"title" : str,
|
||||||
|
"updated_at" : "iso:datetime",
|
||||||
|
"visibility" : "public",
|
||||||
|
"image" : dict,
|
||||||
|
"user" : dict,
|
||||||
|
},
|
||||||
|
"channel" : {
|
||||||
|
"added_to_at" : "2025-11-10T19:52:52.729Z",
|
||||||
|
"base_class" : "Channel",
|
||||||
|
"can_index" : False,
|
||||||
|
"class_name" : "Channel",
|
||||||
|
"collaboration" : False,
|
||||||
|
"collaborator_count": 0,
|
||||||
|
"collaborators" : [],
|
||||||
|
"created_at" : "2025-11-05T15:37:40.626Z",
|
||||||
|
"date" : "dt:2025-11-05 15:37:40",
|
||||||
|
"date_updated" : "type:datetime",
|
||||||
|
"follower_count": 0,
|
||||||
|
"id" : 4422732,
|
||||||
|
"kind" : "default",
|
||||||
|
"length" : 6,
|
||||||
|
"manifest" : dict,
|
||||||
|
"metadata" : {"description": ""},
|
||||||
|
"nsfw?" : False,
|
||||||
|
"open" : False,
|
||||||
|
"page" : 1,
|
||||||
|
"per" : 20,
|
||||||
|
"published" : True,
|
||||||
|
"share_link" : None,
|
||||||
|
"slug" : "touhou-zr5p8idnkag",
|
||||||
|
"status" : "closed",
|
||||||
|
"title" : '''Touhou "東方"''',
|
||||||
|
"updated_at" : "iso:datetime",
|
||||||
|
"user_id" : 1127493,
|
||||||
|
},
|
||||||
|
"owner" : {
|
||||||
|
"avatar" : "",
|
||||||
|
"badge" : None,
|
||||||
|
"base_class" : "User",
|
||||||
|
"can_index" : False,
|
||||||
|
"channel_count" : 3,
|
||||||
|
"class" : "User",
|
||||||
|
"created_at" : "2025-11-05T15:35:15.242Z",
|
||||||
|
"first_name" : "mikf",
|
||||||
|
"follower_count" : 0,
|
||||||
|
"following_count": 0,
|
||||||
|
"full_name" : "mikf .",
|
||||||
|
"id" : 1127493,
|
||||||
|
"initials" : "m.",
|
||||||
|
"is_confirmed" : True,
|
||||||
|
"is_exceeding_connections_limit": False,
|
||||||
|
"is_lifetime_premium": False,
|
||||||
|
"is_pending_confirmation": False,
|
||||||
|
"is_pending_reconfirmation": False,
|
||||||
|
"is_premium" : False,
|
||||||
|
"is_supporter" : False,
|
||||||
|
"last_name" : ".",
|
||||||
|
"metadata" : {"description": None},
|
||||||
|
"profile_id" : 4422723,
|
||||||
|
"slug" : "mikf",
|
||||||
|
"username" : "mikf .",
|
||||||
|
"avatar_image" : {
|
||||||
|
"display": "",
|
||||||
|
"thumb" : "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"user" : {
|
||||||
"avatar" : "",
|
"avatar" : "",
|
||||||
"badge" : None,
|
"badge" : None,
|
||||||
"base_class" : "User",
|
"base_class" : "User",
|
||||||
@@ -163,4 +146,106 @@ __tests__ = (
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"#url" : "https://are.na/evan-collins-1522646491/cassette-futurism",
|
||||||
|
"#class" : arena.ArenaChannelExtractor,
|
||||||
|
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d",
|
||||||
|
"#count" : 160,
|
||||||
|
|
||||||
|
"extension": str,
|
||||||
|
"filename" : str,
|
||||||
|
"count" : 160,
|
||||||
|
"num" : range(1, 160),
|
||||||
|
"source" : None,
|
||||||
|
"block" : dict,
|
||||||
|
"channel" : {
|
||||||
|
"base_class" : "Channel",
|
||||||
|
"can_index" : True,
|
||||||
|
"class_name" : "Channel",
|
||||||
|
"collaboration" : False,
|
||||||
|
"collaborator_count": 0,
|
||||||
|
"collaborators" : [],
|
||||||
|
"created_at" : "2021-05-31T20:38:28.898Z",
|
||||||
|
"date" : "dt:2021-05-31 20:38:28",
|
||||||
|
"date_updated" : "type:datetime",
|
||||||
|
"follower_count": int,
|
||||||
|
"id" : 1102343,
|
||||||
|
"kind" : "default",
|
||||||
|
"length" : 160,
|
||||||
|
"manifest" : dict,
|
||||||
|
"metadata" : {"description": "The 70s-and-80s bulky, gray, angular scifi & hardware aesthetic. Eg. Syd Mead"},
|
||||||
|
"nsfw?" : False,
|
||||||
|
"open" : False,
|
||||||
|
"published" : True,
|
||||||
|
"share_link" : None,
|
||||||
|
"slug" : "cassette-futurism",
|
||||||
|
"status" : "closed",
|
||||||
|
"title" : "Cassette Futurism",
|
||||||
|
"updated_at" : "iso:datetime",
|
||||||
|
"user_id" : 51156,
|
||||||
|
},
|
||||||
|
"owner" : {
|
||||||
|
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
"badge" : "premium",
|
||||||
|
"base_class" : "User",
|
||||||
|
"can_index" : True,
|
||||||
|
"channel_count" : range(250, 300),
|
||||||
|
"class" : "User",
|
||||||
|
"created_at" : "2018-04-02T05:21:30.282Z",
|
||||||
|
"first_name" : "Evan",
|
||||||
|
"follower_count" : range(4900, 6000),
|
||||||
|
"following_count": range(10, 20),
|
||||||
|
"full_name" : "Evan Collins",
|
||||||
|
"id" : 51156,
|
||||||
|
"initials" : "EC",
|
||||||
|
"is_confirmed" : True,
|
||||||
|
"is_exceeding_connections_limit": False,
|
||||||
|
"is_lifetime_premium": False,
|
||||||
|
"is_pending_confirmation": False,
|
||||||
|
"is_pending_reconfirmation": False,
|
||||||
|
"is_premium" : True,
|
||||||
|
"is_supporter" : False,
|
||||||
|
"last_name" : "Collins",
|
||||||
|
"metadata" : {"description": None},
|
||||||
|
"profile_id" : 171860,
|
||||||
|
"slug" : "evan-collins-1522646491",
|
||||||
|
"username" : "Evan Collins",
|
||||||
|
"avatar_image" : {
|
||||||
|
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"user" : {
|
||||||
|
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
"badge" : "premium",
|
||||||
|
"base_class" : "User",
|
||||||
|
"can_index" : True,
|
||||||
|
"channel_count" : range(250, 300),
|
||||||
|
"class" : "User",
|
||||||
|
"created_at" : "2018-04-02T05:21:30.282Z",
|
||||||
|
"first_name" : "Evan",
|
||||||
|
"follower_count" : range(4900, 6000),
|
||||||
|
"following_count": range(10, 20),
|
||||||
|
"full_name" : "Evan Collins",
|
||||||
|
"id" : 51156,
|
||||||
|
"initials" : "EC",
|
||||||
|
"is_confirmed" : True,
|
||||||
|
"is_exceeding_connections_limit": False,
|
||||||
|
"is_lifetime_premium": False,
|
||||||
|
"is_pending_confirmation": False,
|
||||||
|
"is_pending_reconfirmation": False,
|
||||||
|
"is_premium" : True,
|
||||||
|
"is_supporter" : False,
|
||||||
|
"last_name" : "Collins",
|
||||||
|
"metadata" : {"description": None},
|
||||||
|
"profile_id" : 171860,
|
||||||
|
"slug" : "evan-collins-1522646491",
|
||||||
|
"username" : "Evan Collins",
|
||||||
|
"avatar_image" : {
|
||||||
|
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user