github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[arena] update metadata structure

Browse Source 
- provide data in 'block', 'channel', 'user', etc dicts
- convert block dates to 'datetime' objects
- update directory/filename/archive formats
This commit is contained in:
Mike Fährmann
2025-11-11 21:01:02 +01:00
parent 793956d2d5
commit c07421cba6
2 changed files with 220 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
test/results/arena.py
View File

@@ -8,107 +8,6 @@ from gallery_dl.extractor import arena
__tests__ = (
{
"#url" : "https://are.na/evan-collins-1522646491/cassette-futurism",
"#class" : arena.ArenaChannelExtractor,
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d",
"#count" : 160,
"block_class" : "Image",
"block_title" : str,
"channel_id" : 1102343,
"channel_slug": "cassette-futurism",
"count" : 160,
"date" : "dt:2021-05-31 20:38:28",
"date_updated": "dt:2025-10-24 15:25:40",
"gallery_id" : "cassette-futurism",
"id" : int,
"num" : range(1, 160),
"title" : "Cassette Futurism",
"user" : {
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"badge" : "premium",
"base_class" : "User",
"can_index" : True,
"channel_count" : range(250, 300),
"class" : "User",
"created_at" : "2018-04-02T05:21:30.282Z",
"first_name" : "Evan",
"follower_count" : range(4900, 6000),
"following_count": range(10, 20),
"full_name" : "Evan Collins",
"id" : 51156,
"initials" : "EC",
"is_confirmed" : True,
"is_exceeding_connections_limit": False,
"is_lifetime_premium": False,
"is_pending_confirmation": False,
"is_pending_reconfirmation": False,
"is_premium" : True,
"is_supporter" : False,
"last_name" : "Collins",
"metadata" : {"description": None},
"profile_id" : 171860,
"slug" : "evan-collins-1522646491",
"username" : "Evan Collins",
"avatar_image" : {
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
},
},
},
{
"#url" : "https://are.na/lachie/transparent-tech-cobxde9pu40",
"#class" : arena.ArenaChannelExtractor,
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+(\.\w+)?\?\d+\?bc=\d",
"#count" : 89,
"block_class" : str,
"block_title" : str,
"channel_id" : 2599871,
"channel_slug": "transparent-tech-cobxde9pu40",
"count" : 91,
"date" : "dt:2024-01-14 02:37:22",
"date_updated": "dt:2025-10-20 20:52:09",
"gallery_id" : "transparent-tech-cobxde9pu40",
"id" : int,
"num" : int,
"?source_url" : str,
"title" : "🫙 Transparent Tech",
"user" : {
"avatar" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
"badge" : "premium",
"base_class" : "User",
"can_index" : True,
"channel_count" : 219,
"class" : "User",
"created_at" : "2018-05-03T07:13:39.847Z",
"first_name" : "lachie",
"follower_count" : range(80, 120),
"following_count": range(40, 80),
"full_name" : "lachie 🔐",
"id" : 55241,
"initials" : "l🔐",
"is_confirmed" : True,
"is_exceeding_connections_limit": False,
"is_lifetime_premium": False,
"is_pending_confirmation": False,
"is_pending_reconfirmation": False,
"is_premium" : True,
"is_supporter" : False,
"last_name" : "🔐",
"metadata" : {"description": None},
"profile_id" : 188402,
"slug" : "lachie",
"username" : "lachie 🔐",
"avatar_image" : {
"display": "https://static.avatars.are.na/55241/medium_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
"thumb" : "https://static.avatars.are.na/55241/small_fdcab74d56d0d9b93930333bdea50f4a.jpg?1738898629",
},
},
},
{
"#url" : "https://www.are.na/mikf/touhou-zr5p8idnkag",
"#class" : arena.ArenaChannelExtractor,
@@ -120,17 +19,101 @@ __tests__ = (
"https://d2w9rnfcy7mm78.cloudfront.net/40873379/original_289824f61eade100785db100652abd9a.jpg?1762359483?bc=0",
),
"block_class" : str,
"block_title" : str,
"channel_id" : 4422732,
"channel_slug": "touhou-zr5p8idnkag",
"count" : 6,
"date" : "dt:2025-11-05 15:37:40",
"date_updated": "dt:2025-11-10 19:52:52",
"gallery_id" : "touhou-zr5p8idnkag",
"id" : int,
"title" : '''Touhou "東方"''',
"user" : {
"extension": {"jpg", "png", "mp4"},
"filename" : str,
"count" : 6,
"num" : range(1, 5),
# "source" : {None, dict},
"block" : {
# "attachment" : {None, dict},
"base_class" : "Block",
"class" : {"Link", "Attachment", "Image"},
"comment_count" : 0,
"connected_at" : "iso:datetime",
"connected_by_user_id": 1127493,
"connected_by_user_slug": "mikf",
"connected_by_username": "mikf .",
"connection_id" : int,
"content" : {None, ""},
"content_html" : {None, ""},
"created_at" : "iso:datetime",
"date" : "type:datetime",
"date_updated" : "type:datetime",
"description" : str,
"description_html": str,
"embed" : None,
"generated_title" : str,
"id" : int,
"position" : int,
"selected" : False,
"state" : "available",
"title" : str,
"updated_at" : "iso:datetime",
"visibility" : "public",
"image" : dict,
"user" : dict,
},
"channel" : {
"added_to_at" : "2025-11-10T19:52:52.729Z",
"base_class" : "Channel",
"can_index" : False,
"class_name" : "Channel",
"collaboration" : False,
"collaborator_count": 0,
"collaborators" : [],
"created_at" : "2025-11-05T15:37:40.626Z",
"date" : "dt:2025-11-05 15:37:40",
"date_updated" : "type:datetime",
"follower_count": 0,
"id" : 4422732,
"kind" : "default",
"length" : 6,
"manifest" : dict,
"metadata" : {"description": ""},
"nsfw?" : False,
"open" : False,
"page" : 1,
"per" : 20,
"published" : True,
"share_link" : None,
"slug" : "touhou-zr5p8idnkag",
"status" : "closed",
"title" : '''Touhou "東方"''',
"updated_at" : "iso:datetime",
"user_id" : 1127493,
},
"owner" : {
"avatar" : "",
"badge" : None,
"base_class" : "User",
"can_index" : False,
"channel_count" : 3,
"class" : "User",
"created_at" : "2025-11-05T15:35:15.242Z",
"first_name" : "mikf",
"follower_count" : 0,
"following_count": 0,
"full_name" : "mikf .",
"id" : 1127493,
"initials" : "m.",
"is_confirmed" : True,
"is_exceeding_connections_limit": False,
"is_lifetime_premium": False,
"is_pending_confirmation": False,
"is_pending_reconfirmation": False,
"is_premium" : False,
"is_supporter" : False,
"last_name" : ".",
"metadata" : {"description": None},
"profile_id" : 4422723,
"slug" : "mikf",
"username" : "mikf .",
"avatar_image" : {
"display": "",
"thumb" : "",
},
},
"user" : {
"avatar" : "",
"badge" : None,
"base_class" : "User",
@@ -163,4 +146,106 @@ __tests__ = (
},
},
{
"#url" : "https://are.na/evan-collins-1522646491/cassette-futurism",
"#class" : arena.ArenaChannelExtractor,
"#pattern" : r"https://d2w9rnfcy7mm78\.cloudfront\.net/\d+/original_\w+\.\w+\?\d+\?bc=\d",
"#count" : 160,
"extension": str,
"filename" : str,
"count" : 160,
"num" : range(1, 160),
"source" : None,
"block" : dict,
"channel" : {
"base_class" : "Channel",
"can_index" : True,
"class_name" : "Channel",
"collaboration" : False,
"collaborator_count": 0,
"collaborators" : [],
"created_at" : "2021-05-31T20:38:28.898Z",
"date" : "dt:2021-05-31 20:38:28",
"date_updated" : "type:datetime",
"follower_count": int,
"id" : 1102343,
"kind" : "default",
"length" : 160,
"manifest" : dict,
"metadata" : {"description": "The 70s-and-80s bulky, gray, angular scifi & hardware aesthetic. Eg. Syd Mead"},
"nsfw?" : False,
"open" : False,
"published" : True,
"share_link" : None,
"slug" : "cassette-futurism",
"status" : "closed",
"title" : "Cassette Futurism",
"updated_at" : "iso:datetime",
"user_id" : 51156,
},
"owner" : {
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"badge" : "premium",
"base_class" : "User",
"can_index" : True,
"channel_count" : range(250, 300),
"class" : "User",
"created_at" : "2018-04-02T05:21:30.282Z",
"first_name" : "Evan",
"follower_count" : range(4900, 6000),
"following_count": range(10, 20),
"full_name" : "Evan Collins",
"id" : 51156,
"initials" : "EC",
"is_confirmed" : True,
"is_exceeding_connections_limit": False,
"is_lifetime_premium": False,
"is_pending_confirmation": False,
"is_pending_reconfirmation": False,
"is_premium" : True,
"is_supporter" : False,
"last_name" : "Collins",
"metadata" : {"description": None},
"profile_id" : 171860,
"slug" : "evan-collins-1522646491",
"username" : "Evan Collins",
"avatar_image" : {
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
},
},
"user" : {
"avatar" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"badge" : "premium",
"base_class" : "User",
"can_index" : True,
"channel_count" : range(250, 300),
"class" : "User",
"created_at" : "2018-04-02T05:21:30.282Z",
"first_name" : "Evan",
"follower_count" : range(4900, 6000),
"following_count": range(10, 20),
"full_name" : "Evan Collins",
"id" : 51156,
"initials" : "EC",
"is_confirmed" : True,
"is_exceeding_connections_limit": False,
"is_lifetime_premium": False,
"is_pending_confirmation": False,
"is_pending_reconfirmation": False,
"is_premium" : True,
"is_supporter" : False,
"last_name" : "Collins",
"metadata" : {"description": None},
"profile_id" : 171860,
"slug" : "evan-collins-1522646491",
"username" : "Evan Collins",
"avatar_image" : {
"display": "https://static.avatars.are.na/51156/medium_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
"thumb" : "https://static.avatars.are.na/51156/small_8c6098f64217eca6b4bcff44a7abf2d7.jpg?1563035757",
},
},
},
)