[tiktok] extract subtitles and all cover types (#8805)
* Make sure that `img_id`, `audio_id` and `cover_id` fields are always available.
The values are set '' where they are not applicable.
Having `img_id` is necessary for the default `archive_fmt`, the other fields are handled for consistency.
* Allow downloading more than one cover.
The previous behavior is kept as-is, but setting the "covers" option to "all" now grabs all available covers.
* Add support for downloading subtitles
Allows filtering subtitles by source type (ASR, MT) and language.
* Ensure archive uniqueness for covers and subtitles.
* Update the URL test pattern to include the `image` extension.
Although Tiktok may serve the covers with jpeg content, the file ending can be `.image`.
The test before 0c14b164 failed because the asserted URL did not match all cover types, but the now used pattern needs the mentioned file ending.
* Add support for "creator_caption" subtitles in "LC" format.
These subtitles have the keys "Format" set to "creator_caption" and "Source" to "LC".
* Add "LC" (Local Captions) as a subtitle source type in the documentation
* Code deduplication and renaming subtitle metadata
Changed the item type from singular `subtitle` to `subtitles`.
Removed the wrong descriptor `cover` from the subtitles fallback title.
* Refactor subtitle filtering
The filter is now prepared in `_init` to prevent parsing the same config parameter for every item.
The `_extract_subtitles` function will still extract if either filter (source or language) matches.
* Generate a `file_id` for subtitles
Subtitles have multiple fields that determine the unique file, so these are simply concatenated.
This is similar to the cover types, only with more variations.
* Added tests for subtitles
* fix docs entries
* fix '"covers": "all"'
* simplify some code
* Fix fallback title for subtitles
Added the missing "f" to the f-string and added "subtitle" to the title.
The resulting title will look like "TikTok video subtitle #1234567"
This commit is contained in:
bassberry
@@ -36,10 +36,25 @@ class TiktokExtractor(Extractor):
|
||||
self.audio = self.config("audio", True)
|
||||
self.video = self.config("videos", True)
|
||||
self.cover = self.config("covers", False)
|
||||
self.subtitles = self.config("subtitles", False)
|
||||
|
||||
self.range = self.config("tiktok-range") or ""
|
||||
self.range_predicate = util.predicate_range_parse(self.range)
|
||||
|
||||
# If one of these fields is None, the filter for it is disabled.
|
||||
# Therefore, if both fields are none, all subtitles are extracted.
|
||||
self.subtitle_sources = None
|
||||
self.subtitle_langs = None
|
||||
|
||||
if self.subtitles and self.subtitles != "all":
|
||||
if self.subtitles is True or not isinstance(self.subtitles, str):
|
||||
self.subtitles = "ASR"
|
||||
|
||||
known_sources = {"ASR", "MT", "LC"}
|
||||
filters = set(self.subtitles.split(","))
|
||||
self.subtitle_sources = known_sources.intersection(filters) or None
|
||||
self.subtitle_langs = filters.difference(known_sources) or None
|
||||
|
||||
def items(self):
|
||||
for tiktok_url in self.posts():
|
||||
tiktok_url = self._sanitize_url(tiktok_url)
|
||||
@@ -73,13 +88,13 @@ class TiktokExtractor(Extractor):
|
||||
url = img["imageURL"]["urlList"][0]
|
||||
text.nameext_from_url(url, post)
|
||||
post.update({
|
||||
"type" : "image",
|
||||
"image" : img,
|
||||
"title" : title,
|
||||
"num" : i,
|
||||
"type" : "image",
|
||||
"image" : img,
|
||||
"title" : title,
|
||||
"num" : i,
|
||||
"file_id": post["filename"].partition("~")[0],
|
||||
"width" : img["imageWidth"],
|
||||
"height": img["imageHeight"],
|
||||
"width" : img["imageWidth"],
|
||||
"height" : img["imageHeight"],
|
||||
})
|
||||
yield Message.Url, url, post
|
||||
|
||||
@@ -95,9 +110,23 @@ class TiktokExtractor(Extractor):
|
||||
elif self.video and (url := self._extract_video(post)):
|
||||
yield Message.Url, url, post
|
||||
del post["_fallback"]
|
||||
if self.cover and (url := self._extract_cover(post, "video")):
|
||||
yield Message.Url, url, post
|
||||
|
||||
if self.cover:
|
||||
for url in self._extract_covers(post, "video"):
|
||||
yield Message.Url, url, post
|
||||
if self.cover != "all":
|
||||
break
|
||||
|
||||
if self.subtitles:
|
||||
for url in self._extract_subtitles(post, "video"):
|
||||
yield Message.Url, url, post
|
||||
|
||||
# remove the subtitle related fields for the next item
|
||||
post.pop("subtitle_lang_id", None)
|
||||
post.pop("subtitle_lang_codename", None)
|
||||
post.pop("subtitle_format", None)
|
||||
post.pop("subtitle_version", None)
|
||||
post.pop("subtitle_source", None)
|
||||
else:
|
||||
self.log.info("%s: Skipping post", tiktok_url)
|
||||
|
||||
@@ -277,7 +306,7 @@ class TiktokExtractor(Extractor):
|
||||
"title" : post["desc"] or f"TikTok video #{post['id']}",
|
||||
"duration" : video.get("duration"),
|
||||
"num" : 0,
|
||||
"file_id" : video.get("id"),
|
||||
"file_id" : "",
|
||||
"width" : video.get("width"),
|
||||
"height" : video.get("height"),
|
||||
})
|
||||
@@ -334,28 +363,85 @@ class TiktokExtractor(Extractor):
|
||||
post["extension"] = "mp3"
|
||||
return url
|
||||
|
||||
def _extract_cover(self, post, type):
|
||||
def _extract_covers(self, post, type):
|
||||
media = post[type]
|
||||
|
||||
for cover_id in ("thumbnail", "cover", "originCover", "dynamicCover"):
|
||||
if url := media.get(cover_id):
|
||||
break
|
||||
else:
|
||||
return
|
||||
text.nameext_from_url(url, post)
|
||||
post.update({
|
||||
"type" : "cover",
|
||||
"extension": "jpg",
|
||||
"image" : url,
|
||||
"title" : post["desc"] or
|
||||
f"TikTok {type} cover #{post['id']}",
|
||||
"duration" : media.get("duration"),
|
||||
"num" : 0,
|
||||
"file_id" : cover_id,
|
||||
"width" : 0,
|
||||
"height" : 0,
|
||||
})
|
||||
yield url
|
||||
|
||||
text.nameext_from_url(url, post)
|
||||
post.update({
|
||||
"type" : "cover",
|
||||
"extension": "jpg",
|
||||
"image" : url,
|
||||
"title" : post["desc"] or f"TikTok {type} cover #{post['id']}",
|
||||
"duration" : media.get("duration"),
|
||||
"num" : 0,
|
||||
"file_id" : cover_id,
|
||||
"width" : 0,
|
||||
"height" : 0,
|
||||
})
|
||||
return url
|
||||
def _extract_subtitles(self, post, type):
|
||||
media = post[type]
|
||||
sources_filtered = self.subtitle_sources is not None
|
||||
langs_filtered = self.subtitle_langs is not None
|
||||
|
||||
for subtitle in media.get("subtitleInfos", ()):
|
||||
sub_lang_id = subtitle.get("LanguageID")
|
||||
sub_lang_codename = subtitle.get("LanguageCodeName")
|
||||
sub_format = subtitle.get("Format")
|
||||
sub_version = subtitle.get("Version")
|
||||
sub_source = subtitle.get("Source")
|
||||
|
||||
# guard the iterable access
|
||||
sources_match = sources_filtered and \
|
||||
sub_source in self.subtitle_sources
|
||||
langs_match = langs_filtered and \
|
||||
sub_lang_codename in self.subtitle_langs
|
||||
|
||||
# Subtitles will be extracted when either filter matches.
|
||||
if not sources_match and not langs_match and \
|
||||
(sources_filtered or langs_filtered):
|
||||
continue
|
||||
|
||||
if url := subtitle.get("Url"):
|
||||
text.nameext_from_url(url, post)
|
||||
|
||||
# subtitle urls may not specify a filename,
|
||||
# so the metadata can be used to build one.
|
||||
if not post["filename"]:
|
||||
post["filename"] = (f"{post['id']}_{sub_lang_codename}_"
|
||||
f"{sub_version}_{sub_source}")
|
||||
post["extension"] = sub_format.lower()
|
||||
|
||||
# replace extensions for known formats
|
||||
if post["extension"] == "webvtt":
|
||||
post["extension"] = "vtt"
|
||||
elif post["extension"] == "creator_caption":
|
||||
post["extension"] = "json"
|
||||
|
||||
post.update({
|
||||
"type" : "subtitle",
|
||||
"image" : None,
|
||||
"title" :
|
||||
post["desc"] or
|
||||
f"TikTok {type} subtitle #{post['id']}",
|
||||
"duration" : media.get("duration"),
|
||||
"num" : 0,
|
||||
"file_id" :
|
||||
f"{sub_lang_id}_{sub_lang_codename}_{sub_source}_"
|
||||
f"{sub_version}_{sub_format}",
|
||||
"subtitle_lang_id" : sub_lang_id,
|
||||
"subtitle_lang_codename": sub_lang_codename,
|
||||
"subtitle_format" : sub_format,
|
||||
"subtitle_version" : sub_version,
|
||||
"subtitle_source" : sub_source,
|
||||
"width" : 0,
|
||||
"height" : 0,
|
||||
})
|
||||
yield url
|
||||
|
||||
def _check_status_code(self, detail, url, type_of_url):
|
||||
status = detail.get("statusCode")
|
||||
|
||||
Reference in New Issue
Block a user