feat: channel browsing and code overhaul
a lot of code responsible for parsing data into invidious-compatible structures has been moved to the ythdd_struct_parser file
This commit is contained in:
317
ythdd_struct_parser.py
Normal file
317
ythdd_struct_parser.py
Normal file
@@ -0,0 +1,317 @@
|
||||
from ythdd_globals import safeTraverse
|
||||
from html import escape
|
||||
import json
|
||||
import dateparser
|
||||
import ythdd_globals
|
||||
import ythdd_extractor
|
||||
|
||||
def genThumbs(videoId: str):
|
||||
|
||||
result = []
|
||||
thumbnails = [
|
||||
#{'height': 720, 'width': 1280, 'quality': "maxres", 'url': "maxres"}, # for the time being omit the buggy maxres quality
|
||||
{'height': 720, 'width': 1280, 'quality': "maxresdefault", 'url': "maxresdefault"},
|
||||
{'height': 480, 'width': 640, 'quality': "sddefault", 'url': "sddefault"},
|
||||
{'height': 360, 'width': 480, 'quality': "high", 'url': "hqdefault"},
|
||||
{'height': 180, 'width': 320, 'quality': "medium", 'url': "mqdefault"},
|
||||
{'height': 90, 'width': 120, 'quality': "default", 'url': "default"},
|
||||
{'height': 90, 'width': 120, 'quality': "start", 'url': "1"},
|
||||
{'height': 90, 'width': 120, 'quality': "middle", 'url': "2"},
|
||||
{'height': 90, 'width': 120, 'quality': "end", 'url': "3"},
|
||||
]
|
||||
|
||||
for x in thumbnails:
|
||||
width = x['width']
|
||||
height = x['height']
|
||||
quality = x['quality']
|
||||
url = ythdd_globals.config['general']['public_facing_url'] + 'vi/' + videoId + '/' + x['url'] + '.jpg'
|
||||
result.append({'quality': quality, 'url': url, 'width': width, 'height': height})
|
||||
|
||||
return result
|
||||
|
||||
def doesContainNumber(string: str, numeric_system: int = 10) -> bool:
|
||||
try:
|
||||
number = int(string, numeric_system)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
raise BaseException("doesContainNumber(): Unknown error while determining if a string contains a number")
|
||||
|
||||
def parseLengthFromTimeBadge(time_str: str) -> int:
|
||||
# Returns 0 if unsuccessful
|
||||
length = 0
|
||||
time_lookup_list = [1, 60, 3_600, 86_400]
|
||||
time_list = time_str.split(":")
|
||||
if False in map(doesContainNumber, time_list): # works around ['LIVE'] for livestreams or ['Upcoming'] for scheduled videos
|
||||
pass
|
||||
else:
|
||||
for z in range(len(time_list)):
|
||||
length += time_lookup_list[z] * int(time_list[len(time_list) - 1 - z])
|
||||
return length
|
||||
|
||||
def parseViewsFromViewText(viewcounttext: str) -> int:
|
||||
# Returns 0 if unsuccessful
|
||||
views = 0
|
||||
magnitude = {'K': 1_000, 'M': 1_000_000, 'B': 1_000_000_000}
|
||||
if viewcounttext:
|
||||
if viewcounttext.lower() == "no":
|
||||
viewcounttext = "0"
|
||||
views = float("0" + "".join([z for z in viewcounttext if 48 <= ord(z) and ord(z) <= 57 or ord(z) == 46]))
|
||||
viewcounttext = viewcounttext.split(" ")[0]
|
||||
for x in magnitude.keys():
|
||||
if x == viewcounttext[-1].upper():
|
||||
views *= magnitude[x]
|
||||
return int(views)
|
||||
|
||||
def parseRenderers(entry: dict, context: dict = {}) -> dict:
|
||||
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError("parsed entry is not of type dict")
|
||||
|
||||
match safeTraverse(list(entry.keys()), [0], default=""):
|
||||
|
||||
case "videoRenderer": # represents a video
|
||||
|
||||
published_date = safeTraverse(entry, ["videoRenderer", "publishedTimeText", "simpleText"], default="now")
|
||||
published_date = published_date.removeprefix("Streamed ")
|
||||
description, description_html = parseDescriptionSnippet(safeTraverse(entry, ["videoRenderer", "descriptionSnippet", "runs"], default=[]))
|
||||
|
||||
if "author_name" in context:
|
||||
author_name = context["author_name"]
|
||||
else:
|
||||
author_name = safeTraverse(entry, ["videoRenderer", "ownerText", "runs", 0, "text"], default="Unknown author")
|
||||
|
||||
if "author_ucid" in context:
|
||||
author_ucid = context["author_ucid"]
|
||||
else:
|
||||
author_ucid = safeTraverse(entry, ["videoRenderer", "ownerText", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId"], default="UNKNOWNCHANNELID")
|
||||
|
||||
if "verified" in context:
|
||||
verified = context["verified"]
|
||||
else:
|
||||
verified = ythdd_extractor.isVerified(safeTraverse(entry, ["ownerBadges", 0]))
|
||||
|
||||
if "avatar" in context:
|
||||
avatar_url = context["avatar"]
|
||||
else:
|
||||
avatar_url = safeTraverse(entry, ["videoRenderer", "avatar", "decoratedAvatarViewModel", "avatar", "avatarViewModel", "image", "sources", 0, "url"], default="unknown")
|
||||
|
||||
return {
|
||||
"type": "video",
|
||||
"title": safeTraverse(entry, ["videoRenderer", "title", "runs", 0, "text"]),
|
||||
"videoId": safeTraverse(entry, ["videoRenderer", "videoId"]),
|
||||
"author": author_name,
|
||||
"authorId": author_ucid,
|
||||
"authorUrl": "/channel/" + author_ucid,
|
||||
"authorVerified": verified, # TODO
|
||||
"authorThumbnails": ythdd_extractor.generateChannelAvatarsFromUrl(avatar_url),
|
||||
"videoThumbnails": genThumbs(safeTraverse(entry, ["videoRenderer", "videoId"], default="unknown")),
|
||||
"description": description,
|
||||
"descriptionHtml": description_html,
|
||||
"viewCount": parseViewsFromViewText(safeTraverse(entry, ["videoRenderer", "viewCountText", "simpleText"], default="No views")),
|
||||
"viewCountText": safeTraverse(entry, ["videoRenderer", "viewCountText", "simpleText"], default="Unknown amount of views"),
|
||||
"published": int(dateparser.parse(published_date).timestamp()), # sadly best we can do, invidious does this too
|
||||
"publishedText": published_date,
|
||||
"lengthSeconds": parseLengthFromTimeBadge(safeTraverse(entry, ["videoRenderer", "lengthText", "simpleText"], default="0:0")),
|
||||
"liveNow": False,
|
||||
"premium": ythdd_extractor.isPremium(safeTraverse(entry, ["videoRenderer", "badges", 0])), # will fail if it's not the only badge
|
||||
"isUpcoming": False,
|
||||
"isNew": False,
|
||||
"is4k": False,
|
||||
"is8k": False,
|
||||
"isVr180": False,
|
||||
"isVr360": False,
|
||||
"is3d": False,
|
||||
"hasCaptions": False
|
||||
}
|
||||
|
||||
# modify the premiere timestamp afterwards here?
|
||||
|
||||
case "lockupViewModel": # represents playlists/mixes
|
||||
|
||||
playlist_type = safeTraverse(entry, ["lockupViewModel", "contentImage", "collectionThumbnailViewModel", "primaryThumbnail", "thumbnailViewModel", "overlays", 0, "thumbnailOverlayBadgeViewModel", "thumbnailBadges", 0, "thumbnailBadgeViewModel", "icon", "sources", 0, "clientResource", "imageName"], default="PLAYLISTS")
|
||||
|
||||
if playlist_type == "MIX":
|
||||
# mixes aren't currently supported
|
||||
return
|
||||
|
||||
lvm = entry["lockupViewModel"]
|
||||
meta = safeTraverse(lvm, ["metadata"], default=[])
|
||||
lmvm = safeTraverse(meta, ["lockupMetadataViewModel", "metadata", "contentMetadataViewModel", "metadataRows"], default=[])
|
||||
thumbnail = ythdd_globals.translateLinks(safeTraverse(lvm, ["contentImage", "collectionThumbnailViewModel", "primaryThumbnail", "thumbnailViewModel", "image", "sources", -1, "url"], default="no-url?"))
|
||||
verified = safeTraverse(context, ["verified"], default=False)
|
||||
|
||||
playlist_id = safeTraverse(lvm, ["contentId"], default="UNKNOWNPLAYLISTID")
|
||||
length = safeTraverse(lvm, ["contentImage", "collectionThumbnailViewModel", "primaryThumbnail", "thumbnailViewModel", "overlays", 0, "thumbnailOverlayBadgeViewModel", "thumbnailBadges", 0, "thumbnailBadgeViewModel", "text"], default="0 videos")
|
||||
length = parseViewsFromViewText(length.split(" ")[0])
|
||||
|
||||
# Turns out for some responses we do some data, while not on others.
|
||||
# Data from context should be prioritized, thus even if something is found with safeTraverse,
|
||||
# the parser will ignore it in favour of the context.
|
||||
ucid = safeTraverse(lmvm, [0, "metadataParts", 0, "text", "commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId"], default="UNKNOWNCHANNELID")
|
||||
author = safeTraverse(lmvm, [0, "metadataParts", 0, "text", "content"], default="ythdd: unknown author")
|
||||
ucid = safeTraverse(context, ["author_ucid"], default=ucid)
|
||||
author = safeTraverse(context, ["author_name"], default=author)
|
||||
|
||||
return {
|
||||
"type": "playlist",
|
||||
"title": safeTraverse(meta, ["lockupMetadataViewModel", "title", "content"], default="ythdd: unknown title"),
|
||||
"playlistId": playlist_id,
|
||||
"playlistThumbnail": thumbnail,
|
||||
"author": author,
|
||||
"authorId": ucid,
|
||||
"authorUrl": "/channel/" + ucid,
|
||||
"authorVerified": verified,
|
||||
"videoCount": length,
|
||||
"videos": [] # provided for historical reasons i guess
|
||||
}
|
||||
|
||||
case "shelfRenderer": # "people also watched"
|
||||
return
|
||||
|
||||
case "gridShelfViewModel": # shorts?
|
||||
return
|
||||
|
||||
case "shortsLockupViewModel": # shorts on channel pages
|
||||
|
||||
video_id = safeTraverse(entry, ["shortsLockupViewModel", "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"], default="UnknownVideoId")
|
||||
title = safeTraverse(entry, ["shortsLockupViewModel", "overlayMetadata", "primaryText", "content"], default="ythdd: couldn't find title")
|
||||
views_text = safeTraverse(entry, ["shortsLockupViewModel", "overlayMetadata", "secondaryText", "content"], default="No views")
|
||||
|
||||
published_date = "No data about published time" # the view model doesn't provide data about the date a short is published
|
||||
|
||||
if video_id == "UnknownVideoId": # failsafe
|
||||
video_id = safeTraverse(entry, ["shortsLockupViewModel", "entityId"], default="-UnknownVideoId")
|
||||
video_id = video_id[video_id.rfind("-") + 1:]
|
||||
|
||||
if "author_name" in context:
|
||||
author_name = context["author_name"]
|
||||
else:
|
||||
author_name = "Unknown author"
|
||||
|
||||
if "author_ucid" in context:
|
||||
author_ucid = context["author_ucid"]
|
||||
else:
|
||||
author_ucid = "UNKNOWNCHANNELID"
|
||||
|
||||
if "verified" in context:
|
||||
verified = context["verified"]
|
||||
else:
|
||||
verified = False
|
||||
|
||||
if "avatar" in context:
|
||||
avatar_url = context["avatar"]
|
||||
else:
|
||||
avatar_url = "unknown"
|
||||
|
||||
return {
|
||||
"type": "video",
|
||||
"title": title,
|
||||
"videoId": video_id,
|
||||
"author": author_name,
|
||||
"authorId": author_ucid,
|
||||
"authorUrl": "/channel/" + author_ucid,
|
||||
"authorVerified": False,
|
||||
"videoThumbnails": genThumbs(video_id),
|
||||
"description": "",
|
||||
"descriptionHtml": "",
|
||||
"viewCount": parseViewsFromViewText(views_text),
|
||||
"viewCountText": views_text,
|
||||
"published": int(0),
|
||||
"publishedText": published_date,
|
||||
"lengthSeconds": int(60), # invidious locks this to 60s no matter what the actual duration is
|
||||
"liveNow": False,
|
||||
"premium": False,
|
||||
"isUpcoming": False,
|
||||
"premiereTimestamp": 0,
|
||||
"isNew": False,
|
||||
"is4k": False,
|
||||
"is8k": False,
|
||||
"isVr180": False,
|
||||
"isVr360": False,
|
||||
"is3d": False,
|
||||
"hasCaptions": False
|
||||
}
|
||||
|
||||
case "gridVideoRenderer": # videos on channel pages
|
||||
|
||||
# doesn't work on Yattee
|
||||
# thumbnails = safeTraverse(entry, ["gridVideoRenderer", "thumbnail", "thumbnails"], default=[])
|
||||
# for thumbnail in thumbnails:
|
||||
# thumbnail["url"] = ythdd_globals.translateLinks(thumbnail["url"])
|
||||
|
||||
video_id = safeTraverse(entry, ["gridVideoRenderer", "videoId"], default="UnknownVideoId")
|
||||
thumbnails = genThumbs(video_id)
|
||||
|
||||
published_date = safeTraverse(entry, ["gridVideoRenderer", "publishedTimeText", "simpleText"], default="now")
|
||||
published_date = published_date.removeprefix("Streamed ")
|
||||
|
||||
return {
|
||||
"type": "video",
|
||||
"title": safeTraverse(entry, ["gridVideoRenderer", "title", "simpleText"], default="unknown video title"),
|
||||
"videoId": video_id,
|
||||
"author": context["author_name"],
|
||||
"authorId": context["author_ucid"],
|
||||
"authorUrl": "/channel/" + context["author_ucid"],
|
||||
"authorVerified": False, # TODO: handle badge related tasks here using context
|
||||
"videoThumbnails": thumbnails,
|
||||
"description": "", # won't work without using an RSS feed (?)
|
||||
"descriptionHtml": "", # -||-
|
||||
"viewCount": parseViewsFromViewText(safeTraverse(entry, ["gridVideoRenderer", "viewCountText", "simpleText"], default="0 views")),
|
||||
"viewCountText": safeTraverse(entry, ["gridVideoRenderer", "shortViewCountText", "simpleText"], default="0 views"),
|
||||
"published": int(dateparser.parse(published_date).timestamp()),
|
||||
"publishedText": published_date,
|
||||
"lengthSeconds": parseLengthFromTimeBadge(safeTraverse(entry, ["gridVideoRenderer", "thumbnailOverlays", 0, "thumbnailOverlayTimeStatusRenderer", "text", "simpleText"], default="0:0")),
|
||||
"liveNow": True if published_date == "now" else False,
|
||||
"premium": False,
|
||||
"isUpcoming": False,
|
||||
"isNew": False,
|
||||
"is4k": False,
|
||||
"is8k": False,
|
||||
"isVr180": False,
|
||||
"isVr360": False,
|
||||
"is3d": False,
|
||||
"hasCaptions": False
|
||||
}
|
||||
|
||||
case "channelRenderer": # channels in search results
|
||||
|
||||
avatars = ythdd_extractor.generateChannelAvatarsFromUrl(safeTraverse(entry, ["channelRenderer", "thumbnail", "thumbnails", 0, "url"], default="no-avatar"))
|
||||
description, description_html = parseDescriptionSnippet(safeTraverse(entry, ["channelRenderer", "descriptionSnippet", "runs"], default=[]))
|
||||
isVerified = ythdd_extractor.isVerified(safeTraverse(entry, ["channelRenderer", "ownerBadges", 0], default=[]))
|
||||
|
||||
return {
|
||||
"type": "channel",
|
||||
"author": safeTraverse(entry, ["channelRenderer", "title", "simpleText"], default="Unknown channel"),
|
||||
"authorId": safeTraverse(entry, ["channelRenderer", "channelId"], default="UNKNOWNCHANNELID"),
|
||||
"authorUrl": "/channel/" + safeTraverse(entry, ["channelRenderer", "channelId"], default="UNKNOWNCHANNELID"),
|
||||
"authorVerified": isVerified,
|
||||
"authorThumbnails": avatars,
|
||||
"autoGenerated": False,
|
||||
"subCount": parseViewsFromViewText(safeTraverse(entry, ["channelRenderer", "videoCountText", "simpleText"], default="0 subscribers")),
|
||||
"videoCount": 0,
|
||||
"channelHandle": safeTraverse(entry, ["channelRenderer", "navigationEndpoint", "browseEndpoint", "canonicalBaseUrl"], default="/@ythdd_unknown_handle")[1:],
|
||||
"description": description,
|
||||
"descriptionHtml": description_html
|
||||
}
|
||||
|
||||
case _:
|
||||
print("received an entry of unknown type:")
|
||||
print(entry)
|
||||
print("")
|
||||
# breakpoint()
|
||||
return
|
||||
|
||||
def parseDescriptionSnippet(snippet: list):
|
||||
|
||||
text = ""
|
||||
text_html = ""
|
||||
for entry in snippet:
|
||||
text += entry["text"]
|
||||
if "bold" in entry: # is checking entry["bold"] == True necessary?
|
||||
text_html += "<b>" + entry["text"] + "</b>"
|
||||
else:
|
||||
text_html += entry["text"]
|
||||
text_html = escape(text_html).replace("\r\n", "<br>").replace("\n", "<br>")
|
||||
|
||||
return text, text_html
|
||||
Reference in New Issue
Block a user