feat: storyboard generation (json, webvtt) and proxy

adds support for video storyboard extraction, generation and proxying
2025-10-15 00:03:45 +02:00
parent c760104d70
commit b0845d723a
6 changed files with 218 additions and 33 deletions
--- a/ythdd_inv_tl.py
+++ b/ythdd_inv_tl.py
@@ -12,9 +12,11 @@ import json, datetime
 import dateparser
 import html
 import invidious_formats
+import math
 import ythdd_globals
 import ythdd_api_v1
 import ythdd_extractor
+import ythdd_struct_builder
 import ythdd_struct_parser

 # TODOs:
@@ -32,9 +34,9 @@ import ythdd_struct_parser
 # [✓] /api/v1/playlists/:plid
 # [✓] /api/v1/channel/{videos, shorts, playlists, streams, latest?}/:ucid (rewrite)
 # [✓] /api/v1/:videoIdXXXX/maxres.jpg redirects to best quality thumbnail
+# [✓] /api/v1/storyboards/:videoIdXXXX
 # ----------
 # PLANNED:
-# [X] /api/v1/storyboards/:videoIdXXXX
 # [X] /api/v1/videos/:videoIdXXXX does not depend on yt-dlp and offloads stream retrieval elsewhere (making initial response fast)
 # [X] /api/v1/manifest/:videoIdXXXX (above is prerequisite)
 # [X] rewrite the awful lookup logic
@@ -425,7 +427,7 @@ def videos(data):
 				lmvm = safeTraverse(y, ['metadata', 'lockupMetadataViewModel'], default=[])
 				related_entry['videoId']          = safeTraverse(y, ['contentId'])
 				related_entry['title']            = safeTraverse(lmvm, ['title', 'content'])
-				related_entry['videoThumbnails']  = ythdd_struct_parser.genThumbs(related_entry['videoId']) #safeTraverse(y, ['thumbnail', 'thumbnails'])
+				related_entry['videoThumbnails']  = ythdd_struct_builder.genThumbs(related_entry['videoId']) #safeTraverse(y, ['thumbnail', 'thumbnails'])
 				related_entry['author']           = safeTraverse(lmvm, ['metadata', 'contentMetadataViewModel', 'metadataRows', 0, 'metadataParts', 0, 'text', 'content'])
 				related_entry['authorId']         = safeTraverse(lmvm, ['image', 'decoratedAvatarViewModel', 'rendererContext', 'commandContext', 'onTap', 'innertubeCommand', 'browseEndpoint', 'browseId'], default="UNKNOWNCHANNELID")
 				related_entry['authorUrl']        = '/channel/' + related_entry['authorId']
@@ -493,6 +495,10 @@ def videos(data):
 	# requests for the video's comments don't have to
 	# spawn an additional request for initial ctoken
 	ensure_comment_continuation(video_id, wdata)
+	storyboards = []
+	storyboards_extracted = ensure_storyboards(video_id, wdata, length=length)
+	if storyboards_extracted:
+		storyboards = ythdd_struct_builder.genStoryboards(video_id)

 	time_end = time()

@@ -500,8 +506,8 @@ def videos(data):
 		"type": video_type,
 		"title": title,
 		"videoId": video_id,
-		"videoThumbnails": ythdd_struct_parser.genThumbs(video_id),
-		"storyboards": [], # not implemented
+		"videoThumbnails": ythdd_struct_builder.genThumbs(video_id),
+		"storyboards": storyboards,
 		"description": description, # due to change (include ythdd metadata)
 		"descriptionHtml": description_html,
 		"published": published,
@@ -788,6 +794,7 @@ def ensure_comment_continuation(video_id: str, wdata = None):
 			wdata = ythdd_extractor.WEBextractSinglePage(video_id)

 		# search for "top comments" continuation token
+		# todo: replace this with on-demand continuation creation
 		comment_continuation = safeTraverse(wdata, ["ec2", "engagementPanels", 0, "engagementPanelSectionListRenderer", "header", "engagementPanelTitleHeaderRenderer", "menu", "sortFilterSubMenuRenderer", "subMenuItems", 0, "serviceEndpoint", "continuationCommand", "token"], default=None)
 		if comment_continuation is not None:
 			ythdd_globals.general_cache["continuations"]["comments"][video_id].append(comment_continuation)
@@ -795,6 +802,77 @@ def ensure_comment_continuation(video_id: str, wdata = None):
 			print(f"error: couldn't extract comment continuation token from video page ({video_id}). this video likely has comments disabled.")
 			ythdd_globals.general_cache["continuations"]["comments"][video_id].append("")

+def ensure_storyboards(video_id: str, wdata = None, length = 60):
+	# Returns True on successful extraction, False when it failed.
+
+	# Storyboards don't expire. They can be cached indefinitely.
+	if not video_id in ythdd_globals.general_cache["storyboards"]:
+		ythdd_globals.general_cache["storyboards"][video_id] = None
+
+		if wdata is None:
+			wdata = ythdd_extractor.WEBextractSinglePage(video_id)
+
+		# get storyboard template string
+		storyboards = None
+		storyboard_template = safeTraverse(wdata, ["ec1", "storyboards", "playerStoryboardSpecRenderer", "spec"], default=None)
+		# silly sanity check, todo: do a regex one instead?
+		if isinstance(storyboard_template, str):
+			# sample storyboard template url structure, indented for readability
+			# https://i.ytimg.com/sb/:videoId/storyboard3_L$L/$N.jpg?sqp=b64encodedprotobuf
+			# |  48 #  27 # 100 # 10 # 10 #    0 # default # rs$datadatadatadatadatadatadatadatada
+			# |  80 #  45 #  55 # 10 # 10 # 1000 #     M$M # rs$datadatadatadatadatadatadatadatada
+			# | 160 #  90 #  55 #  5 #  5 # 1000 #     M$M # rs$datadatadatadatadatadatadatadatada
+			# | 320 # 180 #  55 #  3 #  3 # 1000 #     M$M # rs$datadatadatadatadatadatadatadatada
+			# ^ width, height, thumb_count, columns, rows, interval, $N, sigh parameter. $L is just the index of a given storyboard, say, 0 for $N=default
+			
+			# try to extract data from the storyboard template
+			try:
+				base_url, *formats = storyboard_template.split("|")
+
+				extracted_formats = []
+				for index, fmt in enumerate(formats):
+					fmt      = fmt.split("#")
+					width    = int(fmt[0])
+					height   = int(fmt[1])
+					count    = int(fmt[2])
+					columns  = int(fmt[3])
+					rows     = int(fmt[4])
+					interval = int(fmt[5])
+					name     =     fmt[6]
+					sigh     =     fmt[7]
+
+					thumbs_per_image = columns * rows
+					images_count = math.ceil(count / thumbs_per_image)
+					interval = interval if interval != 0 else int((length / count) * 1000) # calculated only for $N=default as it's the only one that has interval=0
+
+					extracted_formats.append({
+						"index": index,
+						"width": width,
+						"height": height,
+						"thumb_count": count,
+						"columns": columns,
+						"rows": rows,
+						"interval": interval,
+						"name": name,
+						"sigh": sigh,
+						"images_count": images_count
+					})
+
+				storyboards = {
+					"template_url": ythdd_globals.translateLinks(base_url, remove_params=False), # NOT removing params is crucial, otherwise sqp will be dropped!
+					"formats": extracted_formats
+				}
+
+				ythdd_globals.general_cache["storyboards"][video_id] = storyboards
+				return True
+			except:
+				print("error(ensure_storyboards): storyboard template url layout changed. please update ythdd for latest storyboard extraction fixes.")
+				return False
+		else:
+			print(f"error(ensure_storyboards: couldn't extract storyboards from video page ({video_id}). this video won't have storyboards.")
+			return False
+
+
 def channels(data, req, only_json: bool = False):

 	# prevent potential out of bound read
@@ -998,6 +1076,25 @@ def playlists(data, req, only_json: bool = False):

 	return send(200, response)

+def storyboards(data, req):
+
+	height = req.args.get("height")
+	width  = req.args.get("width")
+	video_id = data[3]
+
+	try:
+		height = int(height)
+	except:
+		pass
+	try:
+		width = int(width)
+	except:
+		pass
+
+	resp = ythdd_struct_builder.genWebvttStoryboard(video_id, width, height)
+
+	return Response(resp, mimetype="text/vtt", status=200)
+
 def lookup(data, req):
 	# possibly TODO: rewrite this mess
 	if len(data) > 2:
@@ -1021,6 +1118,8 @@ def lookup(data, req):
 					return get_comments(data, req)
 				case 'playlists':
 					return playlists(data, req)
+				case 'storyboards':
+					return storyboards(data, req)
 				case _:
 					incrementBadRequests()
 					return notImplemented(data)