feat: videoplayback proxying

adds support for proxying videos through the instance
the support is configurable, and disabled by default

config.default.toml

@@ -1,5 +1,5 @@
 [general]
-db_file_path = "/path/to/ythdd_db.sqlite"          # Preferably stored on an SSD.
+db_file_path = "ythdd_db.sqlite"                   # Path to the databse file, preferably stored on an SSD.
 video_storage_directory_path = "/path/to/videos/"  # Path to video vault.
 is_proxied = false                                 # Set to true if running behind reverse proxy.
 public_facing_url = "http://127.0.0.1:5000/"       # Used for URL rewriting. Note the trailing backslash /.
@@ -17,6 +17,8 @@ preferred_extractor = "" # Leave empty for default (android_vr).
 
 [proxy]
 user-agent = ""               # Leave empty for default (Firefox ESR).
+allow_proxying_videos = false # Whether to allow video proxying through the instance (traffic-intensive).
+match_initcwndbps = true      # Experimental: matches proxying speed to the one suggested by Innertube (may help avoid being ratelimited/banned).
 
 [admin]
 # List of users with admin priviledges.

views.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
-from flask import render_template, Response
+from flask import render_template, request, Response
 from flask_sqlalchemy import SQLAlchemy
 from markupsafe import escape
-import requests, json
+import json, re, requests
 import ythdd_globals
 
 def homepage():
@@ -79,3 +79,63 @@ def imgProxy(received_request):
 	response = Response(thumbnail.raw, mimetype=thumbnail.headers['content-type'], status=thumbnail.status_code)
 
 	return response
+
+def videoplaybackProxy():
+	# inspired by Yotter's video proxy
+	# https://github.com/ytorg/Yotter/blob/b43a72ab7bfa5a59916fa3259cbc39165717c6bb/app/routes.py#L527
+
+	if not ythdd_globals.config['proxy']['allow_proxying_videos']:
+		return Response(json.dumps({"error": "Administrator has disabled this endpoint"}), mimetype="application/json", status=403)
+
+	headers = dict(request.headers)
+	proxy_headers = ythdd_globals.getHeaders(caller='proxy')
+	if "Range" in headers:
+		proxy_headers["Range"] = headers["Range"]
+
+	params = dict(request.args)
+
+	# reconstruct the url
+	# first attempt: from host param
+	host = params.get('host')
+	# failed? then try to get it from the rest of the params
+	if host is None:
+		# second attempt: reconstruct url from mn and mvi?
+		# the host schema seems to be as follows:
+		# rr{mvi[any]/fvip[any]?}---{mn[any]}.googlevideo.com
+		# regarding mvi/fvip, it seems that any value smaller than 5 passes
+		try:
+			mvi = params.get('mvi').split(',')[-1]
+			mn  = params.get('mn').split(',')[-1]
+			if int(mvi) > 5:
+				mvi = 3 # invidious uses this as fallback
+			host = f"rr{mvi}---{mn}.googlevideo.com"
+		except (AttributeError, ValueError):
+			return Response(json.dumps({"error": "Couldn't extract crucial parameters for hostname reconstruction"}, mimetype="application/json", status=400))
+	else:
+		# don't echo host "hint" back to the googlevideo server
+		del params['host']
+	# run a regex sanity check
+	if re.fullmatch(r"[\w-]+\.googlevideo\.com", host) is None:
+		# fallback behavior for unexpected hostnames
+		return Response(json.dumps({"error": "Please either pass a valid host, or don't pass any"}), mimetype="application/json", status=400)
+
+	try:
+		# request the proxied data
+		remote_response = requests.get(f"https://{host}/videoplayback", headers=proxy_headers, params=params, stream=True)
+	except:
+		return Response(json.dumps({"error": "Couldn't connect to googlevideo host"}), mimetype="application/json", status=500)
+
+	# determine the chunk size
+	chunk_size = 10 * 1024 # by default it's 10 MB (as this is the most youtube is willing to send without ratelimiting)
+	# or the one in initcwndbps (if user enabled the config flag to match chunk_size with initcwndbps)
+	if ythdd_globals.config['proxy']['match_initcwndbps']:
+		try:
+			chunk_size = int(params.get('initcwndbps')) / 1024
+		except:
+			pass
+	# return a chunked response
+	resp = Response(remote_response.iter_content(chunk_size=chunk_size), content_type=remote_response.headers['Content-Type'], status=remote_response.status_code, headers=remote_response.headers, direct_passthrough=True)
+	resp.cache_control.public = True
+	resp.cache_control.max_age = int(60_000)
+
+	return resp

ythdd.py

@@ -67,6 +67,7 @@ def setup():
 	app.add_url_rule('/ggpht/<path:received_request>', view_func=views.ggphtProxy)
 	app.add_url_rule('/guc/<path:received_request>', view_func=views.gucProxy)
 	app.add_url_rule('/img/<path:received_request>', view_func=views.imgProxy)
+	app.add_url_rule('/videoplayback', view_func=views.videoplaybackProxy)
 	db = ythdd_db.initDB(app, config)
 
 	with app.app_context():
@@ -134,7 +135,7 @@ def main(args):
 	app_port = port
 
 	setup()
-	app.run(host=host, port=int(port))
+	app.run(host=host, port=int(port), threaded=True)
 
 if __name__ == "__main__":
 	#app.run(host="127.0.0.1", port=5000)

ythdd_globals.py

@@ -32,7 +32,7 @@ def getConfig(configfile):
 	global randomly_generated_passcode
 
 	if not os.path.exists(configfile):
-		dummy_config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False, 'public_facing_url': 'http://127.0.0.1:5000/', 'debug': False, 'cache': True}, 'api': {'api_key': 'CHANGEME'}, 'proxy': {'user-agent': ''}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}}
+		dummy_config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False, 'public_facing_url': 'http://127.0.0.1:5000/', 'debug': False, 'cache': True}, 'api': {'api_key': 'CHANGEME'}, 'proxy': {'user-agent': '', 'allow_proxying_videos': True, 'match_initcwndbps': True}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}}
 		# if a passcode has not been provided by the user (config file doesn't exist, and user didn't specify it using an argument)
 		print(f"{colors.WARNING}WARNING{colors.ENDC}: Using default, baked in config data. {colors.ENDL}"
 			  f"         Consider copying and editing the provided example file ({colors.OKCYAN}config.default.toml{colors.ENDC}).")

ythdd_inv_tl.py

@@ -3,7 +3,7 @@
 # -----
 # Translates requests sent through Invidious API at /api/invidious/
 # to use internal extractors.
-from flask import Response, request, redirect
+from flask import Response, request, redirect, url_for
 from markupsafe import escape
 from time import strftime, gmtime, time
 from ythdd_globals import safeTraverse
@@ -1045,4 +1045,6 @@ def lookup(data, req):
 			return notImplemented(data)
 
 	elif len(data) == 1:
+		if data[0] == "videoplayback":
+			return redirect(url_for('videoplaybackProxy', **req.args))
 		return stats() # /api/invidious/something