feat: support age-restricted videos when cookies are provided

config.default.toml

@@ -12,9 +12,11 @@ api_key_admin = "CHANGEME"   # Empty *admin* API key will autogenerate a random
 enable_debugger_halt = false # Whether to allow to trigger pdb using admin's API key.
 
 [extractor]
-user-agent = ""          # Leave empty for default (Firefox ESR).
-cookies_path = ""        # Leave empty for none.
-preferred_extractor = "" # Leave empty for default (android_vr).
+user-agent = ""                  # Leave empty for default (Firefox ESR).
+cookies_path = ""                # Leave empty for none.
+age_restricted_cookies_path = "" # Cookies to use when bypassing age-gated videos only. Leave empty to disable.
+deno_path = ""                   # Required when using cookies.
+preferred_extractor = ""         # Leave empty for default (android_vr).
 
 [proxy]
 user-agent = ""               # Leave empty for default (Firefox ESR).

ythdd_extractor.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 import brotli, yt_dlp, requests, json, time
+from http.cookiejar import MozillaCookieJar
 from ythdd_globals import safeTraverse
 import ythdd_proto
 import ythdd_globals
@@ -19,7 +20,11 @@ ytdl_opts = {
 				# "formats": ["dashy"]
 			}
 	},
-	"simulate": True
+	"simulate": True,
+	"js_runtimes": {
+		"deno": {}
+	},
+	'remote_components': ['ejs:github']
 }
 
 stage1_headers = {
@@ -129,7 +134,7 @@ web_context_dict = {
     }
 }
 
-def extract(url: str, getcomments=False, maxcomments="", manifest_fix=False):
+def extract(url: str, getcomments=False, maxcomments="", manifest_fix=False, use_cookies=None):
 	# TODO: check user-agent and cookiefile
 
 	ytdl_context = ytdl_opts.copy()
@@ -137,9 +142,6 @@ def extract(url: str, getcomments=False, maxcomments="", manifest_fix=False):
 	if ythdd_globals.config['extractor']['user-agent']:
 		yt_dlp.utils.std_headers['User-Agent'] = ythdd_globals.config['extractor']['user-agent']
 
-	if ythdd_globals.config['extractor']['cookies_path']:
-		ytdl_context['cookiefile'] = ythdd_globals.config['extractor']['cookies_path']
-
 	if len(url) == 11:
 		url = "https://www.youtube.com/watch?v=" + url
 	if getcomments:
@@ -153,7 +155,27 @@ def extract(url: str, getcomments=False, maxcomments="", manifest_fix=False):
 		ytdl_context['extractor_args']['youtube']['player_client'] = [ythdd_globals.config['extractor']['preferred_extractor']]
 	else:
 		ytdl_context['extractor_args']['youtube']['player_client'] = ['android_vr']
-	with yt_dlp.YoutubeDL(ytdl_opts) as ytdl:
+
+	if use_cookies is not None:
+		# can be either "global", "agegated" or None
+		deno_path = ythdd_globals.config['extractor']['deno_path']
+		match use_cookies:
+			case "global":
+				ytdl_context['cookiefile'] = ythdd_globals.config['extractor']['cookies_path']
+				ytdl_context['extractor_args']['youtube']['player_client'] = ['mweb', 'tv']
+				if not deno_path:
+					print("FATAL ERROR: deno path is required for playback using cookies!")
+				ytdl_context['js_runtimes']['deno']['path'] = deno_path if deno_path else ""
+			case "agegated":
+				ytdl_context['cookiefile'] = ythdd_globals.config['extractor']['age_restricted_cookies_path']
+				ytdl_context['extractor_args']['youtube']['player_client'] = ['mweb', 'tv']
+				if not deno_path:
+					print("FATAL ERROR: deno path is required for playback of age-restricted content!")
+				ytdl_context['js_runtimes']['deno']['path'] = deno_path if deno_path else ""
+			case None | _:
+				pass
+
+	with yt_dlp.YoutubeDL(ytdl_context) as ytdl:
 		result = ytdl.sanitize_info(ytdl.extract_info(url, download=False))
 	return result
 
@@ -177,7 +199,7 @@ def WEBrelated(url: str):
 
 	return extracted_json["contents"]['twoColumnWatchNextResults']["secondaryResults"]
 
-def WEBextractSinglePage(uri: str):
+def WEBextractSinglePage(uri: str, use_cookies=None):
 	# WARNING! HIGHLY EXPERIMENTAL, DUE TO BREAK ANYTIME
 	
 	start_time = time.time()
@@ -185,7 +207,21 @@ def WEBextractSinglePage(uri: str):
 	if len(uri) != 11:
 		raise ValueError("WEBextractSinglePage expects a single, 11-character long argument")
 
-	response = requests.get("https://www.youtube.com/watch?v=" + uri, headers=ythdd_globals.getHeaders(caller='extractor'))
+	cookies = None
+	if use_cookies is not None:
+		match use_cookies:
+			case "global":
+				ythdd_globals.print_debug("wdata: using global cookies")
+				cookies = MozillaCookieJar(ythdd_globals.config["extractor"]["cookies_path"])
+				cookies.load()
+			case "agegated":
+				ythdd_globals.print_debug("wdata: using agegated cookies")
+				cookies = MozillaCookieJar(ythdd_globals.config["extractor"]["age_restricted_cookies_path"])
+				cookies.load()
+			case None | _:
+				pass
+
+	response = requests.get("https://www.youtube.com/watch?v=" + uri, headers=ythdd_globals.getHeaders(caller='extractor'), cookies=cookies)
 	extracted_string = str(response.content.decode('utf8', 'unicode_escape'))
 	start = extracted_string.find('{"responseContext":{"serviceTrackingParams":')
 	end = extracted_string.find(';var ', start)

ythdd_inv_tl.py

@@ -163,11 +163,24 @@ def videos(data):
 
 	wdata = ythdd_extractor.WEBextractSinglePage(data[3])
 
+	age_restricted = False
 	error = getError(wdata)
 	if error is not None:
-		return send(500, {"status": "error", "error": error})
-
-	ydata = ythdd_extractor.extract(data[3])
+		if error.startswith("(LOGIN_REQUIRED)") and "inappropriate for some users" in error:
+			# check if user provided age-gated cookies
+			if ythdd_globals.config["extractor"]["age_restricted_cookies_path"]:
+				ythdd_globals.print_debug(f"videos({data[3]}): using agegated cookies to bypass restriction")
+				ydata = ythdd_extractor.extract(data[3], use_cookies="agegated")
+				wdata = ythdd_extractor.WEBextractSinglePage(data[3], use_cookies="agegated")
+				age_restricted = True
+			else:
+				# return error if no age-gated cookies are provided
+				return send(500, {"status": "error", "error": error})
+		else:
+			# return error if it doesn't mention age restriction
+			return send(500, {"status": "error", "error": error})
+	else:
+		ydata = ythdd_extractor.extract(data[3])
 
 	#return send(200, {'ydata': ydata, 'wdata': wdata})
 	#return send(200, {'idata': idata, 'wdata': wdata})
@@ -301,6 +314,11 @@ def videos(data):
 		adaptive_formats, format_streams = [{"url": f"http://a/?expire={int(time_start + 5.9 * 60 * 60)}", "itag": "18", "type": "", "clen": "0", "lmt": "", "projectionType": "RECTANGULAR"}], [] # freetube/clipious shenanigans, see: https://github.com/FreeTubeApp/FreeTube/pull/5997 and https://github.com/lamarios/clipious/blob/b9e7885/lib/videos/models/adaptive_format.g.dart
 		hls_url = safeTraverse(ydata, ["url"], default="ythdd: unable to retrieve stream url")
 
+	if age_restricted:
+		adaptive_formats  = [{"url": f"http://a/?expire={int(time_start + 5.9 * 60 * 60)}", "itag": "18", "type": "", "clen": "0", "lmt": "", "projectionType": "RECTANGULAR"}] # same as above
+		description      += "   \n(ythdd: this video is age-restricted and thus only available in 360p - itag 18)"
+		description_html += "<br/>(ythdd: this video is age-restricted and thus only available in 360p - itag 18)"
+
 	if live_now:
 		video_type         = "livestream"
 		premiere_timestamp = published # ??? that works i guess