feat: video comments endpoint

ythdd_extractor.py

@@ -434,4 +434,86 @@ def WEBgetSearchSuggestions(query: str, previous_query: str = '') -> list:
 	return {
 		"query": query,
 		"suggestions": suggestions
-	}
+	}
+
+def WEBgetVideoComments(ctoken: str) -> tuple:
+
+	# ctoken needs to be passed explicitly.
+	# no guessing or retrieving it from globals.
+	if ctoken is None:
+		return [], ""
+
+	# build web context containing the relevant ctoken
+	web_context = makeWebContext({"continuation": ctoken})
+	response = requests.post('https://www.youtube.com/youtubei/v1/next', 
+		params={"prettyPrint": False}, 
+		headers=stage2_headers, 
+		data=json.dumps(web_context)
+	)
+
+	results = []
+	try:
+		results = json.loads(response.text)
+	except:
+		pass
+
+	comments = safeTraverse(results, ["frameworkUpdates", "entityBatchUpdate", "mutations"], default=[])
+	comment_continuations = []
+	comment_continuations_re = safeTraverse(results, ["onResponseReceivedEndpoints"], default=[])
+	for received_endpoint in comment_continuations_re:
+
+		# this is horrible...
+
+		acia = safeTraverse(received_endpoint, ["appendContinuationItemsAction", "continuationItems"], default=[])
+		rcic = safeTraverse(received_endpoint, ["reloadContinuationItemsCommand", "continuationItems"], default=[])
+
+		for entry in acia:
+			if "commentThreadRenderer" in entry or "continuationItemRenderer" in entry:
+				comment_continuations = acia
+				break
+
+		for entry in rcic:
+			if "commentThreadRenderer" in entry or "continuationItemRenderer" in entry:
+				comment_continuations = rcic
+				break
+
+		if comment_continuations != []:
+			break
+
+	if comment_continuations == []:
+		print("error: received an unknown comment structure, unable to parse continuations (replies)")
+		# breakpoint()
+		# return [], ""
+
+	# extract new continuation
+	new_continuation = ""
+	if "continuationItemRenderer" in safeTraverse(comment_continuations, [-1], default=[]):
+		# first, look for ctoken inside of response for next page of comments
+		new_continuation = safeTraverse(comment_continuations, [-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token"], default=None)
+		# or search elsewhere in case this is a reply thread
+		if new_continuation is None:
+			new_continuation = safeTraverse(comment_continuations, [-1, "continuationItemRenderer", "button", "buttonRenderer", "command", "continuationCommand", "token"], default="")
+
+	# perform a basic mutation check before parsing
+	# will ignore replies liked by video uploader ("hearts")
+	actual_comments = [x for x in comments if "properties" in safeTraverse(x, ["payload", "commentEntityPayload"], default=[], quiet=True)]
+	actual_comment_continuations = [x for x in comment_continuations if "replies" in safeTraverse(x, ["commentThreadRenderer"], default=[], quiet=True)]
+
+	# link reply data (reply count and ctoken) for comments with replies
+	for reply_renderer in actual_comment_continuations:
+
+		mutual_key   = safeTraverse(reply_renderer, ["commentThreadRenderer", "commentViewModel", "commentViewModel", "commentKey"], default="unknown-key")
+		reply_ctoken = safeTraverse(reply_renderer, ["commentThreadRenderer", "replies", "commentRepliesRenderer", "contents", 0, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token"], default="")
+		reply_count  = safeTraverse(reply_renderer, ["commentThreadRenderer", "replies", "commentRepliesRenderer", "viewReplies", "buttonRenderer", "text", "runs", 0, "text"], default="0 replies").split(" ")[0]
+
+		for comment in actual_comments:
+			found_key = safeTraverse(comment, ["entityKey"], default="unknown-key")
+			# try to link a relevant ctoken if a comment has response
+			if found_key == mutual_key:
+				if ythdd_globals.config["general"]["debug"]: print(f"found reply for {found_key}")
+				comment["replies"] = {
+					"replyCount": int(reply_count),
+					"continuation": reply_ctoken
+				}
+
+	return actual_comments, new_continuation