major rework of ythdd.py, new method for setting config in ythdd_globals.py

- slightly modified config, api keys now have the value "CHANGEME"
- requirements.txt has new dependency, flask apscheduler
- ythdd.py has been reworked, support for argument parsing has been added, code is now split into functions
- ythdd_api_v1.py features real uptime as well
- ythdd_db.py is no longer dependent on ythdd_globals.py
- ythdd_globals.py has a method for setting config and getting it from configfile variable

.gitignore

@@ -0,0 +1,8 @@
+# dotfiles, catalogues
+.venv
+.git
+__pycache__
+instance
+
+# config files
+config.toml

config.default.toml

@@ -4,9 +4,8 @@ video_storage_directory_path = "/path/to/videos/" # Path to video vault.
 is_proxied = false
 
 [api]
-# Leave empty to autogenerate api keys every launch (secure).
+api_key = "" # Leave empty API key for public access to non-sensitive backend
-api_key = ""
+api_key_admin = "CHANGEME" # Empty *admin* API key will autogenerate a random one every launch.
-api_key_admin = ""
 
 [extractor]
 user-agent = "" # leave empty for default

requirements.txt

@@ -9,3 +9,6 @@ greenlet>=3.1.0
 SQLAlchemy>=2.0.36.dev0
 Flask-SQLAlchemy>=3.1.1
 toml>=0.10.2
+Flask-APScheduler>=1.13.1
+requests>=2.32.3
+yt_dlp

ythdd.py

@@ -2,29 +2,56 @@
 from flask import Flask, render_template
 from flask_sqlalchemy import SQLAlchemy
 from markupsafe import escape
-#from argparse import ArgumentParser
+from argparse import ArgumentParser
-from ythdd_globals import config, colors
+from ythdd_globals import colors
 import requests, json, toml, time
 import views, downloader, ythdd_api, ythdd_globals, ythdd_db
-
+from flask_apscheduler import APScheduler
-ythdd_globals.starttime = int(time.time())
-ythdd_globals.apiRequests = 0
-ythdd_globals.apiFailedRequests = 0
-ythdd_globals.isProxied = config['general']['is_proxied']
-ythdd_globals.outsideApiHits = 0
 
 app = Flask(__name__)
-app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{config['general']['db_file_path']}"
-app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-app.add_url_rule('/', view_func=views.index)
-app.add_url_rule('/index.html', view_func=views.index)
-app.add_url_rule('/home', view_func=views.home)
-app.add_url_rule('/api/', view_func=ythdd_api.api_greeting)
-app.add_url_rule('/api/<path:received_request>', view_func=ythdd_api.api_global_catchall)
-db = ythdd_db.initDB(app)
 
-with app.app_context():
+def setup():
-	db.create_all()
+
+	# sanity check: make sure config is set
+	# required to make `flask --app ythdd run --debug` work
+	global config
+	try:
+		if not config['general']:
+			ythdd_globals.setConfig(ythdd_globals.configfile)
+			config = ythdd_globals.config
+	except:
+		ythdd_globals.setConfig(ythdd_globals.configfile)
+		config = ythdd_globals.config
+
+	# setting all the variables
+	ythdd_globals.starttime = int(time.time())
+	ythdd_globals.realUptime = 0
+	ythdd_globals.apiRequests = 0
+	ythdd_globals.apiFailedRequests = 0
+	ythdd_globals.isProxied = config['general']['is_proxied']
+	ythdd_globals.outsideApiHits = 0
+
+	app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{config['general']['db_file_path']}"
+	app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+	app.add_url_rule('/', view_func=views.index)
+	app.add_url_rule('/index.html', view_func=views.index)
+	app.add_url_rule('/home', view_func=views.home)
+	app.add_url_rule('/api/', view_func=ythdd_api.api_greeting)
+	app.add_url_rule('/api/<path:received_request>', view_func=ythdd_api.api_global_catchall)
+	db = ythdd_db.initDB(app, config)
+
+	with app.app_context():
+		db.create_all()
+
+	# job scheduler for repetetive tasks
+	scheduler = APScheduler()
+	scheduler.add_job(func=every5seconds, trigger='interval', id='job', seconds=5)
+	scheduler.start()
+
+# gets called every 5 seconds
+def every5seconds():
+	# update the "real" uptime counter
+	ythdd_globals.realUptime += 5
 
 @app.route('/<string:val>', methods=['GET'])
 def blank(val):
@@ -32,23 +59,60 @@ def blank(val):
 	#users = db.session.query(LocalUsers).all()
 	#return users
 
-def main():
+def main(args):
 	print(f"{colors.BOLD + colors.HEADER}Welcome to ythdd ({ythdd_globals.version})!{colors.ENDC}")
 	print(f"To run in development mode (to see changes updated live), use: {colors.OKCYAN}flask --app ythdd run --debug{colors.ENDC}.")
 	print("To run locally, use IP 127.0.0.1. To run on all interfaces, use 0.0.0.0.\n")
-	print("Enter hostname:port to run Flask app on [127.0.0.1:5000]:")
 	try:
-		host_port = input('> ').split(':')
+		host = args.ip
-		if host_port == ['']:
+		port = args.port
-			host_port = ['127.0.0.1', '5000'] # defaults
+		if not host or not port:
+			raise Exception 
+	except:
+		print("Enter hostname:port to run Flask app on [127.0.0.1:5000]:")
+		try:
+			host_port = input('> ').split(':')
+			if host_port == ['']:
+				host_port = ['127.0.0.1', '5000'] # defaults
 
-	except KeyboardInterrupt:
+		except KeyboardInterrupt:
-		print(" ...exiting gracefully."), quit() # handle Ctrl+C
+			print(" ...exiting gracefully."), quit() # handle Ctrl+C
 
-	app.run(host=host_port[0], port=int(host_port[1]))
+		host = host_port[0]
+		port = host_port[1]
+	
+	global config
+	try:
+		# if specified, use custom config file
+		ythdd_globals.configfile = args.config
+		ythdd_globals.setConfig(ythdd_globals.configfile)
+
+	except:
+		# if not, use dummy file
+		ythdd_globals.configfile = ""
+		# but try to set the API secret if provided by the user
+		if args.secret:
+			ythdd_globals.randomly_generated_passcode = args.secret
+		ythdd_globals.setConfig(ythdd_globals.configfile)
+
+	config = ythdd_globals.config
+
+	setup()
+	app.run(host=host, port=int(port))
 
 if __name__ == "__main__":
 	#app.run(host="127.0.0.1", port=5000)
 	#app.run(host="0.0.0.0", port=5000)
-	# TODO: add argument parser to load config files, etc.
+	parser = ArgumentParser(description='A basic yt_dlp-based script for video download.')
-	main()
+
+	parser.add_argument("-i", "--ip",     dest="ip",     help="ip address/interface to bind to")
+	parser.add_argument("-p", "--port",   dest="port",   help="port on which the flask web backend should be ran")
+	parser.add_argument("-c", "--config", dest="config", help="path to TOML config file")
+	parser.add_argument("-s", "--secret", dest="secret", help="admin's secret passcode for sensitive API access")
+
+	args = parser.parse_args()
+
+	main(args)
+
+else:
+	setup()

ythdd_api_v1.py

@@ -20,8 +20,10 @@ def stub_hello():
 
 def stats():
 	data_to_send = {
-		"starttime": ythdd_globals.starttime,
+		# TODO: include yt-dlp version
+		"start_time": ythdd_globals.starttime,
 		"uptime": ythdd_globals.getUptime(),
+		"real_uptime": ythdd_globals.realUptime,
 		"total_api_requests": ythdd_globals.apiRequests,
 		"failed_api_requests": ythdd_globals.apiFailedRequests,
 		"outside_api_requests": ythdd_globals.outsideApiHits,
@@ -29,6 +31,12 @@ def stats():
 	}
 	return 200, "OK", data_to_send
 
+def videoIdSanityCheck(videoId: str):
+	if len(videId) != 11:
+		incrementBadRequests()
+		return 400, f'error: bad request. wrong videoId: {videoId} is {len(videoId)} characters long, but should be 11.', []
+	# elif... check 
+
 def hot(data):
 	#print(data)
 	# if we are given not enough data to work with, return bad request.
@@ -119,7 +127,7 @@ def hot(data):
 
 		case _:
 			incrementBadRequests()
-			return notImplemented(data)
+			return notImplemented([data[1]]) # workaround before notImplemented is reworked
 
 def lookup(data):
 	match data[0]:

ythdd_db.py

@@ -1,14 +1,14 @@
 #!/usr/bin/python3
 from flask_sqlalchemy import SQLAlchemy
 import toml
-import ythdd_globals
+# import ythdd_globals
 
-database_file = ythdd_globals.config["general"]["db_file_path"]
+# database_file = ythdd_globals.config["general"]["db_file_path"]
 global db
 
 #db = SQLAlchemy()
 
-def initDB(app):
+def initDB(app, config):
 	db = SQLAlchemy(app)
 
 	class LocalUsers(db.Model):

ythdd_globals.py

@@ -1,39 +1,58 @@
 #!/usr/bin/python3
 import time, toml, os
 
-global starttime, apiRequests, apiFailedRequests, outsideApiHits, config, version, apiVersion, colors
+global starttime, apiRequests, apiFailedRequests, outsideApiHits, config, version, apiVersion, colors, realUptime
 
 class colors:
-	HEADER = '\033[95m'
+	HEADER    = '\033[95m'
-	OKBLUE = '\033[94m'
+	OKBLUE    = '\033[94m'
-	OKCYAN = '\033[96m'
+	OKCYAN    = '\033[96m'
-	OKGREEN = '\033[92m'
+	OKGREEN   = '\033[92m'
-	WARNING = '\033[93m'
+	WARNING   = '\033[93m'
-	FAIL = '\033[91m'
+	FAIL      = '\033[91m'
-	ENDC = '\033[0m'
+	ENDC      = '\033[0m'
-	BOLD = '\033[1m'
+	BOLD      = '\033[1m'
 	UNDERLINE = '\033[4m'
-	ENDL = '\n'
+	ENDL      = '\n'
 
 def notImplemented(name):
 	return 501, f"not recognised/implemented: {name}", []
 
-configfile = "config.toml" # TODO: implement a way to specify alternative config file path
+configfile = "config.toml"
 version = "0.0.1"
 apiVersion = "1"
+randomly_generated_passcode = 0
 
-# TODO: turn this into function, to make setting configfile with argparser (and effectively using a custom config file) possible
+def getConfig(configfile):
-if not os.path.exists(configfile):
+	
-	# use dummy default config, TODO: update this in the near future
+	# this function is responsible for an unwanted warning when using --help without config.toml
-	config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False}, 'api': {'api_key': 'CHANGEME', 'api_key_admin': str(int(time.time()*1337 % 899_999 + 100_000))}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}}
+	# for now it's not worth it to account for that edge case.
-	print(f"{colors.WARNING}WARNING{colors.ENDC}: Using default, baked in config data. {colors.ENDL}"
+	global randomly_generated_passcode
-		f"Consider copying and editing the provided example file ({colors.OKCYAN}config.default.toml{colors.ENDC}).")
+
-	print(f"{colors.WARNING}WARNING{colors.ENDC}: Default config populated with one-time, insecure pseudorandom admin API key: {colors.OKCYAN}{config['api']['api_key_admin']}{colors.ENDC}."
+	if not os.path.exists(configfile):
-		f" {colors.ENDL}You need to provide a config file for persistence!{colors.ENDL}")
+		dummy_config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False}, 'api': {'api_key': 'CHANGEME'}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}}
-	#with open(configfile, "w") as file:
+		# if a passcode has not been provided by the user (config file doesn't exist, and user didn't specify it using an argument)
-	#	file.write(toml.dumps(config))
+		print(f"{colors.WARNING}WARNING{colors.ENDC}: Using default, baked in config data. {colors.ENDL}"
-else:
+			  f"Consider copying and editing the provided example file ({colors.OKCYAN}config.default.toml{colors.ENDC}).")
-	config = toml.load(configfile)
+		if randomly_generated_passcode == 0:
+			# generate a pseudorandom one and use it in the temporary config
+			randomly_generated_passcode = str(int(time.time() * 1337 % 899_999 + 100_000))
+			
+			print(f"{colors.WARNING}WARNING{colors.ENDC}: Default config populated with one-time, insecure pseudorandom admin API key: {colors.OKCYAN}{randomly_generated_passcode}{colors.ENDC}."
+				  f" {colors.ENDL}The admin API key is not the Flask debugger PIN. You need to provide a config file for persistence!{colors.ENDL}")
+
+		dummy_config['api']['api_key_admin'] = randomly_generated_passcode
+		return dummy_config
+
+	else:
+		return toml.load(configfile)
+
+def setConfig(configfile):
+	global config
+	config = getConfig(configfile)
+
+#setConfig(configfile)
+config = {}
 
 def getUptime():
-	return int(time.time()) - starttime
+	return int(time.time()) - starttime