From 1e4b05c33bcdcba6dc77fd3702335518118b8ee1 Mon Sep 17 00:00:00 2001 From: sherl Date: Thu, 12 Dec 2024 11:02:11 +0100 Subject: [PATCH] major rework of ythdd.py, new method for setting config in ythdd_globals.py - slightly modified config, api keys now have the value "CHANGEME" - requirements.txt has new dependency, flask apscheduler - ythdd.py has been reworked, support for argument parsing has been added, code is now split into functions - ythdd_api_v1.py features real uptime as well - ythdd_db.py is no longer dependent on ythdd_globals.py - ythdd_globals.py has a method for setting config and getting it from configfile variable --- .gitignore | 8 +++ config.default.toml | 5 +- requirements.txt | 3 ++ ythdd.py | 120 +++++++++++++++++++++++++++++++++----------- ythdd_api_v1.py | 12 ++++- ythdd_db.py | 6 +-- ythdd_globals.py | 67 ++++++++++++++++--------- 7 files changed, 161 insertions(+), 60 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a0afb61 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# dotfiles, catalogues +.venv +.git +__pycache__ +instance + +# config files +config.toml \ No newline at end of file diff --git a/config.default.toml b/config.default.toml index 14b0bde..1a66be2 100644 --- a/config.default.toml +++ b/config.default.toml @@ -4,9 +4,8 @@ video_storage_directory_path = "/path/to/videos/" # Path to video vault. is_proxied = false [api] -# Leave empty to autogenerate api keys every launch (secure). -api_key = "" -api_key_admin = "" +api_key = "" # Leave empty API key for public access to non-sensitive backend +api_key_admin = "CHANGEME" # Empty *admin* API key will autogenerate a random one every launch. [extractor] user-agent = "" # leave empty for default diff --git a/requirements.txt b/requirements.txt index 13861a6..bf5b261 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,6 @@ greenlet>=3.1.0 SQLAlchemy>=2.0.36.dev0 Flask-SQLAlchemy>=3.1.1 toml>=0.10.2 +Flask-APScheduler>=1.13.1 +requests>=2.32.3 +yt_dlp \ No newline at end of file diff --git a/ythdd.py b/ythdd.py index c041c8d..64c857f 100644 --- a/ythdd.py +++ b/ythdd.py @@ -2,29 +2,56 @@ from flask import Flask, render_template from flask_sqlalchemy import SQLAlchemy from markupsafe import escape -#from argparse import ArgumentParser -from ythdd_globals import config, colors +from argparse import ArgumentParser +from ythdd_globals import colors import requests, json, toml, time import views, downloader, ythdd_api, ythdd_globals, ythdd_db - -ythdd_globals.starttime = int(time.time()) -ythdd_globals.apiRequests = 0 -ythdd_globals.apiFailedRequests = 0 -ythdd_globals.isProxied = config['general']['is_proxied'] -ythdd_globals.outsideApiHits = 0 +from flask_apscheduler import APScheduler app = Flask(__name__) -app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{config['general']['db_file_path']}" -app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -app.add_url_rule('/', view_func=views.index) -app.add_url_rule('/index.html', view_func=views.index) -app.add_url_rule('/home', view_func=views.home) -app.add_url_rule('/api/', view_func=ythdd_api.api_greeting) -app.add_url_rule('/api/', view_func=ythdd_api.api_global_catchall) -db = ythdd_db.initDB(app) -with app.app_context(): - db.create_all() +def setup(): + + # sanity check: make sure config is set + # required to make `flask --app ythdd run --debug` work + global config + try: + if not config['general']: + ythdd_globals.setConfig(ythdd_globals.configfile) + config = ythdd_globals.config + except: + ythdd_globals.setConfig(ythdd_globals.configfile) + config = ythdd_globals.config + + # setting all the variables + ythdd_globals.starttime = int(time.time()) + ythdd_globals.realUptime = 0 + ythdd_globals.apiRequests = 0 + ythdd_globals.apiFailedRequests = 0 + ythdd_globals.isProxied = config['general']['is_proxied'] + ythdd_globals.outsideApiHits = 0 + + app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{config['general']['db_file_path']}" + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + app.add_url_rule('/', view_func=views.index) + app.add_url_rule('/index.html', view_func=views.index) + app.add_url_rule('/home', view_func=views.home) + app.add_url_rule('/api/', view_func=ythdd_api.api_greeting) + app.add_url_rule('/api/', view_func=ythdd_api.api_global_catchall) + db = ythdd_db.initDB(app, config) + + with app.app_context(): + db.create_all() + + # job scheduler for repetetive tasks + scheduler = APScheduler() + scheduler.add_job(func=every5seconds, trigger='interval', id='job', seconds=5) + scheduler.start() + +# gets called every 5 seconds +def every5seconds(): + # update the "real" uptime counter + ythdd_globals.realUptime += 5 @app.route('/', methods=['GET']) def blank(val): @@ -32,23 +59,60 @@ def blank(val): #users = db.session.query(LocalUsers).all() #return users -def main(): +def main(args): print(f"{colors.BOLD + colors.HEADER}Welcome to ythdd ({ythdd_globals.version})!{colors.ENDC}") print(f"To run in development mode (to see changes updated live), use: {colors.OKCYAN}flask --app ythdd run --debug{colors.ENDC}.") print("To run locally, use IP 127.0.0.1. To run on all interfaces, use 0.0.0.0.\n") - print("Enter hostname:port to run Flask app on [127.0.0.1:5000]:") try: - host_port = input('> ').split(':') - if host_port == ['']: - host_port = ['127.0.0.1', '5000'] # defaults + host = args.ip + port = args.port + if not host or not port: + raise Exception + except: + print("Enter hostname:port to run Flask app on [127.0.0.1:5000]:") + try: + host_port = input('> ').split(':') + if host_port == ['']: + host_port = ['127.0.0.1', '5000'] # defaults - except KeyboardInterrupt: - print(" ...exiting gracefully."), quit() # handle Ctrl+C + except KeyboardInterrupt: + print(" ...exiting gracefully."), quit() # handle Ctrl+C - app.run(host=host_port[0], port=int(host_port[1])) + host = host_port[0] + port = host_port[1] + + global config + try: + # if specified, use custom config file + ythdd_globals.configfile = args.config + ythdd_globals.setConfig(ythdd_globals.configfile) + + except: + # if not, use dummy file + ythdd_globals.configfile = "" + # but try to set the API secret if provided by the user + if args.secret: + ythdd_globals.randomly_generated_passcode = args.secret + ythdd_globals.setConfig(ythdd_globals.configfile) + + config = ythdd_globals.config + + setup() + app.run(host=host, port=int(port)) if __name__ == "__main__": #app.run(host="127.0.0.1", port=5000) #app.run(host="0.0.0.0", port=5000) - # TODO: add argument parser to load config files, etc. - main() + parser = ArgumentParser(description='A basic yt_dlp-based script for video download.') + + parser.add_argument("-i", "--ip", dest="ip", help="ip address/interface to bind to") + parser.add_argument("-p", "--port", dest="port", help="port on which the flask web backend should be ran") + parser.add_argument("-c", "--config", dest="config", help="path to TOML config file") + parser.add_argument("-s", "--secret", dest="secret", help="admin's secret passcode for sensitive API access") + + args = parser.parse_args() + + main(args) + +else: + setup() \ No newline at end of file diff --git a/ythdd_api_v1.py b/ythdd_api_v1.py index 4024074..1555a90 100644 --- a/ythdd_api_v1.py +++ b/ythdd_api_v1.py @@ -20,8 +20,10 @@ def stub_hello(): def stats(): data_to_send = { - "starttime": ythdd_globals.starttime, + # TODO: include yt-dlp version + "start_time": ythdd_globals.starttime, "uptime": ythdd_globals.getUptime(), + "real_uptime": ythdd_globals.realUptime, "total_api_requests": ythdd_globals.apiRequests, "failed_api_requests": ythdd_globals.apiFailedRequests, "outside_api_requests": ythdd_globals.outsideApiHits, @@ -29,6 +31,12 @@ def stats(): } return 200, "OK", data_to_send +def videoIdSanityCheck(videoId: str): + if len(videId) != 11: + incrementBadRequests() + return 400, f'error: bad request. wrong videoId: {videoId} is {len(videoId)} characters long, but should be 11.', [] + # elif... check + def hot(data): #print(data) # if we are given not enough data to work with, return bad request. @@ -119,7 +127,7 @@ def hot(data): case _: incrementBadRequests() - return notImplemented(data) + return notImplemented([data[1]]) # workaround before notImplemented is reworked def lookup(data): match data[0]: diff --git a/ythdd_db.py b/ythdd_db.py index a637080..e4bef1a 100644 --- a/ythdd_db.py +++ b/ythdd_db.py @@ -1,14 +1,14 @@ #!/usr/bin/python3 from flask_sqlalchemy import SQLAlchemy import toml -import ythdd_globals +# import ythdd_globals -database_file = ythdd_globals.config["general"]["db_file_path"] +# database_file = ythdd_globals.config["general"]["db_file_path"] global db #db = SQLAlchemy() -def initDB(app): +def initDB(app, config): db = SQLAlchemy(app) class LocalUsers(db.Model): diff --git a/ythdd_globals.py b/ythdd_globals.py index 96b3520..2e12cc3 100644 --- a/ythdd_globals.py +++ b/ythdd_globals.py @@ -1,39 +1,58 @@ #!/usr/bin/python3 import time, toml, os -global starttime, apiRequests, apiFailedRequests, outsideApiHits, config, version, apiVersion, colors +global starttime, apiRequests, apiFailedRequests, outsideApiHits, config, version, apiVersion, colors, realUptime class colors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' UNDERLINE = '\033[4m' - ENDL = '\n' + ENDL = '\n' def notImplemented(name): return 501, f"not recognised/implemented: {name}", [] -configfile = "config.toml" # TODO: implement a way to specify alternative config file path +configfile = "config.toml" version = "0.0.1" apiVersion = "1" +randomly_generated_passcode = 0 -# TODO: turn this into function, to make setting configfile with argparser (and effectively using a custom config file) possible -if not os.path.exists(configfile): - # use dummy default config, TODO: update this in the near future - config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False}, 'api': {'api_key': 'CHANGEME', 'api_key_admin': str(int(time.time()*1337 % 899_999 + 100_000))}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}} - print(f"{colors.WARNING}WARNING{colors.ENDC}: Using default, baked in config data. {colors.ENDL}" - f"Consider copying and editing the provided example file ({colors.OKCYAN}config.default.toml{colors.ENDC}).") - print(f"{colors.WARNING}WARNING{colors.ENDC}: Default config populated with one-time, insecure pseudorandom admin API key: {colors.OKCYAN}{config['api']['api_key_admin']}{colors.ENDC}." - f" {colors.ENDL}You need to provide a config file for persistence!{colors.ENDL}") - #with open(configfile, "w") as file: - # file.write(toml.dumps(config)) -else: - config = toml.load(configfile) +def getConfig(configfile): + + # this function is responsible for an unwanted warning when using --help without config.toml + # for now it's not worth it to account for that edge case. + global randomly_generated_passcode + + if not os.path.exists(configfile): + dummy_config = {'general': {'db_file_path': 'ythdd_db.sqlite', 'video_storage_directory_path': 'videos/', 'is_proxied': False}, 'api': {'api_key': 'CHANGEME'}, 'extractor': {'user-agent': '', 'cookies_path': ''}, 'admin': {'admins': ['admin']}, 'yt_dlp': {}, 'postprocessing': {'presets': [{'name': 'recommended: [N][<=720p] best V+A', 'format': 'bv[height<=720]+ba', 'reencode': ''}, {'name': '[N][1080p] best V+A', 'format': 'bv[height=1080]+ba', 'reencode': ''}, {'name': '[R][1080p] webm', 'format': 'bv[height=1080]+ba', 'reencode': 'webm'}, {'name': '[N][720p] best V+A', 'format': 'bv[height=720]+ba', 'reencode': ''}, {'name': '[R][720p] webm', 'format': 'bv[height=720]+ba', 'reencode': 'webm'}, {'name': '[N][480p] best V+A', 'format': 'bv[height=480]+ba', 'reencode': ''}, {'name': '[480p] VP9 webm/reencode', 'format': 'bv*[height=480][ext=webm]+ba/bv[height=480]+ba', 'reencode': 'webm'}, {'name': '[N][1080p] best video only', 'format': 'bv[height=1080]', 'reencode': ''}, {'name': '[N][opus] best audio only', 'format': 'ba', 'reencode': 'opus'}]}} + # if a passcode has not been provided by the user (config file doesn't exist, and user didn't specify it using an argument) + print(f"{colors.WARNING}WARNING{colors.ENDC}: Using default, baked in config data. {colors.ENDL}" + f"Consider copying and editing the provided example file ({colors.OKCYAN}config.default.toml{colors.ENDC}).") + if randomly_generated_passcode == 0: + # generate a pseudorandom one and use it in the temporary config + randomly_generated_passcode = str(int(time.time() * 1337 % 899_999 + 100_000)) + + print(f"{colors.WARNING}WARNING{colors.ENDC}: Default config populated with one-time, insecure pseudorandom admin API key: {colors.OKCYAN}{randomly_generated_passcode}{colors.ENDC}." + f" {colors.ENDL}The admin API key is not the Flask debugger PIN. You need to provide a config file for persistence!{colors.ENDL}") + + dummy_config['api']['api_key_admin'] = randomly_generated_passcode + return dummy_config + + else: + return toml.load(configfile) + +def setConfig(configfile): + global config + config = getConfig(configfile) + +#setConfig(configfile) +config = {} def getUptime(): - return int(time.time()) - starttime + return int(time.time()) - starttime \ No newline at end of file