sherl/ythdd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commit files

Browse Source
This commit is contained in:
szakal
2024-09-25 12:22:14 +02:00
parent 61266427c6
commit 09b883c9eb
14 changed files with 1632 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
api.py Normal file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/python3
from flask_sqlalchemy import SQLAlchemy
from markupsafe import escape
import requests, json, ythdd_v
def index():
return json.dumps({status: 200, msg: "OK (%s, api v%s)" % (ythdd_v.version, ythdd_v.api_version)})

35
config.default.toml Normal file
View File

@@ -0,0 +1,35 @@
[general]
db_file_path = "/path/to/ythdd_db.sqlite" # Preferably stored on an SSD.
video_storage_directory_path = "/path/to/videos/" # Path to video vault.
is_proxied = false
[api]
# Leave empty to autogenerate api keys every launch (secure).
api_key = ""
api_key_admin = ""
[admin]
# List of users with admin priviledges.
admins = ["admin"]
[yt_dlp]
# ...
[postprocessing]
# Presets passed to yt-dlp. Reencoding uses ffmpeg to
presets = [
# First entry is the default.
# Naming convention:
# [N] - native video, as downloaded from youtube
# [R] - ffmpeg-reencoded file
# V+A - video with audio
{name="recommended: [N][<=720p] best V+A", format="bv[height<=720]+ba", reencode=""},
{name="[N][1080p] best V+A", format="bv[height=1080]+ba", reencode=""},
{name="[R][1080p] webm", format="bv[height=1080]+ba", reencode="webm"},
{name="[N][720p] best V+A", format="bv[height=720]+ba", reencode=""},
{name="[R][720p] webm", format="bv[height=720]+ba", reencode="webm"},
{name="[N][480p] best V+A", format="bv[height=480]+ba", reencode=""},
{name="[480p] VP9 webm/reencode", format="bv*[height=480][ext=webm]+ba/bv[height=480]+ba", reencode="webm"},
{name="[N][1080p] best video only", format="bv[height=1080]", reencode=""},
{name="[N][opus] best audio only", format="ba", reencode="opus"}
]

17
downloader.py Normal file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/python3
import yt_dlp, toml
ytdl_opts = {
#"format": "bv*[height<=720]+ba", # to be defined by the user
"getcomments": True,
"writeinfojson": True,
#"progress_hooks": my_hook,
"outtmpl": {
"default": "%(id)s.%(ext)s",
"chapter": "%(id)s.%(ext)s_%(section_number)03d_%(section_title)s.%(ext)s"
},
"simulate": True
}
# with yt_dlp.YoutubeDL(ytdl_opts) as ytdl:
# ytdl.download(['https://www.youtube.com/watch?v=SL-0B_I6WCM'])

1275
infotest.json Normal file
View File

File diff suppressed because it is too large Load Diff

6
static/css/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

23
static/html/untitled.html Normal file
View File

@@ -0,0 +1,23 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Bootstrap demo</title>
<link href="/static/css/bootstrap.min.css" rel="stylesheet">
<script src="/static/js/bootstrap.bundle.min.js"></script>
</head>
<body>
<div class="container">
<h1>Hello, world!</h1>
<div class="mb-3">
<label for="exampleFormControlInput1" class="form-label">Email address</label>
<input type="email" class="form-control" id="exampleFormControlInput1" placeholder="name@example.com">
</div>
<div class="mb-3">
<label for="exampleFormControlTextarea1" class="form-label">Example textarea</label>
<textarea class="form-control" id="exampleFormControlTextarea1" rows="3"></textarea>
</div>
</div>
</body>
</html>

7
static/js/bootstrap.bundle.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

14
views.py Normal file
View File

@@ -0,0 +1,14 @@
#!/usr/bin/python3
from flask import render_template
from flask_sqlalchemy import SQLAlchemy
from markupsafe import escape
import requests, json
def homepage():
return "homepage"
def home():
return "welcome home!"
def index():
return "index"

84
ythdd.py Normal file
View File

@@ -0,0 +1,84 @@
#!/usr/bin/python3
from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy
from markupsafe import escape
import requests, json, toml, time
import views, downloader, ythdd_v, ythdd_api, ythdd_globals
config = toml.load("config.toml")
global app
ythdd_globals.starttime = int(time.time())
ythdd_globals.apiRequests = 0
ythdd_globals.apiFailedRequests = 0
ythdd_globals.isProxied = config['general']['is_proxied']
ythdd_globals.outsideApiHits = 0
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{config["general"]["db_file_path"]}"
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.add_url_rule('/', view_func=views.index)
app.add_url_rule('/index.html', view_func=views.index)
app.add_url_rule('/home', view_func=views.home)
app.add_url_rule('/api/', view_func=ythdd_api.api_greeting)
app.add_url_rule('/api/<path:received_request>', view_func=ythdd_api.api_global_catchall)
db = SQLAlchemy(app)
class LocalUsers(db.Model):
uid = db.Column(db.Integer, primary_key=True)
nick = db.Column(db.String)
subscriptions = db.Column(db.JSON) # for RSS feed ???
playlists = db.Column(db.JSON)
watch_history = db.Column(db.JSON)
queue_history = db.Column(db.JSON)
class Videos(db.Model):
ytid = db.Column(db.String(11), primary_key=True)
archive_version = db.Column(db.Integer)
archive_type = db.Column(db.Integer) # 0 - V+A, 1 - V, 2 - A, 3 - other
archive_format = db.Column(db.Integer) # 0, 1, 2, 3... from config.toml[postprocessing][presets]
title = db.Column(db.String)
description = db.Column(db.String)
file_path = db.Column(db.String) # with extension
viewcount = db.Column(db.Integer)
duration = db.Column(db.Integer)
like_count = db.Column(db.Integer)
upload_date = db.Column(db.Integer)
archive_date = db.Column(db.Integer)
channel_id = db.Column(db.String(24)) # author, RemoteUser
requested_by = db.Column(db.Integer) # LocalUser
comments = db.Column(db.JSON) # TODO: watch frequent commenters
class Playlists(db.Model):
# For archiving videos' list from playlist.
ytid = db.Column(db.String(11), primary_key=True)
name = db.Column(db.String)
videos = db.Column(db.String)
modify_date = db.Column(db.Integer)
archive_date = db.Column(db.Integer)
channel_id = db.Column(db.String(24)) # author, RemoteUser
requested_by = db.Column(db.Integer) # LocalUser
class VideoQueue(db.Model):
uid = db.Column(db.Integer, primary_key=True)
ytid = db.Column(db.String(11))
requested_by = db.Column(db.String)
class RemoteUsers(db.Model):
channel_id = db.Column(db.String(24), primary_key=True) # possibly can change?
name = db.Column(db.String)
subscribers = db.Column(db.Integer)
avatar_path = db.Column(db.String)
badge = db.Column(db.Integer) # 0 - no badge, 1 - verified, 2 - music
with app.app_context():
db.create_all()
@app.route('/<string:val>', methods=['GET'])
def blank(val):
return f"{val}: not implemented in ythdd {ythdd_v.version}"
#users = db.session.query(LocalUsers).all()
#return users
if __name__ == "__main__":
#app.run(host="127.0.0.1", port=5000)
app.run(host="0.0.0.0", port=5000)

35
ythdd_api.py Normal file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/python3
from flask import Response, request
from markupsafe import escape
import requests, time, json
import ythdd_v, ythdd_api_v1, ythdd_globals
def api_greeting():
string = {'status': 200, 'msg': f"ok (ythdd {ythdd_v.version})", 'latest_api': f"v{ythdd_v.api_version}"}
string = json.dumps(string)
return Response(string, mimetype='application/json')
def api_global_catchall(received_request):
ythdd_globals.apiRequests += 1
if request.environ['REMOTE_ADDR'] != "127.0.0.1" or (ythdd_globals.isProxied and request.environ['X-Forwarded-For'] != "127.0.0.1"):
ythdd_globals.outsideApiHits += 1
request_list = received_request.split('/')
api_version = request_list[0]
if request_list[0] == 'v1':
# use v1 api
del request_list[0] # v1
# if list is empty, aka /api/v1/
if request_list == ['']:
return api_greeting()
try:
status, received, data = ythdd_api_v1.lookup(request_list)
except Exception as e:
ythdd_globals.apiFailedRequests += 1
status, received, data = 500, f"internal server error: call ended in failure: {e}", []
else:
ythdd_globals.apiFailedRequests += 1
status, received, data = 405, f'error: unsupported api version: "{request_list[0]}". try: "v{ythdd_v.api_version}".', []
response = {'status': status, 'msg': received, 'data': data}
return Response(json.dumps(response), mimetype='application/json', status=status)

88
ythdd_api_v1.py Normal file
View File

@@ -0,0 +1,88 @@
#!/usr/bin/python3
# API is expected to return:
# - HTTP status code,
# - human-readable status message,
# - json with appropriate data
import flask, json, time
import ythdd_globals, ythdd_extractor
#from flask_sqlalchemy import SQLAlchemy
#import ythdd_api_v1_stats, ythdd_api_v1_user, ythdd_api_v1_info, ythdd_api_v1_query, ythdd_api_v1_meta, ythdd_api_v1_admin
def incrementBadRequests():
ythdd_globals.apiFailedRequests += 1
def notImplemented(data):
return 501, f"not recognised/implemented: {data[0]}", []
def stub_hello():
return 200, 'hello from v1!', []
def stats():
data_to_send = {
"starttime": ythdd_globals.starttime,
"uptime": ythdd_globals.getUptime(),
"total_api_requests": ythdd_globals.apiRequests,
"failed_api_requests": ythdd_globals.apiFailedRequests,
"outside_api_requests": ythdd_globals.outsideApiHits,
"local_api_requests": ythdd_globals.apiRequests - ythdd_globals.outsideApiHits
}
return 200, "OK", data_to_send
def hot(data):
if len(data) <= 3:
incrementBadRequests()
return 400, f'error: bad request. supply required arguments.', []
comment_count = ""
if data[1] not in ("video", "channel", "handle", "playlist"):
return notImplemented(data)
if data[2] not in ("c", "nc", "lc"): # comments, no comments, limited comments
return notImplemented(data)
if data[2] == "lc":
if len(data) <= 4:
incrementBadRequests()
return 400, f'error: bad request. limited comments (lc) requires an extra argument specifying amount of comments.', []
try:
comment_count = str(int(data[3]))
except:
incrementBadRequests()
return 400, f'error: bad request. {data[3]} is not a number.', []
videoId = data[4]
else:
videoId = data[3]
try:
url_lookup = {'video': 'https://www.youtube.com/watch?v=', 'channel': 'https://www.youtube.com/channel/', 'handle': 'https://www.youtube.com/@', 'playlist': 'https://www.youtube.com/playlist?list='}
if data[2] == "nc":
getcomments = False
else:
getcomments = True
started = int(time.time())
extracted_dict = ythdd_extractor.extract(url_lookup[data[1]] + videoId, getcomments=getcomments, maxcomments=comment_count)
extracted_dict["took"] = int(time.time()) - started
return 200, "OK", extracted_dict
except Exception as e:
incrementBadRequests()
return 400, f'error: failed to get "{videoId}" ({data[2]}). {e}', []
def lookup(data):
match data[0]:
case 'stats':
return stats()
case 'hot': # retrieve live, uncached data
#print(data)
return hot(data)
case 'user':
return stub_hello()
#do_user()
case 'info':
return stub_hello()
#do_info()
case 'query':
return stub_hello()
case 'meta':
return stub_hello()
case 'admin':
# REQUIRE CREDENTIALS!
return stub_hello()
case _:
incrementBadRequests()
return notImplemented(data)

24
ythdd_extractor.py Normal file
View File

@@ -0,0 +1,24 @@
#!/usr/bin/python3
import yt_dlp, toml
ytdl_opts = {
#"format": "bv*[height<=720]+ba", # to be defined by the user
#"getcomments": True,
#"extractor_args": {"maxcomments": ...},
#"writeinfojson": True,
#"progress_hooks": my_hook,
"outtmpl": {
"default": "%(id)s.%(ext)s",
"chapter": "%(id)s.%(ext)s_%(section_number)03d_%(section_title)s.%(ext)s"
},
"simulate": True
}
def extract(url, getcomments=False, maxcomments=""):
if getcomments:
ytdl_opts['getcomments'] = True
if maxcomments:
ytdl_opts['extractor_args'] = {'youtube': {'max_comments': [maxcomments, "all", "all", "all"]}}
with yt_dlp.YoutubeDL(ytdl_opts) as ytdl:
result = ytdl.extract_info(url, download=False)
return result

13
ythdd_globals.py Normal file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/python3
import time
global starttime, apiRequests, apiFailedRequests, outsideApiHits
#def init():
# starttime = int(time.time())
#def notImplemented(data):
# return 501, f"not recognised/implemented: {data[0]}", []
def getUptime():
return int(time.time()) - starttime

4
ythdd_v.py Normal file
View File

@@ -0,0 +1,4 @@
#!/usr/bin/python3
version = "0.0.1"
api_version = "1"