diff --git a/Program.cs b/Program.cs index 77dec05..43e044d 100644 --- a/Program.cs +++ b/Program.cs @@ -70,6 +70,7 @@ builder.Services.AddHttpLogging(); var app = builder.Build(); bool shutdown = false; +bool scanNeeded = false; if (args.FirstOrDefault() is not null) { // Handle CLI if arguments have been passed. @@ -89,6 +90,7 @@ using (IServiceScope scope = app.Services.CreateScope()) GeneralUseHelpers guhf = scope.ServiceProvider.GetRequiredService(); Seeder seeder = new(db, guhf); shutdown = seeder.Seed(); + if (!shutdown) scanNeeded = await seeder.ScanPrefetchAsync(); } if (shutdown) return; diff --git a/Shadow.csproj b/Shadow.csproj index c1a347f..d5fcfe7 100644 --- a/Shadow.csproj +++ b/Shadow.csproj @@ -19,15 +19,15 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive + + all + runtime; build; native; contentfiles; analyzers; buildtransitive - - all - runtime; build; native; contentfiles; analyzers; buildtransitive + + all + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -43,11 +43,10 @@ - - - - - + + + + diff --git a/Tools/GeneralUseHelperFunctions.cs b/Tools/GeneralUseHelperFunctions.cs index 56d755f..3e25a72 100644 --- a/Tools/GeneralUseHelperFunctions.cs +++ b/Tools/GeneralUseHelperFunctions.cs @@ -5,10 +5,10 @@ using System.Text; namespace Shadow.Tools; -public class GeneralUseHelpers(ApplicationDbContext db, IConfiguration appsettings) +public class GeneralUseHelpers(ApplicationDbContext? db = null, IConfiguration? appsettings = null) { - private readonly ApplicationDbContext _db = db; - private readonly IConfiguration _appsettings = appsettings; + private readonly ApplicationDbContext? _db = db; + private readonly IConfiguration? _appsettings = appsettings; //async public Task GetUserFromEmail(string email) @@ -27,4 +27,23 @@ public class GeneralUseHelpers(ApplicationDbContext db, IConfiguration appsettin // } //} + /// + /// Quick and dirty Dictionary<string, string> to JSON serializer + /// + /// Dictionary with keypair of two strings + /// Minified JSON + public static string DictAsJson(Dictionary dict) + { + string resultJson = String.Empty; + + foreach (string key in dict.Keys) + { + string cleanKey = key.Replace("\"", "\\\""); // "a"b" -> "a\"b" + string cleanValue = dict[key].Replace("\"", "\\\""); + resultJson += $"\"{cleanKey}\": \"{cleanValue}\", " // "key": "val", + .Replace(@"\", @"\\"); // a\b -> a\\b + } + + return "{" + resultJson[..^2] + "}"; + } } diff --git a/Tools/LibraryWatcher.cs b/Tools/LibraryWatcher.cs new file mode 100644 index 0000000..1168600 --- /dev/null +++ b/Tools/LibraryWatcher.cs @@ -0,0 +1,91 @@ +using Shadow.Data; +using Shadow.Entities; +using System; +using System.IO; + +namespace Shadow.Tools; +public class LibraryWatcher(string watchPath, string[] excludedPaths, ApplicationDbContext dbContext) +{ + private readonly string libraryPath = watchPath; + private readonly string[] excludedPaths = excludedPaths; + private readonly ApplicationDbContext db = dbContext; + private readonly GeneralUseHelpers guhf = new(); + + /// + /// Returns a sorted list of paths to all files in a directory, recursively. + /// + /// Path to directory + /// Sorted list of filepaths + public async Task> GetFilesRecursivelyAsync(string directory) + { + string[] allowedExtensions = [".flac", ".m4a", ".mp3", ".ogg", ".wav"]; + try + { + List files = + Directory.GetFiles(directory, "*", SearchOption.AllDirectories) + .Where(file => allowedExtensions.Any(file.ToLower().EndsWith)) + .ToList(); + + files.Sort(); + return files; + } + catch (DirectoryNotFoundException) + { + Console.WriteLine($"[Error] Directory \"{directory}\" does not exist!\n" + + " Please create it manually, or use `Shadow setupWizard`."); + throw new DirectoryNotFoundException(); + } + + } + + /// + /// Return all multimedia content inside of library + /// + /// List of multimedia filepaths + public async Task> GetAllMultimediaAsync() + { + + // List files in cache + // Note: currently, the only excluded path from scanning is the thumbnail cache. + // This might change in the future. + List cacheFiles = await GetFilesRecursivelyAsync(excludedPaths[0]); + + // List files in library excluding cache + List libraryContent = await GetFilesRecursivelyAsync(libraryPath); + List libraryMultimedia = libraryContent.Except(cacheFiles).ToList(); + + return libraryMultimedia; + } + + /// + /// Scan the library in its entirety + /// + /// + public async Task> PerformFullScanAsync() + { + Console.WriteLine("Performing full library scan..."); + + List multimedia = await GetAllMultimediaAsync(); + foreach (string filepath in multimedia) + { + Console.WriteLine(filepath); + Dictionary fileInfo = await MetadataExtractor.ExtractAsync(filepath); + + // Pretend we are doing parsing here... + Console.WriteLine(GeneralUseHelpers.DictAsJson(fileInfo)); + MediaParser.CreateSong(db, fileInfo); + } + + Console.WriteLine($"Full scan complete! Processed {multimedia.Count} files."); + + // Update state inside of DB + string currentLibraryState = MetadataExtractor.GetStringMD5(string.Join("\n", multimedia)); + Global lastLibraryState = db.Globals.FirstOrDefault(g => g.Key == "libraryState") + ?? new() { Key = "libraryState"}; + lastLibraryState.Value = currentLibraryState; + db.Update(lastLibraryState); + await db.SaveChangesAsync(); + + return multimedia; + } +} diff --git a/Tools/MediaParser.cs b/Tools/MediaParser.cs new file mode 100644 index 0000000..daf980b --- /dev/null +++ b/Tools/MediaParser.cs @@ -0,0 +1,126 @@ +using Microsoft.VisualBasic.FileIO; +using Shadow.Data; +using Shadow.Entities; +using System.Security.Cryptography; + +namespace Shadow.Tools; + +public static class MediaParser +{ + /// + /// Generate a random hex string (with length 32 by default) + /// + /// Optional: hexstring length + /// A hexstring of given length + public static string HexStr(int length = 32) + { + return RandomNumberGenerator.GetHexString(length).ToLower(); + } + + /// + /// Get metadata content opportunistically + /// + /// Dictionary to search in + /// Keywords to search for + /// Retrieved value (string) on success, otherwise null + public static string? GetAny(Dictionary metadata, List searchStrings) + { + foreach (string searchString in searchStrings) + { + if (metadata.TryGetValue(searchString, out string? value) && !string.IsNullOrEmpty(value)) + return value; + } + return null; + } + + /// + /// Map exiftool metadata to Song entity. + /// + /// Database context + /// ExifTool metadata + /// New Song entity, or null if song already exists in db + public static Song? CreateSong(ApplicationDbContext db, Dictionary exif) + { + + // First of all, check if song already exists in db + string uri = GetAny(exif, ["_shadow:fileHash"]) + ?? throw new Exception("Fatal error: could not get file hash!"); + if (db.Songs.FirstOrDefault(s => s.Uri == uri) != null) return null; + + // If not, extract exif data + string title = GetAny(exif, [ + "ItemList:Title", // iTunes m4a + "ItemList:SortName", // iTunes m4a + "Vorbis:Title", // Bandcamp ogg + "ID3v2_3:Title", // Generic mp3/wav ID3 v2.3.0 + ]) + ?? Path.Combine(exif["System:Directory"], exif["System:FileName"]); + string filepath = Path.GetFullPath( + Path.Combine(exif["System:Directory"], exif["System:FileName"]) + ); // TODO: bulletproof this + string filetype = exif["File:FileType"].ToLower(); + + // Album/artist related + string artistName = GetAny(exif, [ + "ItemList:Artist", // iTunes m4a + "ItemList:AlbumArtist", // iTunes m4a + "Vorbis:Artist", // Bandcamp m4a + "Vorbis:Albumartist", // Bandcamp m4a + "ID3v2_3:Artist", // Generic mp3/wav ID3 v2.3.0 + ]) ?? "[Unknown Artist]"; // this is a weak line of defense against deliberately crafted + + string albumName = GetAny(exif, [ + "ItemList:Album", // iTunes m4a + "Vorbis:Album", // Bandcamp m4a + "ID3v2_3:Album", // Generic mp3/wav ID3 v2.3.0 + ]) ?? "[Unknown Album]"; // again, weak line of defense + + // Try to find relevant artists and albums + Artist artist = db.Artists + .FirstOrDefault(a => a.NormalizedName == artistName.ToLower()) + ?? new Artist + { + Name = artistName, + NormalizedName = artistName.ToLower() + }; + + Album album = db.Albums + .FirstOrDefault(a => a.Name == albumName && a.Artist == artist) + ?? new Album + { + Name = albumName, + Uri = HexStr(), + Artist = artist + }; + + Song song = new() + { + Title = title, + Uri = uri, + Filepath = filepath, + Filetype = filetype, + Album = album, + Artist = artist + }; + + try + { + // Is Update() safe here? + db.Artists.Update(artist); + db.Albums.Update(album); + db.Songs.Update(song); + artist.Albums.Add(album); + artist.Songs.Add(song); + db.SaveChanges(); + } + catch (Exception e) + { + Console.WriteLine("[Error: MediaParser] Failed to extract metadata from {filepath}:\n" + + $"{e}"); + } + + return song; + } + + +} diff --git a/Tools/MetadataExtractor.cs b/Tools/MetadataExtractor.cs new file mode 100644 index 0000000..ff37b4d --- /dev/null +++ b/Tools/MetadataExtractor.cs @@ -0,0 +1,67 @@ +using SharpExifTool; +using System.Security.Cryptography; +namespace Shadow.Tools; + +public static class MetadataExtractor +{ + private readonly static ExifTool exifTool = new(); + private readonly static GeneralUseHelpers guhf = new(); + public async static Task> ExtractAsync(string fullPath) + { + // Get all relevant metadata + Dictionary fileMetadata = new(await + exifTool.ExtractAllMetadataAsync(fullPath)); + + // Add in a MD5 hint + string md5sum = await GetFileMD5Async(fullPath); + fileMetadata?.Add("_shadow:fileHash", md5sum); + + return fileMetadata ?? []; + } + + /// + /// Compute MD5 checksum of a file + /// + /// Input file absolute path + /// MD5 hexstring + public static async Task GetFileMD5Async(string fullPath) + { + string fallbackValue = String.Empty; + + try + { + if (File.Exists(fullPath)) + using (MD5 md5 = MD5.Create()) + { + using (FileStream stream = File.OpenRead(fullPath)) + { + byte[] hashBytes = await md5.ComputeHashAsync(stream); + string hashString = Convert.ToHexStringLower(hashBytes); + return hashString; + } + } + else return fallbackValue; + } + catch + { + return fallbackValue; + } + } + + /// + /// Compute MD5 checksum of a string + /// + /// Input string for the MD5 hash + /// MD5 hexstring + public static string GetStringMD5(string input) + { + // https://stackoverflow.com/a/24031467 + byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input); + byte[] hashBytes = MD5.HashData(inputBytes); + string hashString = Convert.ToHexStringLower(hashBytes); + + return hashString; + } + + +} diff --git a/Tools/Seeder.cs b/Tools/Seeder.cs index 026993e..0e477c0 100644 --- a/Tools/Seeder.cs +++ b/Tools/Seeder.cs @@ -177,4 +177,41 @@ public class Seeder return migrationSuccess; } + /// + /// Check if the library needs a full rescan + /// + /// True if full rescan is needed + /// Thrown when either last library state, path to music library or cache is unknown + public async Task ScanPrefetchAsync() + { + bool scanNecessary = false; + Global? lastLibraryState = await db.Globals.FirstOrDefaultAsync(g => g.Key == "libraryState"); + Global? libraryPath = await db.Globals.FirstOrDefaultAsync(g => g.Key == "musicLibraryPath"); + Global? cachePath = await db.Globals.FirstOrDefaultAsync(g => g.Key == "musicThumbnailPath"); + + if (libraryPath is null || cachePath is null || lastLibraryState is null) + { + throw new MissingFieldException("[Error] Missing libraryState, musicLibraryPath, musicThumbnailPath. Please rerun the setup wizard with `Shadow setupWizard`."); + } + LibraryWatcher lw = new(libraryPath.Value!, [cachePath.Value!], db); + + // Get library contents + List currentLibraryStateList = await lw.GetAllMultimediaAsync(); + // Compute their hash + string currentLibraryStateString = string.Join("\n", currentLibraryStateList); + string currentLibraryState = MetadataExtractor.GetStringMD5(currentLibraryStateString); + + // Compare against last known library state + if (currentLibraryState != lastLibraryState.Value) + scanNecessary = true; + + // The contents changed? Initiate a full rescan, then call LibraryWatcher. + if (scanNecessary) + await lw.PerformFullScanAsync(); + // State seems identical? Launch just the LibraryWatcher. + // TODO: lw.Watch()... + + return scanNecessary; + + } }