diff --git a/.github/docker/Dockerfile.container-scan b/.github/docker/Dockerfile.container-scan index 9a490b3be..fb00ca45b 100644 --- a/.github/docker/Dockerfile.container-scan +++ b/.github/docker/Dockerfile.container-scan @@ -1,4 +1,5 @@ -FROM mcr.microsoft.com/dotnet/aspnet:10.0.4 +ARG DOTNET_ASPNET_IMAGE=mcr.microsoft.com/dotnet/aspnet:10.0 +FROM ${DOTNET_ASPNET_IMAGE} WORKDIR /app diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 086c677e5..c19f767e0 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -74,7 +74,8 @@ jobs: Jwt__Audience: "test" security__secretKey: "testsecretkeytestsecretkeytestsecretkey" QuartzDisabled: "true" - run: dotnet test --no-build --verbosity normal --collect:"XPlat Code Coverage" --settings coverage.runsettings --results-directory coverage/results + # Parallel solution-level VSTest with coverage can return a non-zero exit after successful test summaries. + run: dotnet test --no-build -m:1 --verbosity normal --collect:"XPlat Code Coverage" --settings coverage.runsettings --results-directory coverage/results - name: Generate coverage report run: | dotnet tool install -g dotnet-reportgenerator-globaltool diff --git a/.github/workflows/sca-container-scan.yml b/.github/workflows/sca-container-scan.yml index 7ff0a4d2c..aab47bf01 100644 --- a/.github/workflows/sca-container-scan.yml +++ b/.github/workflows/sca-container-scan.yml @@ -45,6 +45,8 @@ jobs: permissions: contents: read security-events: write + env: + SCAN_BASE_IMAGE: mcr.microsoft.com/dotnet/aspnet:10.0 steps: - name: Checkout repository @@ -58,18 +60,47 @@ jobs: - name: Publish application for scanning run: dotnet publish src/Melodee.Blazor/Melodee.Blazor.csproj -c Release -o scan/publish --self-contained false -p:PublishTrimmed=false - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Pull Docker base image for scanning + run: | + set -euo pipefail + + for attempt in 1 2 3 4 5; do + if docker pull "$SCAN_BASE_IMAGE"; then + exit 0 + fi + + delay=$((attempt * 20)) + echo "Base image pull failed on attempt ${attempt}; retrying in ${delay}s..." + sleep "$delay" + done + + docker pull "$SCAN_BASE_IMAGE" - name: Build Docker image for scanning - uses: docker/build-push-action@v5 - with: - context: . - file: .github/docker/Dockerfile.container-scan - load: true - tags: melodee:latest - cache-from: type=gha - cache-to: type=gha,mode=max + run: | + set -euo pipefail + + for attempt in 1 2 3; do + if docker build \ + --pull=false \ + --build-arg DOTNET_ASPNET_IMAGE="$SCAN_BASE_IMAGE" \ + -f .github/docker/Dockerfile.container-scan \ + -t melodee:latest \ + .; then + exit 0 + fi + + delay=$((attempt * 20)) + echo "Docker image build failed on attempt ${attempt}; retrying in ${delay}s..." + sleep "$delay" + done + + docker build \ + --pull=false \ + --build-arg DOTNET_ASPNET_IMAGE="$SCAN_BASE_IMAGE" \ + -f .github/docker/Dockerfile.container-scan \ + -t melodee:latest \ + . - name: Run Trivy Vulnerability Scanner uses: aquasecurity/trivy-action@master diff --git a/Directory.Build.props b/Directory.Build.props index 64259da25..9949ef02c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,9 +1,5 @@ true - - $(NoWarn);NU1902 diff --git a/Directory.Packages.props b/Directory.Packages.props index f0ca61a7a..28cdbdc01 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,7 +7,6 @@ - @@ -18,106 +17,106 @@ - - + + + - + - - - + - - - - - - - - - - - - + + + + + + + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + - - + + - + - + - + - + - + - - + + - - + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - \ No newline at end of file + diff --git a/benchmarks/Melodee.Benchmarks/Melodee.Benchmarks.csproj b/benchmarks/Melodee.Benchmarks/Melodee.Benchmarks.csproj index 96cfe685e..2ef396185 100644 --- a/benchmarks/Melodee.Benchmarks/Melodee.Benchmarks.csproj +++ b/benchmarks/Melodee.Benchmarks/Melodee.Benchmarks.csproj @@ -15,7 +15,11 @@ + + + all + diff --git a/benchmarks/Melodee.Benchmarks/MusicBrainzImportBenchmarks.cs b/benchmarks/Melodee.Benchmarks/MusicBrainzImportBenchmarks.cs index 28fd5be05..3b50983b1 100644 --- a/benchmarks/Melodee.Benchmarks/MusicBrainzImportBenchmarks.cs +++ b/benchmarks/Melodee.Benchmarks/MusicBrainzImportBenchmarks.cs @@ -81,9 +81,7 @@ public async Task ImportWithDefaultSettings() await context.Database.EnsureCreatedAsync(); var importer = new DecentDBStreamingMusicBrainzImporter(_logger); - await importer.ImportAsync(context, _testDataPath, null, CancellationToken.None); - - var count = await context.Artists.CountAsync(); + var summary = await importer.ImportAsync(context, _testDataPath, null, CancellationToken.None); try { @@ -91,7 +89,7 @@ public async Task ImportWithDefaultSettings() } catch { } - return count; + return summary.Artists; } [Benchmark] @@ -107,9 +105,7 @@ public async Task ImportWithLargerDataset() await context.Database.EnsureCreatedAsync(); var importer = new DecentDBStreamingMusicBrainzImporter(_logger); - await importer.ImportAsync(context, _testDataPath, null, CancellationToken.None); - - var count = await context.Artists.CountAsync(); + var summary = await importer.ImportAsync(context, _testDataPath, null, CancellationToken.None); try { @@ -117,7 +113,7 @@ public async Task ImportWithLargerDataset() } catch { } - return count; + return summary.Artists; } [Benchmark] @@ -133,18 +129,16 @@ public async Task ImportWithProgressTracking() await context.Database.EnsureCreatedAsync(); var importer = new DecentDBStreamingMusicBrainzImporter(_logger); - await importer.ImportAsync(context, _testDataPath, + var summary = await importer.ImportAsync(context, _testDataPath, (phase, current, total, msg) => { }, CancellationToken.None); - var count = await context.Artists.CountAsync(); - try { File.Delete(dbFile); } catch { } - return count; + return summary.Artists; } } diff --git a/benchmarks/Melodee.Benchmarks/MusicBrainzImportProbe.cs b/benchmarks/Melodee.Benchmarks/MusicBrainzImportProbe.cs new file mode 100644 index 000000000..a00781c06 --- /dev/null +++ b/benchmarks/Melodee.Benchmarks/MusicBrainzImportProbe.cs @@ -0,0 +1,353 @@ +using System.Diagnostics; +using System.Text.Json; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using Microsoft.EntityFrameworkCore; +using Serilog; +using Serilog.Events; + +namespace Melodee.Benchmarks; + +internal static class MusicBrainzImportProbe +{ + public static async Task RunAsync(string[] args) + { + var options = MusicBrainzProbeOptions.Parse(args); + try + { + var storagePath = options.RequireString("storage"); + var databasePath = options.RequireString("db"); + var outputPath = options.GetString("output"); + var cleanTarget = options.GetBool("clean"); + var sampleIntervalMs = Math.Max(250, options.GetInt32("sample-interval-ms", 5000)); + + if (!Directory.Exists(Path.Combine(storagePath, "staging", "mbdump"))) + { + throw new DirectoryNotFoundException( + $"MusicBrainz staging dump was not found under {Path.Combine(storagePath, "staging", "mbdump")}."); + } + + if (cleanTarget) + { + DeleteDatabaseFiles(databasePath); + } + + var report = await RunProbeAsync( + storagePath, + databasePath, + sampleIntervalMs, + CancellationToken.None) + .ConfigureAwait(false); + await WriteReportAsync(report, outputPath, CancellationToken.None).ConfigureAwait(false); + return report.Success ? 0 : 1; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.Message); + return 1; + } + } + + private static async Task RunProbeAsync( + string storagePath, + string databasePath, + int sampleIntervalMs, + CancellationToken cancellationToken) + { + var startedAt = DateTimeOffset.UtcNow; + var stopwatch = Stopwatch.StartNew(); + var samples = new List(); + var phaseTracker = new ImportPhaseTracker(); + DecentDBMusicBrainzImportSummary? summary = null; + string? errorMessage = null; + var success = false; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ConsoleCancelEventHandler cancelHandler = (_, eventArgs) => + { + eventArgs.Cancel = true; + linkedCts.Cancel(); + }; + Console.CancelKeyPress += cancelHandler; + + var sampler = Task.Run( + () => SampleProcessAsync(databasePath, sampleIntervalMs, samples, linkedCts.Token), + linkedCts.Token); + + try + { + var logger = new LoggerConfiguration() + .MinimumLevel.Is(LogEventLevel.Information) + .WriteTo.Console() + .CreateLogger(); + + var dbOptions = new DbContextOptionsBuilder() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + await using var context = new MusicBrainzDbContext(dbOptions); + await context.Database.EnsureCreatedAsync(linkedCts.Token).ConfigureAwait(false); + + var importer = new DecentDBStreamingMusicBrainzImporter(logger); + summary = await importer.ImportAsync( + context, + storagePath, + phaseTracker.Progress, + linkedCts.Token) + .ConfigureAwait(false); + success = true; + } + catch (OperationCanceledException) + { + errorMessage = "Import probe was cancelled."; + } + catch (Exception ex) + { + errorMessage = ex.ToString(); + } + finally + { + stopwatch.Stop(); + phaseTracker.Complete(); + linkedCts.Cancel(); + Console.CancelKeyPress -= cancelHandler; + + try + { + await sampler.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + } + + samples.Add(CaptureSample(databasePath)); + + return new MusicBrainzImportProbeReport( + "musicbrainz-import-probe", + storagePath, + databasePath, + startedAt, + DateTimeOffset.UtcNow, + stopwatch.Elapsed.TotalMilliseconds, + success, + errorMessage, + summary, + phaseTracker.Phases, + samples); + } + + private static async Task SampleProcessAsync( + string databasePath, + int sampleIntervalMs, + List samples, + CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + lock (samples) + { + samples.Add(CaptureSample(databasePath)); + } + + await Task.Delay(sampleIntervalMs, cancellationToken) + .ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } + } + + private static ImportProbeSample CaptureSample(string databasePath) + { + var process = Process.GetCurrentProcess(); + var linuxStatus = LinuxProcessStatus.ReadCurrent(); + return new ImportProbeSample( + DateTimeOffset.UtcNow, + process.WorkingSet64, + process.PeakWorkingSet64, + process.TotalProcessorTime.TotalMilliseconds, + linuxStatus.VmRssBytes, + linuxStatus.VmHwmBytes, + GetFileLength(databasePath), + GetFileLength($"{databasePath}.wal"), + GetFileLength($"{databasePath}-wal")); + } + + private static long GetFileLength(string path) => + File.Exists(path) ? new FileInfo(path).Length : 0; + + private static void DeleteDatabaseFiles(string databasePath) + { + foreach (var path in new[] + { + databasePath, + $"{databasePath}.wal", + $"{databasePath}.shm", + $"{databasePath}-wal", + $"{databasePath}-shm" + }) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } + + private static async Task WriteReportAsync( + MusicBrainzImportProbeReport report, + string? outputPath, + CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true }); + if (string.IsNullOrWhiteSpace(outputPath)) + { + Console.WriteLine(json); + return; + } + + var outputDirectory = Path.GetDirectoryName(outputPath); + if (!string.IsNullOrWhiteSpace(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + + await File.WriteAllTextAsync(outputPath, json, cancellationToken).ConfigureAwait(false); + Console.WriteLine($"Wrote MusicBrainz import probe report to {outputPath}"); + } + + private sealed class ImportPhaseTracker + { + private readonly object gate = new(); + private readonly List phases = []; + private string? currentPhase; + private DateTimeOffset currentStartedAt; + private string? currentMessage; + private int current; + private int total; + + public IReadOnlyList Phases + { + get + { + lock (gate) + { + return phases.ToArray(); + } + } + } + + public void Progress(string phase, int currentValue, int totalValue, string? message) + { + lock (gate) + { + if (!string.Equals(currentPhase, phase, StringComparison.Ordinal)) + { + FinishCurrentPhase(); + currentPhase = phase; + currentStartedAt = DateTimeOffset.UtcNow; + } + + current = currentValue; + total = totalValue; + currentMessage = message; + } + } + + public void Complete() + { + lock (gate) + { + FinishCurrentPhase(); + } + } + + private void FinishCurrentPhase() + { + if (currentPhase is null) + { + return; + } + + var finishedAt = DateTimeOffset.UtcNow; + phases.Add(new ImportProbePhase( + currentPhase, + currentStartedAt, + finishedAt, + (finishedAt - currentStartedAt).TotalMilliseconds, + current, + total, + currentMessage)); + currentPhase = null; + currentMessage = null; + current = 0; + total = 0; + } + } + + private readonly record struct LinuxProcessStatus(long? VmRssBytes, long? VmHwmBytes) + { + public static LinuxProcessStatus ReadCurrent() + { + const string statusPath = "/proc/self/status"; + if (!File.Exists(statusPath)) + { + return new LinuxProcessStatus(null, null); + } + + long? vmRss = null; + long? vmHwm = null; + foreach (var line in File.ReadLines(statusPath)) + { + if (line.StartsWith("VmRSS:", StringComparison.Ordinal)) + { + vmRss = ParseKilobyteLine(line); + } + else if (line.StartsWith("VmHWM:", StringComparison.Ordinal)) + { + vmHwm = ParseKilobyteLine(line); + } + } + + return new LinuxProcessStatus(vmRss, vmHwm); + } + + private static long? ParseKilobyteLine(string line) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + return parts.Length >= 2 && long.TryParse(parts[1], out var kilobytes) + ? kilobytes * 1024 + : null; + } + } + + private sealed record MusicBrainzImportProbeReport( + string ProbeName, + string StoragePath, + string DatabasePath, + DateTimeOffset StartedAtUtc, + DateTimeOffset FinishedAtUtc, + double DurationMilliseconds, + bool Success, + string? ErrorMessage, + DecentDBMusicBrainzImportSummary? ImportSummary, + IReadOnlyList PhaseTimings, + IReadOnlyList Samples); + + private sealed record ImportProbePhase( + string Name, + DateTimeOffset StartedAtUtc, + DateTimeOffset FinishedAtUtc, + double DurationMilliseconds, + int Current, + int Total, + string? LastMessage); + + private sealed record ImportProbeSample( + DateTimeOffset TimestampUtc, + long WorkingSetBytes, + long PeakWorkingSetBytes, + double TotalProcessorMilliseconds, + long? VmRssBytes, + long? VmHwmBytes, + long DatabaseBytes, + long WalBytes, + long DashWalBytes); +} diff --git a/benchmarks/Melodee.Benchmarks/MusicBrainzProbeOptions.cs b/benchmarks/Melodee.Benchmarks/MusicBrainzProbeOptions.cs new file mode 100644 index 000000000..c855db5e6 --- /dev/null +++ b/benchmarks/Melodee.Benchmarks/MusicBrainzProbeOptions.cs @@ -0,0 +1,61 @@ +namespace Melodee.Benchmarks; + +internal sealed class MusicBrainzProbeOptions +{ + private readonly Dictionary values; + + private MusicBrainzProbeOptions(Dictionary values) + { + this.values = values; + } + + public static MusicBrainzProbeOptions Parse(string[] args) + { + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < args.Length; i++) + { + var arg = args[i]; + if (!arg.StartsWith("--", StringComparison.Ordinal)) + { + continue; + } + + var key = arg[2..]; + if (i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + values[key] = args[i + 1]; + i++; + } + else + { + values[key] = "true"; + } + } + + return new MusicBrainzProbeOptions(values); + } + + public string? GetString(string key) => + values.TryGetValue(key, out var value) ? value : null; + + public string RequireString(string key) + { + var value = GetString(key); + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"Missing required --{key} argument."); + } + + return value; + } + + public bool GetBool(string key) => + values.TryGetValue(key, out var value) && + (value is null || bool.TryParse(value, out var result) && result); + + public int GetInt32(string key, int defaultValue) + { + var value = GetString(key); + return int.TryParse(value, out var parsed) ? parsed : defaultValue; + } +} diff --git a/benchmarks/Melodee.Benchmarks/MusicBrainzQueryProbe.cs b/benchmarks/Melodee.Benchmarks/MusicBrainzQueryProbe.cs new file mode 100644 index 000000000..1ce16fb9e --- /dev/null +++ b/benchmarks/Melodee.Benchmarks/MusicBrainzQueryProbe.cs @@ -0,0 +1,459 @@ +using System.Diagnostics; +using System.Text.Json; +using DecentDB.AdoNet; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data.Models.Materialized; +using Microsoft.EntityFrameworkCore; + +namespace Melodee.Benchmarks; + +internal static class MusicBrainzQueryProbe +{ + public static async Task RunAsync(string[] args) + { + try + { + var options = MusicBrainzProbeOptions.Parse(args); + var databasePath = options.RequireString("db"); + var outputPath = options.GetString("output"); + + if (!File.Exists(databasePath)) + { + throw new FileNotFoundException("MusicBrainz DecentDB database was not found.", databasePath); + } + + var report = await RunProbeAsync(databasePath, options, CancellationToken.None) + .ConfigureAwait(false); + await WriteReportAsync(report, outputPath, CancellationToken.None).ConfigureAwait(false); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.Message); + return 1; + } + } + + private static async Task RunProbeAsync( + string databasePath, + MusicBrainzProbeOptions options, + CancellationToken cancellationToken) + { + var startedAt = DateTimeOffset.UtcNow; + await using var sampleContext = CreateContext(databasePath); + var sampleValues = await ResolveSampleAsync(sampleContext, options, cancellationToken) + .ConfigureAwait(false); + var sample = QueryProbeSample.FromValues(sampleValues); + var indexes = GetIndexDiagnostics(sampleContext); + var includeRowExistenceProbe = options.GetBool("include-row-existence"); + + var measurements = new List(); + foreach (var pass in new[] { "cold", "warm" }) + { + await using var context = CreateContext(databasePath); + measurements.Add(await MeasureCanConnectAsync(context, pass, cancellationToken) + .ConfigureAwait(false)); + if (includeRowExistenceProbe) + { + measurements.Add(await MeasureQueryAsync( + context, + pass, + "ordered-first-row-existence", + sampleValues.FirstArtistId.ToString(), + context.Artists + .AsNoTracking() + .OrderBy(a => a.Id) + .Select(a => a.Id) + .Take(1), + databasePath, + """ + SELECT "Id" + FROM "Artist" + ORDER BY "Id" + LIMIT 1 + """, + cancellationToken) + .ConfigureAwait(false)); + } + measurements.Add(await MeasureQueryAsync( + context, + pass, + "exact-normalized-name", + sample.NameNormalized, + context.Artists + .AsNoTracking() + .Where(a => a.NameNormalized == sampleValues.NameNormalized) + .OrderBy(a => a.SortName) + .Take(10), + databasePath, + $""" + SELECT * + FROM "Artist" + WHERE "NameNormalized" = {ToSqlStringLiteral(sampleValues.NameNormalized)} + ORDER BY "SortName" + LIMIT 10 + """, + cancellationToken) + .ConfigureAwait(false)); + + if (!string.IsNullOrWhiteSpace(sampleValues.AliasNormalized)) + { + measurements.Add(await MeasureQueryAsync( + context, + pass, + "exact-normalized-alias", + sampleValues.AliasNormalized, + context.ArtistAliases + .AsNoTracking() + .Where(a => a.NameNormalized == sampleValues.AliasNormalized) + .OrderBy(a => a.MusicBrainzArtistId) + .Take(10), + databasePath, + $""" + SELECT * + FROM "ArtistAlias" + WHERE "NameNormalized" = {ToSqlStringLiteral(sampleValues.AliasNormalized)} + ORDER BY "MusicBrainzArtistId" + LIMIT 10 + """, + cancellationToken) + .ConfigureAwait(false)); + } + + measurements.Add(await MeasureQueryAsync( + context, + pass, + "exact-musicbrainz-id-raw", + sampleValues.MusicBrainzIdRaw, + context.Artists + .AsNoTracking() + .Where(a => a.MusicBrainzIdRaw == sampleValues.MusicBrainzIdRaw) + .OrderBy(a => a.Id) + .Take(1), + databasePath, + $""" + SELECT * + FROM "Artist" + WHERE "MusicBrainzIdRaw" = {ToSqlStringLiteral(sampleValues.MusicBrainzIdRaw)} + ORDER BY "Id" + LIMIT 1 + """, + cancellationToken) + .ConfigureAwait(false)); + } + + return new MusicBrainzQueryProbeReport( + "musicbrainz-query-probe", + databasePath, + sampleContext.Database.ProviderName ?? "unknown", + startedAt, + DateTimeOffset.UtcNow, + sample, + indexes, + measurements); + } + + private static MusicBrainzDbContext CreateContext(string databasePath) + { + var options = new DbContextOptionsBuilder() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + return new MusicBrainzDbContext(options); + } + + private static async Task ResolveSampleAsync( + MusicBrainzDbContext context, + MusicBrainzProbeOptions options, + CancellationToken cancellationToken) + { + var requestedName = options.GetString("name"); + var requestedAlias = options.GetString("alias"); + var requestedMbid = options.GetString("mbid"); + + var artist = !string.IsNullOrWhiteSpace(requestedMbid) + ? await context.Artists + .AsNoTracking() + .Where(a => a.MusicBrainzIdRaw == requestedMbid) + .OrderBy(a => a.Id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false) + : null; + + artist ??= !string.IsNullOrWhiteSpace(requestedName) + ? await context.Artists + .AsNoTracking() + .Where(a => a.NameNormalized == requestedName) + .OrderBy(a => a.Id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false) + : null; + + artist ??= await context.Artists + .AsNoTracking() + .OrderBy(a => a.Id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + if (artist is null) + { + throw new InvalidOperationException("The MusicBrainz database has no materialized artists."); + } + + var alias = requestedAlias; + if (string.IsNullOrWhiteSpace(alias)) + { + alias = await context.ArtistAliases + .AsNoTracking() + .Where(a => a.MusicBrainzArtistId == artist.MusicBrainzArtistId) + .OrderBy(a => a.NameNormalized) + .Select(a => a.NameNormalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + if (string.IsNullOrWhiteSpace(alias)) + { + alias = await context.ArtistAliases + .AsNoTracking() + .OrderBy(a => a.MusicBrainzArtistId) + .ThenBy(a => a.NameNormalized) + .Select(a => a.NameNormalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + return new QueryProbeSampleValues( + artist.Id, + artist.NameNormalized, + alias, + artist.MusicBrainzIdRaw); + } + + private static IReadOnlyList GetIndexDiagnostics(MusicBrainzDbContext context) + { + var entityTypes = new[] + { + typeof(Artist), + typeof(ArtistAliasLookup), + typeof(Album) + }; + + return entityTypes + .SelectMany(entityType => context.Model.FindEntityType(entityType)?.GetIndexes() ?? []) + .Select(index => new QueryProbeIndexDiagnostic( + index.DeclaringEntityType.GetTableName() ?? index.DeclaringEntityType.DisplayName(), + index.GetDatabaseName() ?? "(unnamed)", + index.IsUnique, + index.Properties.Select(property => property.Name).ToArray())) + .OrderBy(index => index.Table) + .ThenBy(index => index.Name) + .ToArray(); + } + + private static async Task MeasureCanConnectAsync( + MusicBrainzDbContext context, + string pass, + CancellationToken cancellationToken) + { + var started = Stopwatch.GetTimestamp(); + var canConnect = await context.Database.CanConnectAsync(cancellationToken).ConfigureAwait(false); + var elapsed = Stopwatch.GetElapsedTime(started); + + return new QueryProbeMeasurement( + "database-can-connect", + pass, + null, + canConnect ? 1 : 0, + elapsed.TotalMilliseconds, + null, + QueryProbePlanDiagnostic.NotApplicable("No SQL plan is available for Database.CanConnectAsync().")); + } + + private static async Task MeasureQueryAsync( + MusicBrainzDbContext context, + string pass, + string name, + string? sampleValue, + IQueryable query, + string databasePath, + string planSql, + CancellationToken cancellationToken) + { + var sql = TryGetSql(query); + var started = Stopwatch.GetTimestamp(); + var rows = await query.ToArrayAsync(cancellationToken).ConfigureAwait(false); + var elapsed = Stopwatch.GetElapsedTime(started); + var planDiagnostics = TryGetPlanDiagnostics(databasePath, planSql); + + return new QueryProbeMeasurement( + name, + pass, + SanitizeSampleValue(sampleValue), + rows.Length, + elapsed.TotalMilliseconds, + sql, + planDiagnostics); + } + + private static string? TryGetSql(IQueryable query) + { + try + { + return query.ToQueryString(); + } + catch (Exception ex) + { + return $"SQL unavailable: {ex.Message}"; + } + } + + private static QueryProbePlanDiagnostic TryGetPlanDiagnostics(string databasePath, string planSql) + { + try + { + using var connection = new DecentDBConnection($"Data Source={databasePath}"); + connection.Open(); + var plan = connection.ExplainQuery(planSql); + return QueryProbePlanDiagnostic.Captured( + plan.Sql, + plan.ExplainSql, + plan.Duration.TotalMilliseconds, + plan.Lines); + } + catch (Exception ex) + { + return QueryProbePlanDiagnostic.Unavailable(planSql, ex.Message); + } + } + + private static string ToSqlStringLiteral(string? value) + { + return value is null ? "NULL" : $"'{value.Replace("'", "''", StringComparison.Ordinal)}'"; + } + + private static string? SanitizeSampleValue(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return value; + } + + return value.Length <= 160 ? value : string.Concat(value.AsSpan(0, 160), "..."); + } + + private static async Task WriteReportAsync( + MusicBrainzQueryProbeReport report, + string? outputPath, + CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true }); + if (string.IsNullOrWhiteSpace(outputPath)) + { + Console.WriteLine(json); + return; + } + + var outputDirectory = Path.GetDirectoryName(outputPath); + if (!string.IsNullOrWhiteSpace(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + + await File.WriteAllTextAsync(outputPath, json, cancellationToken).ConfigureAwait(false); + Console.WriteLine($"Wrote MusicBrainz query probe report to {outputPath}"); + } + + private sealed record MusicBrainzQueryProbeReport( + string ProbeName, + string DatabasePath, + string ProviderName, + DateTimeOffset StartedAtUtc, + DateTimeOffset FinishedAtUtc, + QueryProbeSample Sample, + IReadOnlyList Indexes, + IReadOnlyList Measurements); + + private sealed record QueryProbeSample( + long FirstArtistId, + string NameNormalized, + string? AliasNormalized, + string MusicBrainzIdRaw) + { + public static QueryProbeSample FromValues(QueryProbeSampleValues values) + { + return new QueryProbeSample( + values.FirstArtistId, + SanitizeSampleValue(values.NameNormalized) ?? string.Empty, + SanitizeSampleValue(values.AliasNormalized), + SanitizeSampleValue(values.MusicBrainzIdRaw) ?? string.Empty); + } + } + + private sealed record QueryProbeSampleValues( + long FirstArtistId, + string NameNormalized, + string? AliasNormalized, + string MusicBrainzIdRaw); + + private sealed record QueryProbeIndexDiagnostic( + string Table, + string Name, + bool IsUnique, + IReadOnlyList Properties); + + private sealed record QueryProbeMeasurement( + string Name, + string Pass, + string? SampleValue, + int RowCount, + double ElapsedMilliseconds, + string? Sql, + QueryProbePlanDiagnostic PlanDiagnostics); + + private sealed record QueryProbePlanDiagnostic( + string Status, + string? PlanSql, + string? ExplainSql, + double? ElapsedMilliseconds, + IReadOnlyList Lines, + string? Error) + { + public static QueryProbePlanDiagnostic Captured( + string planSql, + string explainSql, + double elapsedMilliseconds, + IReadOnlyList lines) + { + return new QueryProbePlanDiagnostic( + "captured", + planSql, + explainSql, + elapsedMilliseconds, + lines, + null); + } + + public static QueryProbePlanDiagnostic Unavailable(string planSql, string error) + { + return new QueryProbePlanDiagnostic( + "unavailable", + planSql, + null, + null, + [], + error); + } + + public static QueryProbePlanDiagnostic NotApplicable(string reason) + { + return new QueryProbePlanDiagnostic( + "not-applicable", + null, + null, + null, + [], + reason); + } + } +} diff --git a/benchmarks/Melodee.Benchmarks/Program.cs b/benchmarks/Melodee.Benchmarks/Program.cs index 6d25451dc..fc67d0f54 100644 --- a/benchmarks/Melodee.Benchmarks/Program.cs +++ b/benchmarks/Melodee.Benchmarks/Program.cs @@ -8,7 +8,7 @@ namespace Melodee.Benchmarks; public class Program { - public static void Main(string[] args) + public static async Task Main(string[] args) { if (args.Length == 0) { @@ -21,18 +21,37 @@ public static void Main(string[] args) Console.WriteLine(" cache - Cache performance benchmarks"); Console.WriteLine(" collection - Collection operation benchmarks"); Console.WriteLine(" musicbrainz - MusicBrainz import performance benchmarks"); + Console.WriteLine(" musicbrainz-query-probe - Manual DecentDB MusicBrainz query probe"); + Console.WriteLine(" musicbrainz-import-probe - Manual DecentDB MusicBrainz import probe"); Console.WriteLine(" all - Run all benchmarks"); Console.WriteLine(); Console.WriteLine("Usage: dotnet run -c Release --project benchmarks/Melodee.Benchmarks [category] [-- BenchmarkDotNet args]"); Console.WriteLine("Example: dotnet run -c Release --project benchmarks/Melodee.Benchmarks streaming"); Console.WriteLine("Example: dotnet run -c Release --project benchmarks/Melodee.Benchmarks all -- --exporters json,github,csv --artifacts benchmarks/artifacts"); - return; + Console.WriteLine("Example: dotnet run -c Release --project benchmarks/Melodee.Benchmarks musicbrainz-query-probe -- --db /path/musicbrainz.ddb --output query-probe.json [--include-row-existence]"); + Console.WriteLine("Example: dotnet run -c Release --project benchmarks/Melodee.Benchmarks musicbrainz-import-probe -- --storage /path/musicbrainz --db /tmp/musicbrainz.ddb --output import-probe.json --clean"); + return 0; } var category = args[0].ToLower(); // Parse BenchmarkDotNet arguments (everything after the first argument) var benchmarkArgs = args.Skip(1).ToArray(); + if (benchmarkArgs.Length > 0 && benchmarkArgs[0] == "--") + { + benchmarkArgs = benchmarkArgs.Skip(1).ToArray(); + } + + if (category == "musicbrainz-query-probe") + { + return await MusicBrainzQueryProbe.RunAsync(benchmarkArgs); + } + + if (category == "musicbrainz-import-probe") + { + return await MusicBrainzImportProbe.RunAsync(benchmarkArgs); + } + var config = CreateConfig(benchmarkArgs); switch (category) @@ -61,9 +80,11 @@ public static void Main(string[] args) break; default: Console.WriteLine($"Unknown benchmark category: {category}"); - Console.WriteLine("Available categories: streaming, database, cache, collection, musicbrainz, all"); + Console.WriteLine("Available categories: streaming, database, cache, collection, musicbrainz, musicbrainz-query-probe, musicbrainz-import-probe, all"); break; } + + return 0; } private static IConfig CreateConfig(string[] args) diff --git a/design/2026-05-25.REVIEW.md b/design/2026-05-25.REVIEW.md new file mode 100644 index 000000000..7b7540eb3 --- /dev/null +++ b/design/2026-05-25.REVIEW.md @@ -0,0 +1,914 @@ +## Melodee Large Collection Ingestion Review + +**Date**: 2026-05-25 +**Status**: Architecture review and phased implementation plan +**Scope**: Ingestion, staging, storage promotion, database insertion, and +large-collection runtime concerns + +## Executive Summary + +Melodee has the right high-level ingestion model for a serious media manager: +raw releases enter an inbound area, normalized releases move to staging, valid +staged releases move to canonical storage, and storage metadata is inserted into +the database for serving through the UI and APIs. + +The current implementation is not yet shaped for the stated target scale of +millions of songs and hundreds of thousands of releases. The main issue is not a +single bad loop or one missing index. The broader issue is that ingestion is +still mostly a filesystem sweep with in-memory progress, rather than a durable, +idempotent work pipeline. + +The system should evolve toward: + +- a persistent release work item model +- streaming discovery rather than full-tree materialization +- idempotent file moves and database inserts +- a clear separation between structural media validity and metadata enrichment +- batch-oriented database writes and aggregate maintenance +- materialized browse/search indexes for large library API requests +- durable messaging once the work boundaries need to cross processes + +On the queue question: yes, this is the right time to talk about a real queue +system such as RabbitMQ. It should be planned now, but it should not be the first +implementation move. A broker delivers work; it does not make filesystem moves, +database writes, or metadata enrichment idempotent. Melodee should first make the +database the source of truth for ingestion state. After that, RabbitMQ becomes a +clean way to distribute the work across dedicated workers and survive process +restarts. + +This aligns with the existing internal direction in +`design/requirements/message-bus-strategy-and-requirements.md`, which already +standardizes on Wolverine, RabbitMQ, and PostgreSQL-backed durable inbox/outbox. + +## Current Ingestion Model + +The documented model is accurate at the conceptual level: + +1. Inbound processing reads releases from an inbound library. +2. Media and metadata are normalized into the staging library. +3. Staging revalidation can re-check albums that failed artist enrichment. +4. Valid staged albums move to storage. +5. Storage albums are inserted into the database. +6. API clients then serve from the database and canonical storage paths. + +The CLI `mcli library scan` currently runs four steps in sequence: + +1. `LibraryInboundProcessJob` +2. `StagingAlbumRevalidationJob` +3. `StagingAutoMoveJob` +4. `LibraryInsertJob` + +The scheduled job chain is slightly different: inbound processing chains to +staging auto-move, and staging auto-move chains to library insert. Staging +revalidation runs on its own schedule, but the CLI scan includes it as part of +the full manual workflow. + +That model is understandable for users and operators. The implementation needs +to become more durable and less sweep-oriented before it can be trusted for very +large collections. + +## Primary Findings + +### Finding 1: Ingestion Is Sweep-Based Instead Of Work-Item-Based + +Inbound processing discovers candidate directories, materializes them, and then +processes the list. Staging promotion and database insertion also scan broad +directory trees to find `melodee.json` files. + +Relevant examples: + +- `DirectoryProcessorToStagingService.ProcessDirectoryAsync` calls + `GetFileSystemDirectoryInfosToProcess(...).ToList()`. +- `LibraryService.MoveAlbumsFromLibraryToLibrary` calls + `Directory.GetFiles(..., SearchOption.AllDirectories)`. +- `LibraryInsertJob.GetMelodeeFilesToProcess` recursively enumerates all + `melodee.json` files and returns a `List`. +- `LibraryService.Rebuild` turns the directory enumerable into an array. + +For small libraries this is reasonable. For hundreds of thousands of releases, +even no-op scans become expensive because the application must keep asking the +filesystem the same questions. + +The durable representation should be: + +- one row per discovered release or release directory +- one stable source path +- one normalized destination path +- one manifest hash +- one current state +- attempt count and last error +- lease fields for safe concurrent workers +- timestamps for discovered, staged, stored, indexed, and failed transitions + +The filesystem should be used to discover and verify work, not to be the only +source of truth about what work exists. + +### Finding 2: Checkpoints Can Advance Without A Full Successful Outcome + +The current jobs are written as procedural workflows. If one stage reports +errors but still returns a result object, the caller can still record scan +history or update status values. + +The high-risk pattern is: + +- scan starts +- some releases fail +- job records a scan timestamp +- the next run assumes the library is up to date +- the failed releases do not get retried unless a force scan or a separate + reprocess path catches them + +For large collections, this is dangerous because the operator cannot reasonably +inspect every missed release. Each release needs its own durable state and retry +history. Job-level timestamps are still useful for reporting, but they should not +be the only gate controlling whether a release is eligible for processing. + +### Finding 3: Structural Validity And Metadata Enrichment Are Coupled Too Tightly + +Today an album can be invalid because the artist cannot be resolved to a Melodee +database id, MusicBrainz id, or Spotify id. That makes ingestion success depend +on external or local search-engine coverage. + +That is too strict for large personal collections. Large libraries contain: + +- obscure artists +- private recordings +- live sets +- purchased releases not yet known to metadata providers +- internal or self-produced albums +- legacy releases with imperfect tags + +Melodee should distinguish: + +- **structurally valid**: playable media, sane tags, track numbers, durations, + album title, album artist text, and no missing files +- **enriched**: matched to MusicBrainz, Spotify, Discogs, or another metadata + authority +- **curated**: manually reviewed or explicitly accepted by the operator + +Only structural validity should be required for safe staging and storage. Search +engine enrichment should improve metadata quality, artwork, relations, and +confidence, but it should not always block a clean local release from entering +the canonical library. + +This point matters directly for Blackbeard provenance. Blackbeard can provide +useful release-local facts. It should help Melodee trust a release as a +well-formed local release, while enrichment can remain asynchronous. + +### Finding 4: Database Insert Is Batched, But Still Too Per-Album Oriented + +`LibraryInsertJob` has moved in a better direction by using batches and +short-lived contexts. However, the hot path still does repeated per-album +operations: + +- find or create artist +- find existing album +- calculate song file hash +- build song entities +- save albums +- derive contributors +- save contributors + +This should be pushed toward bulk lookup and set-based persistence: + +- read a batch of manifests +- collect all artist keys, normalized names, MusicBrainz ids, and Spotify ids +- issue one lookup per key shape or one consolidated lookup strategy +- collect all album keys and normalized identifiers +- issue one album lookup +- insert missing artists and albums in a transaction +- insert songs with already-known file metadata +- insert contributors with bulk dedupe + +File hashes should be computed during normalization and stored in the manifest. +The insert phase should not reread every media file just to compute a CRC unless +the manifest is missing or explicitly stale. + +### Finding 5: Aggregate Refresh Is O(Total Library) + +`LibraryService.UpdateAggregatesAsync` loads all artists in a library and issues +count queries per artist. It then updates library counts and clears the entire +cache. + +That is not suitable after every insert batch in a very large collection. + +The target shape should be one of: + +- incremental counter updates inside the insert transaction +- periodic aggregate maintenance as a separate background job +- set-based SQL update statements +- materialized aggregate tables refreshed by changed artist/library ids + +The cache should also be invalidated by region and entity scope. A full cache +clear after aggregate work is a blunt tool that will cause avoidable cold-cache +latency in large deployments. + +### Finding 6: The Final Storage Layout Is Better Than The Transient Layout + +Final storage already shards artist directories by prefix through +`ArtistExtensions.ToDirectoryName`. That is a good direction. It avoids putting +all artists directly under one storage root. + +The transient layout is weaker: + +- inbound may contain many top-level releases +- staging can accumulate many release folders +- staging promotion scans all `melodee.json` files recursively +- invalid releases can remain in staging indefinitely + +For very large ingestion runs, staging should also be sharded. A common shape is: + +```text +staging/ + 2026/ + 05/ + run-20260525-001/ + ok/ + review/ + failed/ +``` + +or: + +```text +staging/ + ab/ + abc123-release-id/ +``` + +The exact structure is less important than the guarantee that no single +directory becomes a massive operational hot spot. + +### Finding 7: Runtime API Paths Need Large-Library Browse Strategies + +The ingestion pipeline is the immediate concern, but a million-song library also +changes API expectations. + +Two examples need attention: + +- `GetIndexesAsync` loads all artists, then groups them in memory. +- `GetRandomSongsAsync` can load all matching songs, then shuffle in memory. + +Those patterns will become visible latency and memory problems. Large libraries +need materialized browse indexes, keyset pagination, indexed random sampling, and +precomputed counts for common OpenSubsonic views. + +### Finding 8: CLI Scans Do Not Get Scheduler Concurrency Guarantees + +Quartz job attributes such as `DisallowConcurrentExecution` help when Quartz is +actually scheduling the job. The CLI scan command directly instantiates and runs +job classes. + +That means a manual CLI scan can race the server scheduler unless there is a +separate database-backed library lock or run lease. + +This matters more once the library is large, because long-running scans become +normal. The system should assume that two processes can exist: + +- the web/server process +- a CLI or worker process + +Both must coordinate through durable state, not in-process objects. + +## Phase Map + +| Phase | Name | Current status | Target status | +|-------|------|----------------|---------------| +| 0 | Baseline and guardrails | Partial | Measured baseline with scale tests and operational budgets | +| 1 | Durable ingestion state | Not started | Database-backed release work items with leases and retries | +| 2 | Streaming discovery | Partial | No full-tree materialization in normal ingestion paths | +| 3 | Validation/enrichment split | Partial | Local structural validity can pass without external ids | +| 4 | Idempotent file operations and layout | Partial | Atomic or resumable moves with sharded staging | +| 5 | Bulk database insertion | Partial | Batch lookup/upsert with transactional checkpointing | +| 6 | Aggregates and API scale | Partial | Incremental aggregates and materialized browse paths | +| 7 | Durable distributed messaging | Design documented | Wolverine + RabbitMQ + durable inbox/outbox in runtime | +| 8 | Operations, replay, and observability | Partial | DLQ/replay/run dashboards and clear operator controls | + +Status definitions: + +- **Not started**: no meaningful implementation exists in the current shape. +- **Partial**: some supporting pieces exist, but the phase goal is not satisfied. +- **Design documented**: architectural direction exists, but runtime behavior is + not yet implemented. +- **Complete**: implementation and validation are both in place. + +No phase in this map should be marked complete until there are tests or measured +evidence proving the claim at representative scale. + +## Phase 0: Baseline And Guardrails + +### Goal + +Create enough measurement and regression coverage that future ingestion work can +be evaluated objectively. + +### Current Status + +Partial. + +Melodee has useful logs and several recent performance-oriented changes, but the +normal test suite does not yet prove behavior at the intended collection size. + +### Work Items + +1. Add a synthetic filesystem generator for release trees. +2. Add a synthetic manifest generator for `melodee.json` files. +3. Add a database seed path for: + - 100,000 artists + - 500,000 albums + - 5,000,000 songs +4. Add no-op scan benchmarks. +5. Add staged-to-storage promotion benchmarks. +6. Add storage-to-database insert benchmarks. +7. Add OpenSubsonic browse benchmarks for: + - indexes + - random songs + - album lists + - search +8. Track wall-clock time, peak memory, database query count, files enumerated, + and bytes read/written. + +### Exit Criteria + +Phase 0 is complete when a developer can run one command that produces a +repeatable ingestion performance report. The report should show both a small +developer dataset and a large synthetic dataset. + +The large synthetic dataset does not need real audio. It can use placeholder +files or fake filesystem abstractions for most tests. Full media tests should be +reserved for smaller representative samples. + +## Phase 1: Durable Ingestion State + +### Goal + +Make ingestion state explicit, durable, resumable, and visible. + +### Current Status + +Not started. + +The code currently uses library scan history, filesystem state, logs, and +manifest files. That is not enough to safely coordinate large ingestion. + +### Proposed Data Model + +Add a table such as `IngestionItems` or `ReleaseWorkItems`. + +Suggested fields: + +| Field | Purpose | +|-------|---------| +| `Id` | Stable database identity | +| `ApiKey` | Public/stable id for operator views and APIs | +| `SourceLibraryId` | Inbound or staging library source | +| `TargetLibraryId` | Staging or storage target when known | +| `SourcePath` | Original release directory | +| `CurrentPath` | Current known directory | +| `TargetPath` | Planned destination | +| `ManifestPath` | Current `melodee.json` path, if created | +| `ManifestHash` | Hash of normalized manifest content | +| `SourceFingerprint` | Hash or summary of source file names, sizes, mtimes | +| `State` | Current state enum | +| `StateReason` | Machine-readable reason for review/failure | +| `AttemptCount` | Retry count | +| `LastError` | Truncated operator-safe error | +| `LockedBy` | Worker/process lease owner | +| `LockedUntil` | Lease expiration | +| `DiscoveredAt` | First discovery timestamp | +| `LastAttemptAt` | Last work attempt timestamp | +| `CompletedAt` | Terminal success timestamp | +| `CorrelationId` | Run or message correlation | + +Suggested states: + +```text +Discovered +Normalizing +Normalized +Staged +StructurallyValid +NeedsReview +EnrichmentPending +Enriched +MovingToStorage +Stored +Indexing +Indexed +Failed +Quarantined +Ignored +``` + +### Required Behavior + +- Work is claimed by lease, not by in-process assumptions. +- A worker can crash and another worker can resume after lease expiration. +- A release is retried based on item state, not based on directory timestamps. +- A scan history row reports what happened, but does not decide what still needs + to happen. +- State transitions are monotonic unless an explicit reprocess command is issued. +- Every transition records enough context for an operator to understand why the + release is where it is. + +### Exit Criteria + +Phase 1 is complete when inbound processing can be interrupted and restarted +without losing track of every discovered release. + +## Phase 2: Streaming Discovery + +### Goal + +Remove full-tree materialization from normal ingestion paths. + +### Current Status + +Partial. + +Some helpers use enumeration APIs, but multiple hot paths immediately convert +the result to `List`, `Array`, or full `Directory.GetFiles` output. + +### Work Items + +1. Replace `ToList()` and `ToArray()` discovery points with paged work item + creation. +2. Store discovered candidates in the ingestion table in batches. +3. Use keyset paging over ingestion items when workers process releases. +4. Keep progress reporting approximate when exact totals require a full scan. +5. Avoid repeated file enumeration inside the same release directory. +6. Change `GetFileSystemDirectoryInfosToProcess` so it does not enumerate files + twice for each directory. +7. Review the `*.*` search patterns. Some directory names without dots may be + skipped depending on platform behavior and API call shape. +8. Avoid root-directory last-write time as the primary scan gate. + +### Design Notes + +Streaming discovery should not mean "do one release and forget everything." It +should mean: + +- discover a bounded page +- persist candidates +- process candidates by durable state +- continue until the page source is exhausted + +That gives the system resumability without requiring a perfect filesystem +watcher. + +### Exit Criteria + +Phase 2 is complete when a no-op scan of a large already-processed library does +not recursively deserialize or inspect every release unless explicitly requested. + +## Phase 3: Split Structural Validation From Enrichment + +### Goal + +Allow well-formed local releases to flow through ingestion even when external +metadata providers cannot identify the artist or album. + +### Current Status + +Partial. + +Melodee already has validation and enrichment concepts, but current album +validity is still strongly tied to artist identity from Melodee, MusicBrainz, or +Spotify. + +### Proposed Validation Categories + +#### Structural validation + +This should answer: can Melodee safely store and play this release? + +Required checks: + +- media files exist +- media files are readable +- song count is coherent +- track numbers are coherent +- duration values are present +- album title is present +- album artist text is present +- storage path can be generated +- required artwork policy is satisfied, if enabled + +#### Enrichment validation + +This should answer: how confident is Melodee that this release is matched to +external metadata? + +Signals: + +- MusicBrainz artist id +- MusicBrainz release id +- Spotify artist id +- Spotify album id +- Discogs id +- local artist cache match +- Blackbeard provenance +- NFO/SFV/CUE/M3U metadata confidence + +#### Curation validation + +This should answer: did a trusted operator explicitly accept this release? + +Signals: + +- manual approval +- trusted source rule +- trusted provenance source +- ignored warning rule + +### Blackbeard Role + +Blackbeard provenance should be treated as a first-class directory metadata +source. It should help establish local structural facts, such as: + +- source tool version +- normalized tags +- final validation outcome from the upstream tool +- track list +- file names +- release-level artist/title/year/genre + +It should not automatically import secrets or unstable internal paths. The +provenance file can contain source URLs, commands, absolute paths, and internal +details. Melodee should retain only safe fields or store sensitive fields in a +redacted operator-only diagnostic area. + +### Exit Criteria + +Phase 3 is complete when a local release with good tags, media, and artwork can +be marked structurally valid even if MusicBrainz and Spotify return no match. + +## Phase 4: Idempotent File Operations And Scalable Layout + +### Goal + +Make file movement safe, resumable, and suitable for large directories and +network-mounted storage. + +### Current Status + +Partial. + +The code has improved copy helpers and final storage sharding by artist prefix, +but staging and movement workflows are not yet modeled as durable, resumable +operations. + +### Work Items + +1. Add operation ids for normalization, staging move, storage move, and insert. +2. Use temporary destination directories during copy/move. +3. Write a move manifest before starting a multi-file move. +4. Verify destination file size and hash when required. +5. Mark the operation complete only after manifest verification. +6. Delete source files only after successful destination verification. +7. Make repeated execution safe. +8. Add staging sharding. +9. Add collision strategy for same artist/year/title. +10. Keep invalid releases in review or failed state, not in an unbounded flat + staging root. + +### Suggested Move Shape + +```text +target/ + .melodee-op-123.tmp/ + files... + move-manifest.json + Final Directory/ + files... + melodee.json +``` + +The operation should only rename the temporary directory to the final directory +after all files are present and verified. + +### Exit Criteria + +Phase 4 is complete when killing the process during a file move never leaves a +release in an ambiguous state. On restart, Melodee should either resume, roll +forward, or mark the release failed with a specific reason. + +## Phase 5: Bulk Database Insertion + +### Goal + +Turn storage-to-database insertion into a batch lookup/upsert pipeline with +transactional checkpointing. + +### Current Status + +Partial. + +`LibraryInsertJob` uses batches, but still does too much per album and per song. + +### Work Items + +1. Read a bounded page of stored work items that are ready for indexing. +2. Deserialize manifests in parallel with bounded concurrency. +3. Collect artist lookup keys for the whole batch. +4. Query existing artists in bulk. +5. Insert missing artists in bulk. +6. Collect album lookup keys for the whole batch. +7. Query existing albums in bulk. +8. Insert missing albums and songs in one transaction per batch. +9. Insert contributors after song ids are known. +10. Mark each ingestion item indexed only after the transaction commits. +11. Do not compute media file hashes in the insert phase when a trusted manifest + already has them. + +### Transaction Rule + +The database transaction should own: + +- artist insert +- album insert +- song insert +- contributor insert +- ingestion item state transition to `Indexed` +- outbox message creation for downstream consumers + +If the transaction fails, the ingestion item should remain eligible for retry. + +### Exit Criteria + +Phase 5 is complete when inserting a large batch has bounded query counts and +does not reread every media file. + +## Phase 6: Aggregates, Cache, And Runtime API Scale + +### Goal + +Keep the application responsive after the library is large. + +### Current Status + +Partial. + +The database has many useful indexes, and services use `AsNoTracking` in many +read paths. However, some high-level API methods still load too much data into +memory, and aggregate refreshes are too broad. + +### Work Items + +1. Replace full aggregate refresh after inserts with incremental aggregate + changes. +2. Add a periodic aggregate repair job for consistency. +3. Invalidate caches by affected entity/library/region, not globally. +4. Materialize OpenSubsonic artist indexes. +5. Add keyset pagination for large browse views. +6. Replace in-memory random song shuffling with indexed random sampling. +7. Normalize genres into queryable tables if genre filtering becomes common. +8. Add query plans or benchmark assertions for common API calls. + +### Candidate Materialized Tables + +```text +ArtistBrowseIndex +AlbumBrowseIndex +SongSearchIndex +GenreIndex +LibraryAggregateSnapshot +``` + +These do not need to replace normalized source tables. They can be denormalized +read models maintained by ingestion and repair jobs. + +### Exit Criteria + +Phase 6 is complete when common API operations do not need to load all artists, +albums, or matching songs into memory. + +## Phase 7: Durable Distributed Messaging + +### Goal + +Introduce a real queue/bus when Melodee is ready to distribute ingestion work +across processes. + +### Current Status + +Design documented. + +The existing design direction is Wolverine plus RabbitMQ plus PostgreSQL durable +inbox/outbox. The current runtime still needs to be brought into that shape. + +### Should RabbitMQ Be Discussed Now? + +Yes. + +RabbitMQ should be part of the architecture now because Melodee's target scale +implies long-running background work, retries, multiple worker processes, and +clear operational boundaries. A single web process with in-memory local +messaging is not a strong long-term fit for large ingestion. + +However, RabbitMQ should not be treated as the first fix. Adding a broker before +the ingestion operations are idempotent can make failures harder to reason about. +The database-backed work item model should come first. + +The right sequence is: + +1. Make release state durable in PostgreSQL. +2. Make operations idempotent and lease-based. +3. Make database writes transactional with work item state changes. +4. Add durable outbox messages from those transactions. +5. Use RabbitMQ to distribute commands/events to workers. +6. Add durable inbox and idempotent consumers. +7. Add dead-letter handling and replay tools. + +### Recommended Message Types + +Commands: + +- `DiscoverInboundLibrary` +- `NormalizeRelease` +- `ValidateRelease` +- `EnrichReleaseMetadata` +- `MoveReleaseToStorage` +- `IndexStoredRelease` +- `RepairLibraryAggregates` + +Events: + +- `ReleaseDiscovered` +- `ReleaseNormalized` +- `ReleaseMarkedStructurallyValid` +- `ReleaseNeedsReview` +- `ReleaseMovedToStorage` +- `ReleaseIndexed` +- `ReleaseFailed` +- `ArtistCreated` +- `AlbumCreated` +- `SongCreated` + +### Queue Boundary Rule + +RabbitMQ should carry commands and events. It should not be the source of truth +for release state. + +The source of truth should be PostgreSQL: + +- current item state +- leases +- attempts +- errors +- idempotency keys +- outbox messages +- inbox dedupe + +This lets Melodee survive: + +- web process restarts +- worker process restarts +- RabbitMQ restarts +- temporary storage outages +- retry storms +- duplicate message delivery + +### Exit Criteria + +Phase 7 is complete when ingestion workers can run outside the web host, process +work through RabbitMQ, survive restarts, and safely replay failed messages. + +## Phase 8: Operations, Replay, And Observability + +### Goal + +Give operators enough visibility and control to run Melodee against very large +libraries without guessing what happened. + +### Current Status + +Partial. + +Logs exist, and the CLI reports useful summaries. The missing piece is a +release-level operational view. + +### Work Items + +1. Add an ingestion run table. +2. Add an ingestion item detail view. +3. Add state transition history. +4. Add retry controls. +5. Add replay controls. +6. Add quarantine/review controls. +7. Add DLQ browser once RabbitMQ is introduced. +8. Add per-phase metrics: + - releases discovered + - releases normalized + - releases moved + - releases indexed + - bytes copied + - files copied + - average duration per release + - failure count by reason +9. Add operator-safe error messages that do not expose secrets from provenance + files or source metadata. + +### Exit Criteria + +Phase 8 is complete when an operator can answer these questions from the UI or +CLI: + +- What is currently being processed? +- What failed? +- Why did it fail? +- Is it safe to retry? +- What will retry do? +- Which worker owns this item? +- How many releases are waiting in each phase? +- How long will the backlog likely take? + +## Suggested Implementation Order + +### First Implementation Slice + +The first slice should be small but structural: + +1. Add `IngestionItem` and `IngestionRun` tables. +2. Add state enums and transition helpers. +3. Add a discovery path that records inbound release directories as work items. +4. Keep existing processing logic, but have it claim one item at a time. +5. Mark items as staged, failed, or needing review. +6. Add CLI reporting for item counts by state. + +This slice creates the durable backbone without forcing a full rewrite. + +### Second Implementation Slice + +Convert staging auto-move to work from durable state: + +1. Select staged items where structural status is valid. +2. Lease items before moving. +3. Move with a temporary destination and operation manifest. +4. Mark moved items as stored. +5. Keep a compatibility fallback for existing staging directories during the + migration period. + +### Third Implementation Slice + +Convert library insert to work from stored items: + +1. Select stored items waiting for indexing. +2. Batch lookup artists and albums. +3. Insert in transactions. +4. Mark indexed only after commit. +5. Emit outbox events if the message infrastructure is ready. + +### Fourth Implementation Slice + +Introduce Wolverine and RabbitMQ for one non-critical command path first. + +Good candidates: + +- enrichment retry +- aggregate repair +- non-blocking image enrichment + +Avoid starting with the file move path. File moves are the least forgiving part +of the workflow and should first become idempotent under direct execution. + +## Risk Register + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Broker introduced before idempotency | Duplicate or partial side effects become harder to debug | Implement durable work items and operation ids first | +| Full scans remain in no-op paths | Large libraries spend most time proving nothing changed | Store item state and use paged work queues | +| External enrichment blocks storage | Obscure valid releases remain stuck | Split structural validity from enrichment | +| Per-song hashing during DB insert | Insert throughput limited by media reads | Compute hashes during normalization and store them | +| Full aggregate rebuild after insert | Insert jobs slow down as library grows | Use incremental counters and repair jobs | +| Global cache clear after maintenance | Large deployments see repeated cold-cache latency | Invalidate by region/entity | +| Staging grows flat and unbounded | Filesystem operations degrade | Shard staging and use item states | +| CLI and scheduler race | Duplicate processing or file conflicts | Use DB leases for library and item work | +| Provenance imports secrets | Sensitive source data leaks into logs or manifests | Redact by default and retain only safe fields | + +## Non-Goals + +The following should not be attempted in the first pass: + +- replacing the entire ingestion pipeline at once +- making RabbitMQ mandatory before local ingestion is durable +- turning `melodee.json` into the only operational state store +- removing manual review workflows +- requiring MusicBrainz or Spotify matches for all stored releases +- building a perfect filesystem watcher + +## Recommended Decision + +Adopt the following direction: + +1. **Approve a durable ingestion state model.** +2. **Treat PostgreSQL as the source of truth for release work.** +3. **Keep `melodee.json` as the portable release manifest, not the workflow + database.** +4. **Split structural validation from enrichment.** +5. **Plan RabbitMQ now, but phase it in after idempotency.** +6. **Use Wolverine + RabbitMQ + durable inbox/outbox when the worker boundary is + introduced.** + +This path preserves the current mental model while changing the execution model +to one that can handle very large collections. diff --git a/design/DECENTDB_IMPROVEMENTS.md b/design/DECENTDB_IMPROVEMENTS.md index 644427f1a..4a206f0e5 100644 --- a/design/DECENTDB_IMPROVEMENTS.md +++ b/design/DECENTDB_IMPROVEMENTS.md @@ -1,1009 +1,782 @@ -# DecentDB EntityFramework Improvements for Melodee +## DecentDB EntityFramework Improvements for Melodee **Date**: 2026-03-21 -**Status**: Targeted Melodee and DecentDB fixes delivered; synthetic validation complete; -final clean real-file metrics still outstanding - -## Summary - -Melodee is using `DecentDB.EntityFrameworkCore` for two very different workloads: - -- `MusicBrainzConnection` backs the large, read-mostly MusicBrainz materialized database. -- `ArtistSearchEngineConnection` backs the much smaller local artist search cache used by - Melodee's search-engine enrichment workflow. - -Those two workloads should not be treated as if they have the same performance envelope. - -The large MusicBrainz `.ddb` is viable for Melodee today, but only if Melodee stays on -index-friendly query shapes and avoids scan-heavy existence or substring patterns on the -request path. The evidence from the real 2.31 GB file is very clear: - -- opening the database is fast -- `CanConnectAsync()` is fast -- first-row projection is fast -- exact normalized-name lookup is fast -- `AnyAsync()` and `CountAsync()` existence probes are pathologically slow -- substring searches over `AlternateNames` and tokenized `Contains(...)` fallbacks are slow -- exact `MusicBrainzIdRaw` lookup is unexpectedly slow and needs explicit investigation - -For the local artist search cache database, the current Melodee usage is acceptable today -because that database is expected to stay much smaller, and the code has multiple caching -layers in front of it. Even so, a few query/index choices will become pain points if that -local cache is allowed to grow substantially. - -The short version is: - -- Melodee's doctor path is now using the correct large-file pattern: - `CanConnectAsync()` plus projected first-row fetches instead of `AnyAsync()` or `CountAsync()`. -- Melodee now materializes exact-match MusicBrainz aliases into a dedicated normalized lookup - structure instead of relying on substring scans over the `AlternateNames` blob. -- Melodee removed the tokenized `Contains(...)` fallback from the default large-file request path - and prefers the fast exact-name path before touching the slower exact-MBID path when both values - are supplied. -- The first real rebuild run proved the large MusicBrainz `.ddb` file was not the startup problem. - The real problem was Melodee's own import materialization strategy. -- The first import fix addressed memory pressure by keeping album writes batch-bounded and by - collapsing duplicate valid `ReleaseCountryStaging` rows so one logical release does not fan out - into duplicate albums. -- Phase-level profiling then showed that the next bottleneck was not album inserts at all. It was - the album materialization query shape itself. -- Melodee now precomputes helper staging tables for resolved release-country rows and primary - artist-credit rows so the hot album query joins simple keyed tables instead of repeatedly paying - for the old aggregate/derived-table shape. -- Melodee now writes large staging/materialization batches through prepared ADO commands where that - shape is materially better than EF `AddRange(...)` plus `SaveChanges()`. -- The DecentDB .NET bindings were also corrected at the source: - - local-source builds now load the correct native library instead of stale packaged native - assets - - EF `DateTime` / `DateTimeOffset` mapping is back on true DecentDB `TIMESTAMP` - - `DecentDBDataReader.GetDateTime()` now decodes DecentDB timestamp microseconds correctly - - UUID-backed values can be read as canonical strings more safely - - repeated single-statement `ExecuteNonQuery()` calls now reuse prepared statements -- The current synthetic importer validation is green again: - `StreamingMusicBrainzImporterTests` passed `11/11` in `26s`. -- The main outstanding work is no longer "find the bottleneck." The remaining work is to finish a - clean monitored real-file rerun and capture the final wall-clock, RSS, file-growth, and - post-import probe numbers. -- `MusicBrainzIdRaw` exact lookup still deserves explicit validation on the rebuilt database before - it is trusted as a first-class fast path. -- The local artist cache is in a better place than MusicBrainz, but its wide OR search and admin - listing patterns are still the most obvious future scale concerns. - -## Completed implementation and validation in this session - -### Melodee-side changes delivered - -The following Melodee changes are no longer theoretical recommendations. They were implemented and -validated during this session: - -1. **Doctor/request-path fixes** - - `DoctorService` now uses `Database.CanConnectAsync()` and projected first-row fetches instead - of expensive large-table existence patterns. - - Regression tests were added so the large-file doctor path does not silently regress back to - `AnyAsync()`/`CountAsync()` behavior. - -2. **MusicBrainz lookup hardening** - - Exact alias lookup moved onto a dedicated normalized alias structure rather than a substring - scan over `AlternateNames`. - - The tokenized `Contains(...)` fallback was removed from the normal large-file request path. - - The repository now prefers exact-name success before paying the suspicious MBID path when both - values are provided. - -3. **MusicBrainz import pipeline fixes** - - Album materialization was changed from a giant in-memory accumulation model to a batch-bounded - pipeline. - - Duplicate valid release-country rows are collapsed before album materialization so one logical - release does not create duplicate albums. - - Artist alternate-name materialization was moved off the per-row update loop and into bulk SQL - aggregation. - - Album insert persistence moved onto raw prepared ADO inserts with lightweight row buffering - rather than EF tracking-heavy bulk adds. - - The album materialization query now uses precomputed helper tables: - - `ReleaseCountryResolvedStaging` - - `ArtistCreditPrimaryArtistStaging` - -4. **Local cache maintenance** - - Melodee now ensures the local artist-cache housekeeping index even on already-created DecentDB - files, rather than assuming a fresh database creation path is the only place the index can be - established. - -### DecentDB provider / binding changes delivered - -The following DecentDB .NET changes were implemented in the DecentDB repository and then wired into -Melodee through local project references: - -1. **Native asset selection for local/source builds** - - The .NET build/runtime asset flow was corrected so local-source validation does not quietly - load stale packaged native binaries. - -2. **TIMESTAMP correctness** - - EF mapping was restored to DecentDB `TIMESTAMP` instead of the temporary integer workaround. - - `GetDateTime()` decoding was corrected to use microseconds since Unix epoch UTC, matching the - DecentDB engine's native representation. - -3. **Query/execution improvements** - - `Any()` / existence translation was optimized in the provider. - - repeated single-statement `ExecuteNonQuery()` now benefits from prepared statement reuse - - `Prepare()` now actually primes that reusable prepared statement path instead of being a no-op - -4. **Data-reader compatibility** - - UUID-backed values read as strings now come back as canonical GUID strings, which is safer for - mixed-schema or legacy-read scenarios. - -### Validation completed - -The fixes above were not left at the "looks right" stage. Validation completed includes: - -- full DecentDB .NET solution tests passing after the provider changes (`526/526`) -- targeted ADO validation passing for the prepared statement reuse work (`43/43`) -- direct ADO coverage added for prepared statement reuse -- EF type-mapping and timestamp regression coverage added/updated -- Melodee importer regression coverage for duplicate release-country rows -- latest `StreamingMusicBrainzImporterTests` run: `11/11` passing in `26s` - -### Outstanding work - -The remaining work is much narrower now: - -1. **Finish the clean monitored real-file rerun** - - the rerun was started from a compiled `Release` probe harness and then intentionally stopped - at user request to wrap the session - - final full-run metrics are still needed - -2. **Capture final full-file metrics** - - total wall time - - peak RSS / `VmHWM` - - final `.ddb` and `.wal` sizes - - post-import probe timings - -3. **Validate exact `MusicBrainzIdRaw` lookup on the rebuilt database** - - this remains the most suspicious exact-match path based on the earlier probe - -4. **Separate environment issues** - - the true CLI job path is still blocked in this shell by PostgreSQL-backed configuration access - - the `Melodee.Tests.Blazor` local-reference test-host crash remains a separate issue from the - DecentDB work itself - -## Scope of this analysis - -This analysis focused on the Melodee surfaces that currently use `DecentDB.EntityFrameworkCore` -and that are relevant to real-world user workflows: - -- `src/Melodee.Blazor/Program.cs` -- `src/Melodee.Cli/Command/CommandBase.cs` -- `src/Melodee.Blazor/Services/DoctorService.cs` -- `src/Melodee.Cli/Command/DoctorCommand.cs` -- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/MusicBrainzDbContext.cs` -- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs` -- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzArtistSearchEnginePlugin.cs` -- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzCoverArtArchiveSearchEngine.cs` -- `src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs` -- `src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/*.cs` -- `src/Melodee.Common/Jobs/ArtistSearchEngineRepositoryHousekeepingJob.cs` -- `src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs` - -It also used a temporary probe against the real MusicBrainz DecentDB file: - -- `/mnt/incoming/melodee_test/search-engine-storage/musicbrainz/musicbrainz.ddb` - -## Where Melodee uses DecentDB.EntityFramework today - -### DecentDB database registrations - -Melodee registers two DecentDB-backed EF Core contexts in both the Blazor app and the CLI: - -- `ArtistSearchEngineServiceDbContext` via `ArtistSearchEngineConnection` -- `MusicBrainzDbContext` via `MusicBrainzConnection` - -That wiring exists in: - -- `src/Melodee.Blazor/Program.cs` -- `src/Melodee.Cli/Command/CommandBase.cs` - -This is the correct high-level split. The important point is that these are not just two -copies of the same use case: - -- the MusicBrainz database is a large, read-mostly lookup store -- the artist search cache database is a smaller, mutable working set - -The provider and Melodee code should optimize differently for each one. - -## Workload 1: MusicBrainz lookup database - -The MusicBrainz DecentDB file is used by the following Melodee paths: - -- Blazor doctor checks -- CLI doctor checks -- artist lookup through `MusicBrainzArtistSearchEnginePlugin` -- cover-art lookup through `MusicBrainzCoverArtArchiveSearchEngine` -- MusicBrainz update/import job -- optional global search when `SearchInclude.MusicBrainz` is enabled - -The main repository is `DecentDBMusicBrainzRepository`. - -Its search behavior matters the most, because that is where user-facing latency appears. - -### Current MusicBrainz query strategy - -`DecentDBMusicBrainzRepository.SearchArtist(...)` does a bounded search and caps the -working result size to `10` items internally. That is a good decision for a large -file-backed store. - -The search path is currently centered on exact-match phases: - -1. exact `NameNormalized` match -2. exact reversed-name match -3. exact alias lookup through the normalized alias structure -4. exact `MusicBrainzIdRaw` match when an MBID is provided and the earlier exact phases did not - already return results -5. album fan-out query by the found `MusicBrainzArtistId` values - -This is a much better mix of shapes for a large store than what Melodee started with: - -- exact normalized-name match is good -- exact reversed-name match is good -- exact alias lookup is the correct structure for alternate names on a large file -- exact `MusicBrainzIdRaw` is still suspicious enough to deserve targeted validation -- album fan-out by `MusicBrainzArtistId` is good as long as the artist set is small -- the old substring `AlternateNames.Contains(...)` path was not kept on the default large-file - request path -- the old tokenized `Contains(...)` fallback was intentionally removed from that path - -## Workload 2: local artist search cache / "media artists" - -The local search-engine cache is backed by `ArtistSearchEngineServiceDbContext`. - -This database stores artist and album data discovered from external providers and is used by: - -- `ArtistSearchEngineService` -- the housekeeping refresh job -- CRUD/list operations for cached artist records -- doctor connectivity checks - -This database is not the same scale as the MusicBrainz materialized database. That matters. - -Melodee also has multiple protections in front of it: - -- `ArtistSearchCache` caches positive and negative lookup results -- `DirectoryRunContext.ArtistSearchCache` adds per-run single-flight caching -- the search engine plugin order checks local Melodee data before MusicBrainz - -That means the local cache database is already acting as a pressure-relief valve that keeps -many requests away from the large MusicBrainz file. - -This is good architecture and should be preserved. - -## Probe results against the real 2.31 GB MusicBrainz file - -I ran a focused probe against the real database file and measured both generic EF access and -query shapes that closely resemble Melodee's actual MusicBrainz repository logic. - -### Historical real-file timings that drove the request-path fixes - -| Operation | Cold | Warm | Assessment | +**Last updated**: 2026-06-15 +**Status**: Complete; DecentDB `2.13.1` package validation resolved DDB-002 and DDB-003, and Melodee follow-up hardening is implemented + +## Purpose + +This document is now an active implementation backlog for the remaining +DecentDB and Melodee scale work. Historical fixes that have already landed were +removed so coding agents can treat the table below as the source of truth for +remaining validation and provider follow-up. The current package baseline is +`DecentDB.AdoNet` `2.13.1`, `DecentDB.EntityFrameworkCore` `2.13.1`, and +`DecentDB.EntityFrameworkCore.NodaTime` `2.13.1`. The real-file measurements +below remain labeled with the DecentDB version or local worktree that produced +them. + +Melodee uses `DecentDB.EntityFrameworkCore` for two different workloads: + +- `MusicBrainzConnection` backs the large, read-mostly MusicBrainz materialized + database. +- `ArtistSearchEngineConnection` backs the smaller local artist search cache + used by the search-engine enrichment workflow. + +The large MusicBrainz database can support Melodee when query shapes stay +index-friendly, bounded, and explicit. The remaining work is focused on +validation, regression probes, and the smaller number of query shapes that can +still become expensive at very large scale. + +## Status Values + +| Status | Meaning | +| --- | --- | +| TODO | Implementation or documentation work is still required. | +| TODO/VERIFY | The code path may exist, but real-data proof is still missing. | +| TODO/CONDITIONAL | Implement after the dependency row proves the work is needed, or while already touching the same area. | +| PARTIAL | Some implementation or instrumentation exists, but the acceptance target is not satisfied. | +| DONE | Melodee implementation and validation work is complete. | +| UPSTREAM-FIXED | DecentDB upstream has code and regression coverage, but Melodee has not yet validated an available package against the required real-file scenario. | +| PROVIDER-FOLLOWUP | Melodee has a repro or mitigation; remaining work belongs in DecentDB provider or tooling. | + +## Active Work Table + +| ID | Status | Priority | Owner | Area | Acceptance target | +| --- | --- | --- | --- | --- | --- | +| DDB-001 | DONE | High | Melodee | Real-file MusicBrainz rebuild | Clean monitored rebuild completed against DecentDB `2.8.0`; final wall-clock, memory, `.ddb`, `.wal`, and post-import probe timings are recorded below. | +| DDB-002 | DONE | High | Melodee / DecentDB | Exact `MusicBrainzIdRaw` lookup | DecentDB `2.13.1` NuGet package validation reproduced the checkpointed real-file MBID fix: cold measured query `11.04 ms`, warm `0.56 ms`, both with `IndexSeek`. | +| DDB-003 | DONE | High | DecentDB provider | Indexed string equality | DecentDB `2.13.1` NuGet package validation reproduced checkpointed real-file indexed string equality: cold `Artist.NameNormalized` first-use hydration `9,996 ms`, warm `0.66 ms`; alias cold `780 ms`, warm `0.80 ms`; all with `IndexSeek`. | +| DDB-004 | DONE | Medium | Melodee | Local artist cache search | Local cache search now uses staged exact provider-ID, normalized-name, and normalized-alias phases; normal lookup no longer uses mixed `OR` or substring alias matching. | +| DDB-005 | DONE | Medium | Melodee | Local cache listing | Normal page requests now page in the database and avoid mandatory full count and full materialization; count-only requests still return exact counts. | +| DDB-006 | DONE | Low | Melodee | Import completion counts | Streaming import now returns materialization counts; normal completion logging no longer performs final full-table `CountAsync()` probes. | +| DDB-007 | DONE | Medium | Melodee | MusicBrainz query perf probes | `musicbrainz-query-probe` emits JSON cold/warm timings, SQL shape, DecentDB `EXPLAIN` output, sample values, row counts, and index metadata. | +| DDB-008 | DONE | Medium | Melodee | MusicBrainz import perf probes | `musicbrainz-import-probe` emits JSON phase timings, memory samples, database/WAL growth, CPU time, and importer-reported row counts. | +| DDB-009 | DONE | Medium | DecentDB provider / Melodee docs | Large-text search strategy | Melodee strategy is documented in `design/docs/decentdb-large-text-search-strategy.md`; provider candidates are listed separately. | +| DDB-010 | DONE | Medium | DecentDB provider / Melodee | Large-file diagnostics | Manual probes now expose practical app-level SQL, timing, row-count, index, memory, and file-growth diagnostics. | +| DDB-011 | DONE | Medium | Melodee | MusicBrainz DecentDB warm-up | `MusicBrainzDecentDbWarmupService` warms request-path indexes after Blazor startup and after a successful MusicBrainz database promotion. | +| DDB-012 | DONE | Medium | Melodee | Avoid broad row-existence probes | Normal query validation excludes the broad `ordered-first-row-existence` measurement by default; it is now available only through `--include-row-existence`. | +| DDB-013 | DONE | Medium | Melodee | Package-upgrade validation gate | `scripts/run-decentdb-package-upgrade-gate.sh` and `design/docs/decentdb-package-upgrade-validation-runbook.md` provide a repeatable DDB-002/DDB-003 validation path. | +| DDB-014 | DONE | Low | Melodee | Checkpoint and WAL operating guidance | MusicBrainz imports checkpoint with native .NET bindings before promotion, and the package-upgrade runbook documents checkpoint/WAL acceptance notes. | +| DDB-015 | DONE | Low | Melodee docs | Future fuzzy/full-text guidance | Exact-match and lookup-table request paths remain the accepted strategy; fuzzy/full-text search is documented as explicit future non-hot-path work. | + +## DecentDB 2.13.0 Binding Update + +Melodee consumed the DecentDB `2.13.0` .NET packages during the historical +validation run. The MusicBrainz import checkpoint path uses +`DecentDBMaintenance.CheckpointAsync(...)`, so Melodee no longer opens the +database manually for this maintenance step and does not invoke an external +DecentDB process anywhere in the repository. + +The current binding baseline covers the Melodee-requested .NET maintenance, +diagnostic, and planner surface: + +- file-path maintenance helpers for WAL status, checkpoint, compact, vacuum, + and index rebuild +- checkpoint result snapshots with before/after WAL sizes +- ADO.NET query-plan diagnostics for provider-level investigation +- clearer ADO.NET open failure diagnostics for unsupported database formats +- provider regression coverage for indexed string equality translation +- native ordered indexed equality projection support for provider-generated + `ORDER BY`, `LIMIT`, and `OFFSET` query shapes + +## DecentDB Local Worktree Validation + +Melodee was temporarily switched to direct local DecentDB binding binaries from +`/home/steven/src/github/decentdb/bindings/dotnet` while the DecentDB engine was +fixed in-place. The local native library hash used by Melodee was: + +```text +3d91afb34ed0d13d82095b61c8b535072f66770d96c69fc91c9b4f6d571ce9db +``` + +The direct ADO.NET probe against +`.tmp/decentdb-2.13-validation/musicbrainz.ddb` proved the rejected +open-time-hydration behavior was gone: + +| Direct probe step | Time | +| --- | ---: | +| `DecentDBConnection.Open()` | `1.83 ms` | +| First `Artist.NameNormalized` indexed equality | `9.19 s` | +| Same `Artist.NameNormalized` prepared parameterized equality | `3.87 ms` | +| First `Artist.MusicBrainzIdRaw` indexed equality | `8.65 s` | +| Same `Artist.MusicBrainzIdRaw` prepared parameterized equality | `0.32 ms` | + +The committed Melodee `musicbrainz-query-probe` then completed against the same +checkpointed real MusicBrainz file and wrote: +`.tmp/decentdb-local-validation/musicbrainz-query-probe-targeted.json`. + +| Query | Cold | Warm | Plan | | --- | ---: | ---: | --- | -| `DbContextOptions` build | ~22 ms | n/a | Fine | -| `new MusicBrainzDbContext(...)` | ~1 ms | ~0 ms | Fine | -| `context.Model` access | ~12 ms | ~0.4 ms | Fine | -| `Database.CanConnectAsync()` | ~13 ms | ~3 ms | Fine | -| `Artists.AnyAsync()` | ~13.3 s | ~13.0 s | Bad | -| `Artists.Take(1).CountAsync()` | ~13.3 s | ~13.0 s | Bad | -| `Artists.Select(x => x.Id).FirstOrDefaultAsync()` | ~15 ms | ~7 ms | Good | -| MusicBrainz exact normalized-name flow | ~40 ms | ~13 ms | Good | -| MusicBrainz alternate-name substring search | ~15.3 s | ~15.1 s | Historical red flag | -| MusicBrainz tokenized `Contains(...)` fallback | ~22.3 s | ~22.4 s | Historical red flag | -| MusicBrainz exact-MBID flow | ~34.6 s | ~34.2 s | Red flag | - -### What these timings mean - -These numbers show that the large `.ddb` file is not inherently too slow for Melodee. - -The store can absolutely support request-path usage when the query shape is compatible with: - -- existing indexes -- first-row short-circuiting -- small bounded result sets - -The store becomes a problem when the query shape implies: - -- full or near-full scans -- non-short-circuiting existence checks -- substring searches on large string columns -- query plans that do not make effective use of equality indexes - -That distinction is the core lesson for Melodee and for the DecentDB EF provider work. - -These measurements also matter historically even though some of those paths are no longer the -current design. They explain why the request path was changed: - -- the alternate-name substring path was replaced structurally rather than micro-optimized -- the tokenized fallback was removed from the normal large-file path -- the exact MBID path was left under explicit suspicion until rebuilt-db validation can confirm it - -## MusicBrainz use cases: what is safe and what is not - -### 1. Dashboard doctor and health checks - -### Current verdict - -**Healthy now, after the Melodee-side fix.** - -### Why - -`DoctorService` now checks MusicBrainz health with: - -- `Database.CanConnectAsync()` -- `Select(x => x.Id).FirstOrDefaultAsync()` - -That is exactly the right pattern for a large file-backed DecentDB table. - -This should remain the rule for all request-path existence checks against large DecentDB -tables: - -- do not use `AnyAsync()` for "does at least one row exist?" -- do not use `CountAsync()` for "does at least one row exist?" -- do use a projected first-row fetch such as `Select(Id).FirstOrDefaultAsync()` - -Even if the provider continues to improve, Melodee should keep this pattern because it is -the most explicit and least surprising shape for large stores. - -### 2. Exact normalized-name artist lookup - -### Current verdict - -**Good.** - -### Why - -The `Artist` materialized model has an index on `NameNormalized`, and the probe showed the -exact normalized-name flow completing in tens of milliseconds, even against the 2.31 GB file. - -That means Melodee's first and most important MusicBrainz search phase is already aligned -with the large-file reality. - -The reversed-name phase reuses the same exact-match pattern, so it should have the same -general performance characteristics. - -### 3. Album fan-out after artist match - -### Current verdict - -**Good, if kept bounded.** - -### Why - -After finding a small artist set, Melodee loads albums by: - -- collecting `MusicBrainzArtistId` values -- issuing a bounded `Contains(...)` query on `Albums` -- relying on the `MusicBrainzArtistId` index - -This is fine because: - -- the repository intentionally limits the search result set -- the album lookup is downstream of a selective artist match -- the `Album` materialized entity has an index on `MusicBrainzArtistId` - -This is a good pattern for DecentDB: narrow first, then fan out. - -### 4. Alternate-name lookup - -### Current verdict - -**Now on the correct structural path, but rebuilt databases are required to benefit fully and the -final post-rebuild real-file timing is still outstanding.** - -### Why - -The old query shape was: - -- `AlternateNames != null && AlternateNames.Contains(query.NameNormalized)` - -On the large file, that substring scan took about fifteen seconds. That was not acceptable for -interactive lookup. - -Melodee no longer has to rely on that design for the normal large-file request path. - -Instead, Melodee now materializes normalized aliases into a dedicated lookup structure and resolves -exact alias matches there. That is the right large-file shape because it turns alternate-name lookup -back into an equality lookup rather than a blob substring scan. - -### Root cause - -The original root cause was structural: - -- `AlternateNames` was stored as a single string blob -- the lookup was a substring scan over that blob -- there was no realistic indexing strategy that made that shape attractive on a huge file - -The delivered fix was also structural: - -- materialize one normalized alias per lookup row -- index the alias lookup column -- keep the request path on exact-match semantics +| `Artist.NameNormalized == value` | `9,120 ms` | `0.65 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| `ArtistAlias.NameNormalized == value` | `694 ms` | `0.75 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| `Artist.MusicBrainzIdRaw == value` | `3.10 ms` | `0.55 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | -### Recommendation +This validated the DecentDB worktree fix for DDB-002 and DDB-003 before package +publication. The DecentDB `2.13.1` package validation below reproduced these +results without local binary references and completed both items. -Keep substring scanning over `AlternateNames` off the normal request path for the large -MusicBrainz database. +## DecentDB 2.13.1 NuGet Package Validation -Preferred options: +Melodee was switched back from direct local DecentDB binding references to the +published `2.13.1` NuGet packages on 2026-06-15. The Release solution was +cleaned, restored, and rebuilt against package references, and +`project.assets.json` resolved: -1. keep the new normalized alias lookup structure as the canonical online fallback -2. rebuild existing databases so the alias lookup structure is actually populated -3. if fuzzy alias search is needed later, give it a separate opt-in structure and cost model -4. do not fall back to blob substring scans silently on the large-file request path +- `DecentDB.AdoNet` `2.13.1` +- `DecentDB.EntityFrameworkCore` `2.13.1` +- `DecentDB.EntityFrameworkCore.NodaTime` `2.13.1` -### 5. Tokenized `Contains(...)` fallback +The same checkpointed real MusicBrainz file was validated with: +`.tmp/decentdb-2.13.1-validation/musicbrainz-query-probe-targeted.json`. -### Current verdict - -**Intentionally removed from the default large-file request path.** - -### Why - -The old tokenized fallback issued substring searches like: - -- `NameNormalized.Contains(word)` -- `AlternateNames.Contains(word)` - -The probe showed roughly twenty-two seconds for this shape on the large file. That was worse than -the already-bad alternate-name substring path and clearly outside acceptable interactive latency. - -This is exactly the kind of search phase that looks user-friendly in code review but quietly -guarantees scan-heavy behavior on a very large file-backed store. - -### Recommendation - -Keep the tokenized `Contains(...)` phase out of the normal request path for the large MusicBrainz -DecentDB file. - -If Melodee wants fuzzy fallback behavior here, it should come from: - -- a separate alias/token table -- a precomputed search index -- an explicit "slow fallback" mode that is opt-in and off the main UI path - -### 6. Exact MusicBrainz ID lookup - -### Current verdict - -**Unexpectedly slow; treat as a red-flag concern until explicitly verified.** - -### Why - -This path should have been one of the best-performing paths because: - -- it is an equality lookup -- `MusicBrainzIdRaw` is indexed -- the result set is tiny - -Instead, the probe showed the exact-MBID flow taking roughly thirty-four seconds. - -That is not a normal result and should be investigated as a provider/schema/data issue rather -than accepted as normal workload behavior. - -### Additional concern - -While probing, a sampled `MusicBrainzIdRaw` value did not round-trip as a printable GUID string -in console output. I have not yet proven whether the cause is: - -- provider string decoding -- imported data representation -- console rendering of returned bytes - -However, when that observation is combined with the thirty-four-second exact-MBID lookup time, -`MusicBrainzIdRaw` becomes a high-priority verification area. - -### Recommendation - -Before Melodee leans harder on MBID-exact lookup, validate all of the following: - -1. the stored DecentDB column type for `MusicBrainzIdRaw` -2. the data produced by the importer for that column -3. provider string round-tripping for that column -4. index usage/effectiveness for equality on that column -5. whether the current `.ddb` file was created before an importer/provider fix and needs - regeneration - -Until that is resolved, exact-name lookup is more trustworthy than exact-MBID lookup in the -observed file. - -### 7. Cover-art lookup through MusicBrainz - -### Current verdict - -**Mostly acceptable, but it inherits the strengths and weaknesses of `SearchArtist(...)`.** - -### Why - -`MusicBrainzCoverArtArchiveSearchEngine` uses `SearchArtist(...)` with artist and album context -and then extracts the release-group ID for Cover Art Archive. - -That means: - -- if the query resolves through exact normalized-name match, it is fine -- if it resolves through the new exact alias lookup path, it should also be fine once the rebuilt - database has populated that structure -- the old tokenized `Contains(...)` fallback is no longer supposed to be part of the default path -- if it relies on the exact-MBID path that is currently suspicious, it can also become slow - -### Recommendation - -Keep cover-art lookup dependent on the indexed search phases as much as possible. Do not reintroduce -slow fallback search behavior here silently. - -## MusicBrainz import/update path - -### Current verdict - -**Viable as an offline workflow after multiple Melodee-side and provider-side fixes. Synthetic -validation is green again, but final full-file completion metrics are still outstanding.** - -### What is already good - -The importer is not trying to use high-level EF for every row-level operation. It uses: - -- staging tables -- batched streaming -- raw SQL for materialization -- direct command execution for alias updates -- helper-table precomputation for the hot album join path -- prepared ADO inserts for the write-heavy materialization phases - -That is the right general shape for a large import pipeline. - -### Real rebuild findings - -I ran the import path against the real staged MusicBrainz dump already present under: - -- `/mnt/incoming/melodee_test/search-engine-storage/musicbrainz/staging` - -That staging footprint was already substantial: - -- `mbdump.tar.bz2`: ~6.43 GB -- `mbdump-derived.tar.bz2`: ~455.62 MB -- extracted `mbdump/`: ~23.94 GB -- total staging footprint: ~30.81 GB - -The first live rebuild attempt did **not** fail because DecentDB could not open or write the -large `.ddb` file. Instead, it exposed a Melodee importer algorithm problem during album -materialization. - -Observed timeline from the first live run: - -- `11:49:05` - streaming artist file began -- `11:57:02` - streamed `808,816` artist links to staging -- `12:08:28` - materialized `2,792,908` artists -- `12:09:04` - materialized `7` artist relations -- `12:12:25` - artist staging cleared and album-side staging began -- `13:04:51` - streamed `5,265,580` releases and started album materialization - -At that point the process became the bottleneck: - -- RSS climbed to roughly `20 GB` -- the `.ddb` size plateaued at about `7.12 GB` -- sampled `/proc//io` counters stopped showing meaningful additional file I/O -- CPU remained heavily utilized - -That signature is important. It means the importer was mostly burning CPU and memory inside the -application process rather than being blocked on DecentDB file open or write throughput. - -That first run identified the original failure mode, but it was not the end of the investigation. -After the first streaming/batching fixes were applied, the importer became functionally correct -again and memory behavior improved, yet the performance tests were still failing badly. That led to -phase-level profiling of the importer itself. - -Phase probe results from the temporary diagnostic harness showed: - -- synthetic dataset: `1000` artists with `5` albums each -- total import time: about `69.57s` -- `Loading Artists`: about `0.30s` -- `Materializing Artists`: about `0.35s` -- `Loading Albums`: about `2.36s` -- `Materializing Albums`: about `63.54s` - -That result narrowed the remaining bottleneck to the album phase. - -The next probe split album materialization into query time versus insert time: - -- album query time: about `65.75s` -- album insert time: about `0.34s` - -That was the decisive result. Once album inserts were made cheap, the remaining hotspot was the -album materialization query shape itself, not the write loop. - -### Root cause - -The root cause turned out to be layered, not singular. - -1. **Initial failure mode** - - `DecentDBStreamingMusicBrainzImporter.MaterializeAlbumsAsync(...)` was reading the entire - album join result into one huge `List` before writing that batch to the database. - - For a real dump containing millions of releases, that creates avoidable allocation pressure, - delays persistence too long, and magnifies correctness issues such as duplicate valid - `release_country` rows. - -2. **Second-order bottleneck after the first memory fix** - - Once writes were made bounded and cheap, the hot album query still performed very poorly. - - The expensive parts were the grouped/derived release-country resolution and the primary-artist - lookup shape inside the album materialization query. - - In other words, the importer no longer had just a write-path problem. It also had a - planner-unfriendly query-shape problem. - -3. **Provider-side write overhead that was worth fixing, but was not the main bottleneck** - - repeated single-statement `ExecuteNonQuery()` calls were reparsing and re-preparing every time - before the provider change - - fixing that was the right DecentDB ADO improvement, but the phase probe proved it was not the - dominant remaining import cost once the album query was isolated - -### Fix delivered - -The importer and provider now include the following delivered fixes: - -1. **Correct raw staging parameter binding** - - while converting staging writes to raw commands, a regression was found: - `$p0`, `$p1`, ... placeholders were not recognized by the provider's parameter rewriter - - the raw staging helper was corrected to use `@p0`, `@p1`, ... placeholders instead - -2. **Prepared raw staging and album inserts** - - raw staging commands now call `Prepare()` - - album persistence moved to prepared ADO inserts with lightweight row buffering - -3. **Provider prepared statement reuse** - - `DecentDBCommand.ExecuteNonQuery()` now reuses prepared statements for repeated - single-statement non-query execution - - `Prepare()` now primes that path instead of being a no-op - -4. **Artist alternate-name materialization** - - the importer no longer performs a per-artist C# update loop to build alternate names - - bulk SQL aggregation is used instead - -5. **Album materialization batching** - - keyset pagination on `ReleaseStaging.Id` - - bounded `LIMIT`-based reads per batch - - release-country selection collapsed to the first valid row per release before album creation - - progress messages emitted after each persisted batch - -6. **Album query simplification** - - precompute `ReleaseCountryResolvedStaging` - - precompute `ArtistCreditPrimaryArtistStaging` - - join those keyed helper tables from the album materialization query instead of repeatedly - resolving those relationships in the hot query itself - -This does two things: - -1. it keeps memory bounded during the album phase -2. it removes a substantial amount of unnecessary work from the dominant album query path - -Focused validation was added or rerun to lock the important behavior in place: - -- `ImportAsync_WithDuplicateReleaseCountries_MaterializesSingleAlbumPerRelease` -- direct DecentDB ADO tests for prepared statement reuse -- importer suite validation, with the latest `StreamingMusicBrainzImporterTests` run passing - `11/11` in `26s` - -### Latest clean monitored rerun status +| Query | Cold | Warm | Plan | +| --- | ---: | ---: | --- | +| `Artist.NameNormalized == value` | `9,996 ms` | `0.66 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| `ArtistAlias.NameNormalized == value` | `780 ms` | `0.80 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| `Artist.MusicBrainzIdRaw == value` | `11.04 ms` | `0.56 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +This completes DDB-002 and DDB-003. The first cold query for a deferred B-tree +index still pays bounded first-use hydration cost, but warm and repeated +short-lived connection paths are request-safe and use the expected indexes. + +## Melodee Follow-Up Hardening + +After DecentDB `2.13.1` resolved the remaining provider behavior, Melodee +completed the application-side follow-up items: + +- Blazor startup now runs an opportunistic background + `MusicBrainzDecentDbWarmupService` pass against the configured MusicBrainz + database. +- `MusicBrainzUpdateDatabaseJob` runs the same warm-up against the newly + promoted checkpointed database before the search engine is re-enabled. +- The warm-up targets the same bounded indexed shapes used by request paths: + exact normalized artist name, exact raw MusicBrainz ID, exact alias lookup by + name, and albums by artist ID. The sample-selection step also reads one alias + for the selected artist. +- The broad first-row existence probe is no longer part of normal + `musicbrainz-query-probe` output; use `--include-row-existence` only for + explicit investigation. +- `scripts/run-decentdb-package-upgrade-gate.sh` can run package-upgrade gates + with explicit sample values from prior probe output and does not use the + DecentDB CLI. + +## DecentDB 2.13.0 Real-File Validation + +The DecentDB `2.13.0` package was validated on 2026-06-15 with the committed +Melodee probes against the real MusicBrainz dump. Melodee restored, cleaned, +and rebuilt against the published packages without code changes beyond the +central package version update. Validation used a fresh real-file rebuild so +any DecentDB file or index layout changes could participate. + +Validation artifacts: + +- Import probe: + `.tmp/decentdb-2.13-validation/musicbrainz-import-probe.json` +- ADO.NET plan capture: + `.tmp/decentdb-2.13-validation/musicbrainz-explain-plans.txt` +- Checkpointed query probes were attempted with explicit sample values, but + both targeted runs timed out before a JSON report was written. + +Real-file import and maintenance results: + +| Metric | Value | Notes | +| --- | ---: | --- | +| Import duration | `46.62 min` | `2,797.22 s`; build time excluded. | +| Imported artists / aliases / relations / albums | `2,887,383` / `492,790` / `834,044` / `3,705,849` | Counts reported by the streaming importer summary. | +| Peak working set | `20.33 GiB` | From the import probe process `PeakWorkingSetBytes`. | +| Post-import `.ddb` / `.wal` before checkpoint | `8 KB` / `5.0 GiB` | Import probe leaves the WAL uncheckpointed. | +| Native checkpoint duration | `635.93 s` | Run through DecentDB `2.13.0` ADO.NET `DecentDBMaintenance.CheckpointAsync(...)`. | +| Post-checkpoint `.ddb` / `.wal` | `2.4 GiB` / `32 B` | Checkpoint completed without DecentDB CLI usage. | + +ADO.NET `EXPLAIN` plan results: + +| Query shape | Plan capture | Plan | +| --- | ---: | --- | +| Exact normalized-name lookup | `17.28 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)`. | +| Exact alias lookup | `0.27 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)`. | +| Exact `MusicBrainzIdRaw` lookup | `0.16 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. | + +Checkpointed execution validation: + +| Targeted probe | Result | +| --- | --- | +| Explicit `MusicBrainzIdRaw` sample, with name and alias supplied | Timed out after `180 s` before the probe could write a report. | +| Explicit normalized-name sample, with alias supplied | Timed out after `180 s` before the probe could write a report. | + +At the DecentDB `2.13.0` package baseline, DDB-002 and DDB-003 therefore +remained `PROVIDER-FOLLOWUP`. A later local DecentDB worktree fix is documented +above, and DecentDB `2.13.1` package validation completed the items. No Melodee +CLI fallback or ad hoc DecentDB process invocation should be added. + +## DecentDB 2.12.0 Real-File Validation + +The DecentDB `2.12.0` package was validated on 2026-06-13 with the committed +Melodee probes against the real MusicBrainz dump. Melodee restored and built +against the published packages without code changes beyond the central package +version update. The existing production MusicBrainz database could not be +opened because it was an older DecentDB file format, so validation used a fresh +real-file rebuild. + +Validation artifacts: + +- Import probe: + `.tmp/decentdb-2.12-validation/musicbrainz-import-probe.json` +- WAL-backed query probe: + `.tmp/decentdb-2.12-validation/musicbrainz-query-probe-wal-backed.json` +- Checkpointed query probe: + `.tmp/decentdb-2.12-validation/musicbrainz-query-probe-checkpointed.json` + +Real-file import and maintenance results: + +| Metric | Value | Notes | +| --- | ---: | --- | +| Import duration | `48.72 min` | `2,922.97 s`; build time excluded. | +| Imported artists / aliases / relations / albums | `2,887,383` / `492,790` / `834,044` / `3,705,849` | Counts reported by the streaming importer summary. | +| Peak working set | `22.70 GiB` | From the import probe process `PeakWorkingSetBytes`. | +| Post-import `.ddb` / `.wal` before checkpoint | `8 KB` / `5.0 GiB` | Import probe leaves the WAL uncheckpointed. | +| Native checkpoint duration | `573.60 s` | Run through DecentDB `2.12.0` ADO.NET `DecentDBMaintenance.CheckpointAsync(...)`. | +| Post-checkpoint `.ddb` / `.wal` | `2.4 GiB` / `32 B` | Checkpoint completed without DecentDB CLI usage. | + +Real-file query results: + +| Query shape | WAL-backed warm | Checkpointed warm | Plan | +| --- | ---: | ---: | --- | +| `Database.CanConnectAsync()` | `52.15 ms` | `52.68 ms` | Not applicable. | +| Ordered first-row projection | `7,869.50 ms` | `2,918.39 ms` | `OrderedRowIdScan(table=Artist, column=Id)`. | +| Exact normalized-name lookup | `16,907.93 ms` | `10,293.83 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)`. | +| Exact alias lookup | `7,210.72 ms` | `855.80 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)`. | +| Exact `MusicBrainzIdRaw` lookup | `22,463.17 ms` | `9,398.26 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. | + +DDB-002 and DDB-003 therefore remain `PROVIDER-FOLLOWUP`. DecentDB `2.12.0` +still chooses the expected indexes, but checkpointed real-file indexed +equality on the large `Artist` table remains far outside the request-safe +acceptance target. No Melodee CLI fallback or ad hoc DecentDB process +invocation should be added. + +## DecentDB 2.11.0 Real-File Validation + +The DecentDB `2.11.0` package was validated on 2026-06-12 with the committed +Melodee probes against the real MusicBrainz dump. Melodee restored and built +against the published packages without code changes beyond the central package +version update. The package still captures `IndexSeek` plans for the tested +equality shapes, but the runtime acceptance target is not met for the large +`Artist` table. + +Validation artifacts: + +- Import probe: + `.tmp/decentdb-2.11-validation/musicbrainz-import-probe.json` +- WAL-backed query probe: + `.tmp/decentdb-2.11-validation/musicbrainz-query-probe-wal-backed.json` +- Checkpointed query probe: + `.tmp/decentdb-2.11-validation/musicbrainz-query-probe-checkpointed.json` + +Real-file import and maintenance results: + +| Metric | Value | Notes | +| --- | ---: | --- | +| Import duration | `45.68 min` | `2,740.87 s`; build time excluded. | +| Imported artists / aliases / relations / albums | `2,887,383` / `492,790` / `834,044` / `3,705,849` | Counts reported by the streaming importer summary. | +| Peak working set | `22.73 GiB` | From the import probe process `PeakWorkingSetBytes`. | +| Post-import `.ddb` / `.wal` before checkpoint | `8 KB` / `5.0 GiB` | Import probe leaves the WAL uncheckpointed. | +| Native checkpoint duration | `560.91 s` | Run through DecentDB `2.11.0` ADO.NET `DecentDBMaintenance.CheckpointAsync(...)`. | +| Post-checkpoint `.ddb` / `.wal` | `2.4 GiB` / `32 B` | Checkpoint completed without DecentDB CLI usage. | + +Real-file query results: + +| Query shape | WAL-backed warm | Checkpointed warm | Plan | +| --- | ---: | ---: | --- | +| `Database.CanConnectAsync()` | `65.94 ms` | `54.65 ms` | Not applicable. | +| Ordered first-row projection | `7,842.93 ms` | `3,199.05 ms` | `TableScan(table=Artist)`. | +| Exact normalized-name lookup | `7,896.98 ms` | `3,373.36 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)`. | +| Exact alias lookup | `5,867.64 ms` | `256.62 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)`. | +| Exact `MusicBrainzIdRaw` lookup | `7,758.18 ms` | `3,417.25 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. | + +A scratch ADO.NET probe on the same checkpointed file also measured unordered +raw SQL below EF Core: + +| Raw SQL shape | Warm timing | Plan | +| --- | ---: | --- | +| `SELECT "Id" FROM "Artist" WHERE "MusicBrainzIdRaw" = value LIMIT 1` | `3,356.3 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. | +| `SELECT * FROM "Artist" WHERE "NameNormalized" = value LIMIT 10` | `3,354.1 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)`. | +| `SELECT * FROM "ArtistAlias" WHERE "NameNormalized" = value LIMIT 10` | `242.8 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)`. | + +DDB-002 and DDB-003 therefore remain `PROVIDER-FOLLOWUP`. The remaining +DecentDB issue is below Melodee's EF translation and ordering choices: even +unordered raw ADO.NET indexed equality against the checkpointed large `Artist` +table still requires multi-second execution. No Melodee CLI fallback or ad hoc +DecentDB process invocation should be added. + +## DecentDB 2.10.0 Real-File Validation + +The DecentDB `2.10.0` package was validated on 2026-06-10 with the committed +Melodee probes against the real MusicBrainz dump. The planner/provider fix is +present: exact normalized-name, exact alias, and exact `MusicBrainzIdRaw` +queries all capture DecentDB `EXPLAIN` output with `IndexSeek`. The runtime +acceptance target is still not met for the large `Artist` table. + +Validation artifacts: + +- Import probe: + `.tmp/decentdb-2.10-validation/musicbrainz-import-probe.json` +- WAL-backed query probe: + `.tmp/decentdb-2.10-validation/musicbrainz-query-probe-wal-backed.json` +- Checkpointed query probe: + `.tmp/decentdb-2.10-validation/musicbrainz-query-probe-checkpointed.json` + +Real-file import and maintenance results: + +| Metric | Value | Notes | +| --- | ---: | --- | +| Import duration | `46.72 min` | `2,802.96 s`; build time excluded. | +| Imported artists / aliases / relations / albums | `2,887,383` / `492,790` / `834,044` / `3,705,849` | Counts reported by the streaming importer summary. | +| Peak working set | `21.04 GiB` | From the import probe process `PeakWorkingSetBytes`. | +| Post-import `.ddb` / `.wal` before checkpoint | `8 KB` / `5.1 GB` | Import probe leaves the WAL uncheckpointed. | +| Native checkpoint duration | `557.47 s` | Run through DecentDB `2.10.0` ADO.NET `DecentDBMaintenance.CheckpointAsync(...)`. | +| Post-checkpoint `.ddb` / `.wal` | `2.4 GB` / `32 B` | Checkpoint completed without DecentDB CLI usage. | + +Real-file query results: + +| Query shape | WAL-backed warm | Checkpointed warm | Plan | +| --- | ---: | ---: | --- | +| `Database.CanConnectAsync()` | `52.86 ms` | `55.59 ms` | Not applicable. | +| Ordered first-row projection | `7,763.99 ms` | `3,372.08 ms` | `TableScan(table=Artist)`. | +| Exact normalized-name lookup | `8,094.76 ms` | `3,495.39 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)`. | +| Exact alias lookup | `6,119.85 ms` | `260.08 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)`. | +| Exact `MusicBrainzIdRaw` lookup | `8,173.73 ms` | `3,626.46 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. | -After the synthetic validation went green again, a clean real-file rerun was started from a -compiled `Release` probe harness so the monitor data would represent importer work rather than -`dotnet run` build time. That rerun was intentionally stopped at user request before completion so -the session could be wrapped up. +DDB-002 and DDB-003 therefore remain `PROVIDER-FOLLOWUP`. The remaining +DecentDB issue is no longer EF translation or parameter binding. It is a +provider/runtime/storage issue where `IndexSeek` plans on the checkpointed +large `Artist` table still require multi-second execution. No Melodee CLI +fallback or ad hoc DecentDB process invocation should be added. -What was completed before stopping: +## Historical Measurements That Still Drive The Work -- the existing `musicbrainz.ddb` and `musicbrainz.ddb-wal` were deleted before the rerun -- the rerun was launched from the compiled harness DLL -- the monitored root PID was `1908758` -- the importer had cleanly started and was still in the artist-file staging phase when stopped +The following measurements came from a focused probe against a real +MusicBrainz DecentDB file of roughly `2.31 GB`. They remain useful because they +explain why the open items exist. -Partial metrics captured before stopping: +| Query shape | Prior timing | Current instruction | +| --- | ---: | --- | +| `Database.CanConnectAsync()` | ~3-13 ms | Safe connectivity probe. | +| Ordered first-row projection, for example `OrderBy(x => x.Id).Select(x => x.Id).FirstOrDefaultAsync()` | ~7-15 ms | DecentDB `2.13.1` still measured this around `2.9 s` in the checkpointed probe; keep off hot paths until a bounded row-id/limit fast path is validated. | +| `Artists.AnyAsync()` / `Artists.Take(1).CountAsync()` | ~13 s | Keep off request paths and avoid in large-table probes. | +| Exact normalized-name MusicBrainz artist lookup | ~13-40 ms | Intended primary large-file search path. DecentDB `2.13.1` validates this as complete for warm request-path use. | +| `AlternateNames.Contains(...)` substring lookup | ~15 s | Do not use blob substring scans on the large request path. | +| Tokenized `Contains(...)` fallback | ~22 s | Do not reintroduce as an automatic large-file fallback. | +| Exact `MusicBrainzIdRaw` lookup | ~34 s | DecentDB `2.13.1` validates this as complete for warm request-path use. | -- last sampled elapsed time: `00:08:00` -- last sampled total RSS: `180,948 KB` -- sampled peak `VmHWM`: `186,964 KB` -- sampled total CPU time: `478.61s` -- last sampled `.ddb` size: `591,761,408` bytes -- last sampled `.wal` size: `42,372,368` bytes -- on-disk sizes immediately after stopping: - - `.ddb`: `615,456,768` bytes - - `.wal`: `84,744,736` bytes +## DDB-001: Clean Monitored Real-File Rebuild -That partial rerun is not a substitute for final completion metrics, but it is still useful: +**Status**: DONE -- it shows the cleaned-up rerun starts correctly -- it shows healthy early-stage `.ddb`/`.wal` growth -- it shows sampled memory remaining dramatically below the earlier pre-fix high-memory plateau +The MusicBrainz import was run against the real dump to completion and the +final metrics were recorded below. The proof used DecentDB `2.8.0`. -### Concerns +Relevant entry points: -There are still a few areas to watch: +- `src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs` +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs` +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs` +- `tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs` -1. `ImportData(...)` finishes by calling `CountAsync()` on `Artists` and `Albums`. That is fine - for an offline job, but it is still expensive work on huge tables and does not belong on a - request path. -2. the fully representative CLI job path is still harder to validate end-to-end than the direct - repository harness path in this shell environment because the PostgreSQL-backed configuration - store was unreachable from the test shell. -3. the clean monitored rerun was intentionally paused at user request after about eight minutes of - artist-file staging, so the final full-run metrics are still missing. -4. `MusicBrainzIdRaw` exact lookup is still suspicious enough that it should be re-measured on the - rebuilt database before it is treated as a strong fast path. +Implementation guide: -### Recommendation +1. Use a compiled `Release` harness or a production-equivalent CLI job so build + time is not counted as import time. +2. Start from a clean target by deleting the target `musicbrainz.ddb` and + `musicbrainz.ddb-wal`. +3. Monitor the root worker PID and any child processes until import finishes. +4. Capture wall-clock time, total RSS, peak `VmHWM`, total CPU time, final + `.ddb` size, final `.wal` size, and post-import probe timings. +5. Record the final numbers in this document under "Final Real-File Metrics". -For the import pipeline, prefer: +Acceptance target: -- offline-only counts or logged inserted-row counts already known from materialization steps -- batch-limited materialization for every very-large table fan-out phase, not just album writes -- helper-table precomputation when a hot join repeatedly has to rediscover the same small keyed - relationships -- direct perf logging around major import phases so memory and I/O regressions are obvious early -- compiled-harness monitoring for real rebuild validation so build time and worker time are not - conflated +- The real-file import finishes without manual interruption. +- The final metrics table is filled in. +- Post-import probes include at least exact name, exact alias, first-row + existence, and exact MBID lookup. -This is not the top priority compared with request-path search behavior, but it is worth -tracking. +## DDB-002: Exact MusicBrainzIdRaw Lookup Validation -## Local artist search cache ("media artists") use cases +**Status**: DONE -### 1. Why this database is in a better place than MusicBrainz +Exact MBID lookup should be one of the fastest MusicBrainz paths because it is +an equality search on an indexed column. The prior probe showed roughly +`34s`, which is not acceptable for request-path use. DecentDB `2.13.0` +captures `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` through +ADO.NET `EXPLAIN`, but the checkpointed targeted MBID probe timed out after +`180 s`. -The local artist search cache is not being asked to behave like the 2.31 GB MusicBrainz file. -That matters more than any single query detail. +The DecentDB local worktree fix validated this path before package publication, +and DecentDB `2.13.1` NuGet package validation reproduced the result without +local binary references. The committed Melodee query probe measured the +checkpointed MBID query at `11.04 ms` cold and `0.56 ms` warm, both with +`IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)`. -Melodee already reduces pressure on this database through: +Relevant code: -- in-memory `ArtistSearchCache` -- per-run `SingleFlightCache` in `DirectoryRunContext` -- plugin ordering that checks Melodee/local data before MusicBrainz +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs` +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/MusicBrainzDbContext.cs` +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs` + +Validation guide: + +1. Use the rebuilt database from DDB-001. +2. Sample known artist MBIDs directly from the materialized `Artist` table. +3. Verify that `MusicBrainzIdRaw` values are canonical strings and round-trip + through the provider without binary or formatting artifacts. +4. Time direct equality lookup: + `context.Artists.Where(a => a.MusicBrainzIdRaw == mbid).OrderBy(a => a.Id).Take(1)`. +5. Time repository lookup through `SearchArtist(...)` when only an MBID is + supplied. +6. Time repository lookup when both exact name and MBID are supplied. +7. Compare cold and warm timings to exact normalized-name lookup. +8. Re-run the targeted query probe after future DecentDB package updates to + catch regressions in checkpointed indexed equality. + +Acceptance target: + +- Warm MBID lookup is in the same broad request-safe class as exact + normalized-name lookup. +- Stored values are readable canonical strings. +- DecentDB `2.13.1` reproduces the local worktree result. + +## DDB-003: Indexed String Equality Provider Follow-Up + +**Status**: DONE + +DDB-002 proved that rebuilt-file indexed string equality was still slow enough +to keep off Melodee request paths under the DecentDB `2.8.0` baseline. Exact +normalized-name and exact `MusicBrainzIdRaw` lookups both remained around +`7.6 s` warm against the clean rebuild, with exact alias lookup around `5.7 s` +warm. + +DecentDB `2.10.0` includes the concrete provider/planner follow-up from this +document: + +- simple indexed equality projections accept simple ordering, limits, and + offsets instead of falling through to slower projection paths +- the native executor has a regression that fails if ordered indexed equality + does not stay on the fast path +- the EF Core provider has a large indexed string regression for + `Where(...).OrderBy(...).Take(1)` +- ADO.NET has binding-native index rebuild helpers for maintenance workflows + +The `2.13.0` real-file validation proves those planner fixes are still present +through ADO.NET `EXPLAIN`, but the runtime target was still not satisfied for +the checkpointed large `Artist` table. Targeted checkpointed probes for both +`Artist.NameNormalized == value` and `Artist.MusicBrainzIdRaw == value` timed +out after `180 s`, even though both plans show `IndexSeek`. + +The DecentDB local worktree fix validated the runtime path before package +publication, and DecentDB `2.13.1` NuGet package validation reproduced the +fix without local binary references. The checkpointed real-file probe measured +`Artist.NameNormalized` at `9,996 ms` cold first-use hydration and `0.66 ms` +warm, `ArtistAlias` at `780 ms` cold and `0.80 ms` warm, and +`Artist.MusicBrainzIdRaw` at `11.04 ms` cold and `0.56 ms` warm. All three +plans use `IndexSeek`. + +Provider enhancement candidates are tracked in +`design/docs/decentdb-provider-enhancements.md`. + +Release validation guide: + +1. Keep the `2.13.0` import, plan, timeout, local-worktree, and `2.13.1` + package-validation artifacts attached to any future regression issue. +2. Rerun the same targeted checkpointed query probe after future DecentDB + package updates. +3. Check indexed row lookup/materialization cost for checkpointed large tables, + not only planner index selection. +4. Re-run `musicbrainz-query-probe` after the next provider/runtime fix. + +Acceptance target: + +- Equality lookup on a representative indexed string column is consistently + request-safe. +- Provider tests fail before the fix and pass after it. + +## DecentDB Provider Enhancement Candidates + +The current Melodee work exposed several DecentDB provider or .NET binding +enhancement candidates: + +- native .NET checkpoint, vacuum, compact, and WAL-status APIs published in + DecentDB `2.9.0` +- clearer WAL checkpoint status published in DecentDB `2.9.0`; documented + `WalAutoCheckpoint` guidance remains useful provider documentation +- regression coverage for equality on indexed string columns published in + DecentDB `2.9.0`; ordered indexed equality fast-path coverage and binding + index rebuild helpers published in DecentDB `2.10.0` +- opt-in query plan and index usage diagnostics through ADO.NET, published in + DecentDB `2.9.0` +- real-file DecentDB `2.13.0` validation showed `IndexSeek` plans but + checkpointed targeted probes timed out after `180 s`; DecentDB `2.13.1` + package validation completes the runtime path +- documented large-text or full-text search strategy +- clearer unsupported file-format diagnostics published in DecentDB `2.9.0`; + explicit upgrade guidance remains a provider documentation candidate + +See `design/docs/decentdb-provider-enhancements.md` for the detailed list and +the exact observations behind each item. + +## DDB-004: Local Artist Cache Search Refactor + +**Status**: DONE + +The local artist cache is smaller than MusicBrainz and has caching in front of +it, but it still contains query shapes that will not age well if the cache grows +to hundreds of thousands or millions of rows. + +Current risk areas: + +- `ArtistSearchEngineService.DoSearchAsync(...)` performs a wide mixed `OR` + query across normalized name, MusicBrainz ID, Spotify ID, and tagged + `AlternateNames.Contains(...)`. +- The second lookup after provider results repeats a similar mixed `OR` query + across several external IDs and alternate names. +- `Include(x => x.Albums)` is applied before the candidate set is known to be + small. + +Relevant code: -That means Melodee usually pays the large-file MusicBrainz cost only when the faster sources do -not already have the answer. +- `src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs` +- `src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData` -Architecturally, that is the correct direction. +Implementation guide: -### 2. Local cache search path +1. Split local cache lookup into ordered phases: + exact provider IDs, exact `NameNormalized`, exact normalized alias, then any + intentionally slower fallback. +2. Avoid `AlternateNames.Contains(...)` for the normal fast path. If alternate + names must be searchable, add or reuse a normalized alias structure. +3. Load albums only after a bounded candidate artist set has been selected. +4. Preserve current caching behavior and provider ordering. +5. Add tests proving an exact local hit does not call external providers and + that duplicate provider results do not create duplicate artists. -### Current verdict +Acceptance target: -**Acceptable today, but should be hardened before this database grows much larger.** +- The normal local-cache path no longer relies on one wide mixed `OR` query. +- The normal local-cache path does not use substring alternate-name matching. +- Album loading happens after candidate narrowing. -### Why +## DDB-005: Local Cache Listing Scale Hardening -The local cache search uses multiple indexed equality lookups: +**Status**: DONE -- `NameNormalized` -- `MusicBrainzId` -- `SpotifyId` -- various provider IDs with unique indexes +`ArtistSearchEngineService.ListAsync(...)` currently performs a total count and +then materializes the filtered artist query before applying `Skip` and `Take` +in memory. That is acceptable only while the local cache stays small. -Those are fine. +Relevant code: -However, the query also mixes those indexed predicates with: +- `src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs` -- `AlternateNames.Contains(...)` -- a wide `OR` chain -- eager `Include(x => x.Albums)` +Implementation guide: -That combination is survivable for a small working set but is not something I would want to -trust blindly once the local cache becomes large. +1. Make total count optional for UI paths that do not strictly require it. +2. Keep pagination on the database side when the provider can translate the + shape reliably. +3. If provider limitations require a workaround, prefer keyset pagination or a + two-step ID page query over full table materialization. +4. Keep album counts as a second grouped query over the page's artist IDs only. +5. Add tests for count-only requests, normal page requests, and filtered page + requests. -### Recommendation +Acceptance target: -For future-proofing, split the local cache search into staged phases the same way MusicBrainz -search should be split: +- A normal page request does not materialize all matching artists before paging. +- Total count can be skipped by callers that do not need it. +- Album counts remain limited to the returned page. -1. exact ID lookups -2. exact normalized-name lookup -3. exact alternate-name token lookup via a normalized alias structure -4. only then consider slower fallbacks +## DDB-006: Import Completion Counts -That keeps the fast path obviously index-driven. +**Status**: DONE -### 3. Local cache admin listing +The import path still uses large-table `CountAsync()` calls for completion +logging and some progress calculations. These are offline operations, so they +are lower risk than request-path queries, but they still add avoidable work on +very large databases. -### Current verdict +Relevant code: -**Fine for a small cache, but contains scale risks.** +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs` +- `src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs` -### Why +Implementation guide: -`ArtistSearchEngineService.ListAsync(...)` always does: +1. Track inserted artist, alias, album, and relation counts during + materialization. +2. Return an import summary from the streaming importer instead of requiring the + repository to count materialized tables at the end. +3. Use known staging/materialization counts for progress when they are already + available. +4. Keep any full-table verification counts behind explicit diagnostic or + validation settings. -- `CountAsync()` for total rows -- pagination with `Skip/Take` -- a correlated album count per projected artist row +Acceptance target: -That is okay for a modest working set and likely only exercised in admin tooling, but it is not -the shape I would choose if this database becomes large. +- Normal import completion logging does not need final full-table `CountAsync()` + probes. +- Diagnostic full-table counts remain available when explicitly requested. -### Recommendation +## DDB-007: MusicBrainz Query Perf Probe -If the local cache grows materially, revisit this method: +**Status**: DONE -- make total count optional when the UI does not strictly need it -- replace per-row correlated album counts with a precomputed counter or grouped query -- keep ordering aligned with indexed columns whenever possible +Add a committed probe so future agents can verify the query shapes that matter +without rebuilding ad hoc scripts each time. -This is not urgent in the same way the large MusicBrainz fallback scans are urgent, but it is -the most obvious local-cache scale concern. +Probe coverage: -### 4. Housekeeping job +- database open and `CanConnectAsync()` +- ordered first-row projection existence check +- exact normalized-name lookup +- exact alias lookup +- exact `MusicBrainzIdRaw` lookup +- any intentionally slow fallback, if one is ever reintroduced -### Current verdict +Implementation guide: -**Improved and in a reasonable place today.** +1. Put the probe somewhere explicit and non-flaky, such as a benchmark project, + a manual integration test category, or a documented CLI diagnostic. +2. Require an explicit database path so CI does not need the full MusicBrainz + file. +3. Emit machine-readable output, preferably JSON, with cold and warm timings. +4. Record sanitized sample query values and row counts. -### Why +Acceptance target: -`ArtistSearchEngineRepositoryHousekeepingJob` filters on: +- A developer can run one documented command against a target `.ddb` file and + get comparable timings for all critical query shapes. +- The probe output is suitable for attaching to future performance issues. -- `IsLocked` -- `LastRefreshed` +## DDB-008: MusicBrainz Import Perf Probe -and orders by `LastRefreshed`. +**Status**: DONE -Earlier in the session, this was one of the obvious scale mismatches in the local cache workflow. -That gap has now been addressed by ensuring the housekeeping-supporting index exists even on -already-created DecentDB files. +Extend the committed import perf harnesses so they separate import work from +build time and record the data needed to catch regressions in large MusicBrainz +imports. -### Recommendation +Current code state: -Keep the housekeeping index in place and only revisit the exact shape if the local cache workload -changes materially. If the cache grows much larger, a composite `(IsLocked, LastRefreshed)` shape -may still be worth measuring explicitly. +- `benchmarks/Melodee.Benchmarks/MusicBrainzImportBenchmarks.cs` provides + BenchmarkDotNet coverage for synthetic MusicBrainz import data. +- `tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs` + provides opt-in synthetic perf tests behind `MELODEE_RUN_PERF_TESTS=true`. +- `benchmarks/Melodee.Benchmarks/MusicBrainzImportProbe.cs` provides the + manual real-file import mode, OS-level RSS and `VmHWM` samples on Linux, + periodic `.ddb` and `.wal` growth samples, phase timings, and final counts + from importer counters. -### 5. EnsureCreated and doctor checks on the local cache DB +Probe coverage: -### Current verdict +- artist staging time +- artist materialization time +- album staging time +- album materialization time +- relation materialization time +- peak RSS / `VmHWM` +- `.ddb` and `.wal` size growth over time +- final imported row counts from known import counters -**Healthy.** +Implementation guide: -### Why +1. Run from compiled `Release` output. +2. Emit periodic metric samples to JSON or CSV. +3. Support cancellation without corrupting the metric file. +4. Keep the real-file probe manual or opt-in; retain synthetic coverage for + regular CI or local smoke checks. -The local cache DB is only checked for connectivity in doctor paths and is initialized with -`EnsureCreatedAsync()` during service startup. Those are reasonable operations for this smaller -database. +Acceptance target: -No major concern here. +- Real-file import runs can be compared across commits without relying on + shell-only monitoring scripts. +- Phase timing and memory regressions are visible from the probe output. -## What Melodee is already doing right +## DDB-009: Large-Text Search Strategy -Melodee is already doing several things that should be kept: +**Status**: DONE -1. **Separate databases for separate workloads** - - MusicBrainz is isolated from the local cache. +The large MusicBrainz request path should not silently rely on substring scans +over blob-like text fields. Decide whether this belongs in DecentDB provider +features, Melodee schema design, or explicit documentation. -2. **Bounded MusicBrainz result sets** - - `SearchArtist(...)` internally caps search result size. +Implementation guide: -3. **Caching around search** - - repeated searches are not forced back through the same path every time. +1. Decide whether DecentDB should support a full-text or token index strategy + for this class of workload. +2. If provider support is not planned, document that large substring workloads + require schema-level lookup tables. +3. Keep MusicBrainz alternate-name lookup based on normalized exact-match rows. +4. Do not reintroduce automatic substring fallback on the large request path. -4. **Local-first search ordering** - - Melodee's own data is consulted before the large MusicBrainz database. +Acceptance target: -5. **Doctor fast-path fix** - - first-render health checks now use an appropriate query shape. +- The intended approach is documented in the provider and/or Melodee docs. +- Any future fuzzy search work has an explicit schema and cost model. -6. **Offline-oriented import pipeline** - - bulk import is using staging + SQL materialization instead of naive row-by-row EF inserts. +## DDB-010: Large-File Diagnostics -These are the correct building blocks. The remaining work is mostly about making the query -shapes match the scale of the MusicBrainz file. +**Status**: DONE -## Recommended improvements by owner +Large-file plan problems should be visible before they turn into production +latency surprises. -### Melodee-side improvements +Current code state: -| Status | Priority | Area | Recommendation | -| --- | --- | --- | --- | -| Done | High | MusicBrainz request path | Keep `Select(Id).FirstOrDefaultAsync()` as the standard existence probe for large tables. | -| Done | High | MusicBrainz search | Keep substring/tokenized fallback phases off the normal large-file request path. | -| Done | High | MusicBrainz schema/search design | Keep the normalized alias lookup structure as the online alternate-name path; rebuilt databases are required to populate it. | -| Done | High | MusicBrainz import materialization | Keep album materialization batch-bounded and keep the helper-table query simplification in place. | -| Outstanding | High | MBID exact search | Treat `MusicBrainzIdRaw` lookup as suspect until verified against the rebuilt database and provider/data file. | -| Outstanding | Medium | Local cache search | Split local cache search into staged exact-match phases instead of one wide OR query. | -| Done | Medium | Local cache housekeeping | Maintain the new housekeeping index and revisit the exact composite shape only if the workload grows materially. | -| Outstanding | Medium | Local cache listing | Revisit `CountAsync()` and correlated album-count projections if the local cache grows substantially. | -| Outstanding | Low | Import completion | Prefer pre-known inserted counts over end-of-job `CountAsync()` when possible. | +- `DecentDBMusicBrainzRepository` logs phase timing for MusicBrainz artist + searches. +- `MusicBrainzUpdateDatabaseJob` logs WAL size before and after the DecentDB + checkpoint step through `DecentDBMaintenance.CheckpointAsync(...)`. +- The query and import probes expose opt-in sanitized SQL, row counts, index + metadata, query-plan output from DecentDB's ADO.NET `ExplainQuery` helper, + memory samples, file growth, and phase timings. -### DecentDB provider / binding improvements +Implementation guide: -| Status | Priority | Area | Recommendation | -| --- | --- | --- | --- | -| Done | High | Native loading / TIMESTAMP correctness | Keep the local-source native asset fix, true `TIMESTAMP` mapping, and microsecond timestamp decoding in place. | -| Done | High | Provider guidance / existence semantics | Keep the `Any()` optimization work and continue documenting that large-table request paths should still prefer explicit first-row projection. | -| Done | Medium | UUID string compatibility | Preserve canonical GUID string reads for UUID-backed values read as strings. | -| Done | Medium | Repeated non-query execution | Keep prepared statement reuse for repeated single-statement `ExecuteNonQuery()` and the now-real `Prepare()` path. | -| Outstanding | High | Equality on indexed strings | Verify index usage and string round-tripping for large indexed string columns such as `MusicBrainzIdRaw`. | -| Outstanding | Medium | Large-text search | Decide whether the provider should support additional indexing/search strategies for substring-heavy workloads, or clearly document that these require a different schema. | -| Outstanding | Medium | Diagnostics | Add easy SQL/perf diagnostics so large-file plan problems are visible sooner in app-level probes. | +1. Add or expose sanitized SQL logging for DecentDB-backed probes. +2. Include elapsed time, row count, database path or logical connection name, + and query phase name. +3. Add plan/index diagnostics if DecentDB exposes enough information. +4. Keep diagnostics opt-in so normal production logs do not become noisy. +5. Wire the diagnostics into DDB-007 and DDB-008 where practical. -## Will Melodee's current DecentDB use cases be successful and performant? +Acceptance target: -### Short answer +- A slow query report includes enough detail to distinguish a bad query shape + from provider/index behavior. +- Diagnostics can be enabled without editing source code. -**Yes for the key implemented paths, with two important caveats: the rebuilt real-file rerun still -needs final completion metrics, and exact `MusicBrainzIdRaw` lookup still needs direct validation.** +## Historical DecentDB 2.8.0 Real-File Metrics -### Use-case verdict matrix +This table records the completed DDB-001 and DDB-002 run against DecentDB `2.8.0`. -| Use case | Verdict | Notes | +| Metric | Value | Notes | | --- | --- | --- | -| Blazor dashboard doctor | **Success / performant** | Now uses a fast first-row projection. | -| CLI doctor DecentDB connectivity | **Success / performant** | Uses `CanConnectAsync()` only. | -| MusicBrainz exact normalized-name search | **Success / performant** | Real-file probe shows low-millisecond to tens-of-milliseconds performance. | -| MusicBrainz reversed-name exact search | **Likely success / performant** | Same query family as exact normalized-name. | -| MusicBrainz alternate-name lookup | **Success / structurally improved** | Now uses the normalized alias lookup path; rebuilt DBs are required and final post-rebuild timing is still outstanding. | -| MusicBrainz tokenized `Contains(...)` fallback | **Not on the default path** | Intentionally removed from the large-file request path. | -| MusicBrainz exact MBID lookup | **Questionable** | Probe showed ~34 seconds and suspicious raw-value behavior. Needs explicit validation. | -| MusicBrainz cover-art lookup | **Conditionally performant** | Good if exact name/alias phases hit; still questionable if it falls to the suspicious exact-MBID path. | -| Local artist cache search | **Success / acceptable today** | Smaller DB, cached, local-first. Still has scale concerns. | -| Local artist cache admin listing | **Success / acceptable today** | Count and correlated album count should be watched as the cache grows. | -| Local cache housekeeping | **Success / improved** | Housekeeping index is now ensured; revisit only if workload changes materially. | -| MusicBrainz import/update job | **Success / synthetic validation green** | Batch/query fixes are in place and importer tests now pass; final full-file completion metrics are still outstanding. | - -## Recommended next steps - -### Immediate next steps - -1. Resume the clean monitored real-file rerun and let it finish so this document can carry final - wall-clock, RSS, file-growth, and post-import probe metrics instead of only partial numbers. -2. Re-measure exact `MusicBrainzIdRaw` lookup on the rebuilt database before treating MBID-exact - search as a strong fast path. -3. Keep the doctor-query, normalized alias, and tokenized-fallback removals in place. Those are - now part of the intended design, not temporary experiments. -4. Record the final monitored rerun metrics back into this document and compare them directly with - the earlier bad run. - -### Near-term implementation candidates - -1. Refactor local cache search into exact-match phases instead of one mixed OR query. -2. Add a committed perf regression probe for the query shapes that matter: - - exact normalized-name search - - exact MBID search - - first-row projection existence check - - exact alias lookup - - any explicit slow fallback mode, if one is reintroduced later -3. Add a committed import perf regression probe for: - - artist materialization - - album materialization - - peak RSS - - final `.ddb` size growth over time -4. Add easier provider/app diagnostics for large-file query shape investigation. - -## Final assessment - -Melodee does **not** need to abandon DecentDB.EntityFrameworkCore for the large MusicBrainz file. -The work completed in this session actually strengthens that conclusion. Once the query shapes and -provider behavior are corrected, the large file is usable and performant for the paths that are -supposed to be interactive. - -The core issue is not "DecentDB cannot handle a large MusicBrainz file." The core issue is: - -> Melodee and the .NET bindings both needed targeted, scale-aware fixes. Once those were applied, -> the remaining problem narrowed to finishing real-file validation rather than searching for the -> root cause. - -What is completed now: - -- doctor checks are on the right path -- exact-name and exact-alias lookup design is on the right path -- tokenized substring fallback is off the normal request path -- TIMESTAMP support in the provider is fixed properly -- prepared non-query execution is better -- the importer is no longer designed around giant in-memory album buffering -- the hot album query has been simplified structurally -- synthetic importer validation is green again - -What is still outstanding is much smaller and better defined: - -- finish one clean monitored real-file rerun to completion -- capture and record the final full-file metrics -- explicitly validate `MusicBrainzIdRaw` exact lookup on the rebuilt database - -That is a much better place to end a session than where this started. The remaining risk is no -longer "we do not know what is wrong." The remaining risk is "we still need the last completion -numbers and one suspicious exact-match path verified." +| Rebuild command or harness | `dotnet benchmarks/Melodee.Benchmarks/bin/Release/net10.0/Melodee.Benchmarks.dll musicbrainz-import-probe --storage /mnt/fileserver_db_storage/melodee/search-engine-storage/musicbrainz --db /tmp/melodee-decentdb-probes/musicbrainz.ddb --output /tmp/melodee-decentdb-probes/musicbrainz-import-probe.json --clean --sample-interval-ms 10000` | Compiled `Release` output. | +| Source dump path | `/mnt/fileserver_db_storage/melodee/search-engine-storage/musicbrainz/staging/mbdump` | Existing extracted real MusicBrainz dump. | +| Target database path | `/tmp/melodee-decentdb-probes/musicbrainz.ddb` | Clean target. Existing files were deleted first. | +| Wall-clock time | `45.49 min` | `2,729.35 s`; build time excluded. | +| Imported artists / aliases / relations / albums | `2,887,383` / `492,790` / `834,044` / `3,705,849` | Counts reported by the streaming importer summary, not final `CountAsync()` probes. | +| Peak RSS / `VmHWM` | `20.68 GiB` peak process working set; manual `/proc` check observed `VmHWM` around `21 GiB` | Probe report also contains periodic memory samples. | +| Final `.ddb` size | `8 KB` | Main file stayed small because checkpointing was unavailable. | +| Final `.wal` size | `5.1 GB` | This run happened before DecentDB `2.9.0` binding maintenance helpers were consumed; Melodee now checkpoints imported databases through `DecentDBMaintenance.CheckpointAsync(...)`. | +| First-row existence probe | cold `7,419 ms`; warm `7,509 ms` | Final query probe: `/tmp/melodee-decentdb-probes/musicbrainz-query-probe-final.json`. | +| Exact normalized-name lookup | cold `7,748 ms`; warm `7,620 ms` | Indexed `Artist.NameNormalized` equality. | +| Exact alias lookup | cold `5,659 ms`; warm `5,665 ms` | Indexed `ArtistAlias.NameNormalized` equality. | +| Exact `MusicBrainzIdRaw` lookup | cold `7,718 ms`; warm `7,597 ms` | Indexed `Artist.MusicBrainzIdRaw` equality. | + +## Current Working Assumption + +Melodee does not need to abandon `DecentDB.EntityFrameworkCore` for the large +MusicBrainz file. The Melodee-side query-shape, import-count, local-cache, and +diagnostic work is implemented. DecentDB `2.13.1` provides the binding-native +checkpoint, WAL-status, query-plan, compatibility diagnostic, ordered indexed +equality, index rebuild helpers, and checkpointed deferred-index runtime fix +requested by Melodee. Melodee consumes the package references directly and the +checkpointed real-file query probe validates DDB-002 and DDB-003 as complete. diff --git a/design/DECENTDB_ISSUE_UNABLE_TO_FIX_PROMPT.md b/design/DECENTDB_ISSUE_UNABLE_TO_FIX_PROMPT.md new file mode 100644 index 000000000..4223fcbce --- /dev/null +++ b/design/DECENTDB_ISSUE_UNABLE_TO_FIX_PROMPT.md @@ -0,0 +1,489 @@ +## DecentDB Indexed Equality Performance Issue Handoff Prompt + +**Status**: Historical handoff prompt. The DecentDB local worktree fix was +validated on 2026-06-15 by wiring Melodee directly to +`/home/steven/src/github/decentdb/bindings/dotnet`, and DecentDB `2.13.1` +NuGet package validation reproduced the fix. See +`design/DECENTDB_IMPROVEMENTS.md` for the current `DONE` status. + +Use this prompt for a DecentDB coding agent working in the local DecentDB +repository at `/home/steven/src/github/decentdb`. + +## Situation + +Melodee has tried to resolve DDB-002 and DDB-003 across multiple DecentDB +package releases. The current Melodee package baseline is: + +- `DecentDB.AdoNet` `2.13.0` +- `DecentDB.EntityFrameworkCore` `2.13.0` +- `DecentDB.EntityFrameworkCore.NodaTime` `2.13.0` + +The `2.13.0` package still does not satisfy Melodee's real-file acceptance +target for DDB-002 or DDB-003. + +The important distinction is that DecentDB now chooses the expected query +plans, but the real execution remains far too slow: + +- `Artist.NameNormalized == value` reports `IndexSeek`. +- `Artist.MusicBrainzIdRaw == value` reports `IndexSeek`. +- Checkpointed execution against the real MusicBrainz-sized `Artist` table + still does not complete in a request-safe window; targeted `2.13.0` probes + timed out after `180 s`. + +This means the remaining problem is likely below EF Core translation and below +simple planner index selection. The issue appears to be in DecentDB runtime, +storage, index lookup, row materialization, sort/limit execution after an +index seek, checkpointed-file access, or a closely related path. + +Do not mark this issue fixed until the DecentDB repository has tests that fail +before the fix and pass after it, and Melodee's real-file probe proves the +checkpointed query timings are request-safe. + +## Why This Matters + +Melodee uses DecentDB for a large, read-mostly MusicBrainz materialized +database. This database is used by ingestion and artist lookup workflows. +MusicBrainz artist data is large enough that full scans or inefficient +post-index work become user-visible delays. + +DDB-002 is the exact `MusicBrainzIdRaw` lookup path. This should be one of the +fastest possible lookups because it is an equality search over an indexed +canonical identifier column. + +DDB-003 is the more general indexed string equality path. Melodee depends on +indexed normalized strings such as `NameNormalized` and normalized alias lookup +rows to avoid substring scans over large text columns. + +If exact indexed equality on a checkpointed multi-million-row table takes +multiple seconds, Melodee cannot safely put these lookups on request paths. +The application can avoid bad query shapes, but it cannot compensate for a +multi-second indexed equality execution path inside the database engine. + +## Current Melodee Evidence + +The latest Melodee validation used DecentDB `2.13.0` on 2026-06-15. + +The existing production MusicBrainz database could not be reused because it was +an older unsupported DecentDB file format. Validation therefore used a fresh +real-file rebuild from the MusicBrainz dump. + +Artifacts from the Melodee validation run: + +- Import probe: + `.tmp/decentdb-2.13-validation/musicbrainz-import-probe.json` +- ADO.NET plan capture: + `.tmp/decentdb-2.13-validation/musicbrainz-explain-plans.txt` +- Checkpointed targeted query probes were attempted, but timed out before a + JSON report was written. + +Import and checkpoint results: + +| Metric | Value | +| --- | ---: | +| Import duration in minutes | `46.62 min` | +| Import duration in seconds | `2,797.22 s` | +| Artists | `2,887,383` | +| Artist aliases | `492,790` | +| Artist relations | `834,044` | +| Albums | `3,705,849` | +| Peak working set | `20.33 GiB` | +| Post-import main `.ddb` before checkpoint | `8 KB` | +| Post-import `.wal` before checkpoint | `5.0 GiB` | +| Native checkpoint duration | `635.93 s` | +| Post-checkpoint main `.ddb` | `2.4 GiB` | +| Post-checkpoint `.wal` | `32 B` | + +The checkpoint was run through the DecentDB .NET binding: + +```csharp +await DecentDBMaintenance.CheckpointAsync(databasePath, cancellationToken); +``` + +No DecentDB CLI was used for the Melodee validation. + +## Current Failing Validation + +DecentDB `2.13.0` still reports the expected index plans: + +| Query shape | Plan capture | Plan | +| --- | ---: | --- | +| Exact normalized-name lookup | `17.28 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| Exact alias lookup | `0.27 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| Exact `MusicBrainzIdRaw` lookup | `0.16 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +The actual checkpointed execution still fails: + +| Targeted probe | Result | +| --- | --- | +| Explicit `MusicBrainzIdRaw` sample, with name and alias supplied | Timed out after `180 s`. | +| Explicit normalized-name sample, with alias supplied | Timed out after `180 s`. | + +The checkpointed values are the relevant acceptance case for Melodee because +the MusicBrainz update job checkpoints the imported database before promoting +it. + +These timeouts are not acceptable. An indexed equality lookup on one row, or a +very small candidate set, should not exceed a `180 s` validation timeout after +the file is checkpointed and reopened. + +## Important Sample Values + +The `2.13.0` query probe used this sample: + +```text +FirstArtistId: 1 +NameNormalized: 0JTQvtC60YLQvtGAINCh0LDRgtCw0L3QsA +AliasNormalized: 0KDQsNC30L3QuCDQmNC30LLQtdC00YPQstCw0YfQuA +MusicBrainzIdRaw: fadeb38c-833f-40bc-9d8c-a6383b38b1be +``` + +The exact `MusicBrainzIdRaw` value should be highly selective. If the engine +is doing significant work after the index seek, that work needs to be +explained and reduced. + +## Melodee Query Shapes + +Melodee's benchmark probe measures these EF Core query shapes. + +Ordered first-row existence: + +```csharp +context.Artists + .AsNoTracking() + .OrderBy(a => a.Id) + .Select(a => a.Id) + .Take(1) +``` + +Exact normalized artist name: + +```csharp +context.Artists + .AsNoTracking() + .Where(a => a.NameNormalized == sampleValues.NameNormalized) + .OrderBy(a => a.SortName) + .Take(10) +``` + +Exact normalized alias: + +```csharp +context.ArtistAliases + .AsNoTracking() + .Where(a => a.NameNormalized == sampleValues.AliasNormalized) + .OrderBy(a => a.MusicBrainzArtistId) + .Take(10) +``` + +Exact MusicBrainz ID: + +```csharp +context.Artists + .AsNoTracking() + .Where(a => a.MusicBrainzIdRaw == sampleValues.MusicBrainzIdRaw) + .OrderBy(a => a.Id) + .Take(1) +``` + +The probe also sends equivalent SQL to DecentDB's ADO.NET `ExplainQuery` +helper so the report includes plan diagnostics. + +Exact normalized artist name SQL shape: + +```sql +SELECT * +FROM "Artist" +WHERE "NameNormalized" = '0JTQvtC60YLQvtGAINCh0LDRgtCw0L3QsA' +ORDER BY "SortName" +LIMIT 10 +``` + +Exact MusicBrainz ID SQL shape: + +```sql +SELECT * +FROM "Artist" +WHERE "MusicBrainzIdRaw" = 'fadeb38c-833f-40bc-9d8c-a6383b38b1be' +ORDER BY "Id" +LIMIT 1 +``` + +## Captured Plans + +Checkpointed exact normalized-name plan: + +```text +Limit(limit=10, offset=none) + Sort(SortName) + Project(*) + IndexSeek(table=Artist, index=IX_Artist_NameNormalized, predicate=(NameNormalized = '0JTQvtC60YLQvtGAINCh0LDRgtCw0L3QsA')) +``` + +Checkpointed exact MusicBrainz ID plan: + +```text +Limit(limit=1, offset=none) + Sort(Id) + Project(*) + IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw, predicate=(MusicBrainzIdRaw = 'fadeb38c-833f-40bc-9d8c-a6383b38b1be')) +``` + +The `EXPLAIN` helper itself is fast. The expensive part is query execution. +For example, in the checkpointed `2.13.0` probe, the plan capture time was +about `0.16 ms` for the exact `MusicBrainzIdRaw` plan, but the actual targeted +query probe timed out after `180 s`. + +## Historical Attempts And Why They Were Not Enough + +Melodee previously validated DecentDB `2.10.0` and `2.11.0`. + +DecentDB `2.10.0` proved the planner/provider could select `IndexSeek`, but +runtime execution was still slow on the real checkpointed `Artist` table: + +| Query shape | Checkpointed warm timing | Plan | +| --- | ---: | --- | +| Ordered first-row projection | `3,372.08 ms` | `TableScan(table=Artist)` | +| Exact normalized-name lookup | `3,495.39 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| Exact alias lookup | `260.08 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| Exact `MusicBrainzIdRaw` lookup | `3,626.46 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +DecentDB `2.11.0` still did not meet the acceptance target: + +| Query shape | Checkpointed warm timing | Plan | +| --- | ---: | --- | +| Ordered first-row projection | `3,199.05 ms` | `TableScan(table=Artist)` | +| Exact normalized-name lookup | `3,373.36 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| Exact alias lookup | `256.62 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| Exact `MusicBrainzIdRaw` lookup | `3,417.25 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +DecentDB `2.12.0` still does not meet the target and appears worse for the two +large `Artist` equality paths: + +| Query shape | Checkpointed warm timing | Plan | +| --- | ---: | --- | +| Ordered first-row projection | `2,918.39 ms` | `OrderedRowIdScan(table=Artist, column=Id)` | +| Exact normalized-name lookup | `10,293.83 ms` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| Exact alias lookup | `855.80 ms` | `IndexSeek(table=ArtistAlias, index=IX_ArtistAlias_NameNormalized)` | +| Exact `MusicBrainzIdRaw` lookup | `9,398.26 ms` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +DecentDB `2.13.0` still does not meet the target: + +| Query shape | Checkpointed execution result | Plan | +| --- | --- | --- | +| Exact normalized-name lookup | Timed out after `180 s` | `IndexSeek(table=Artist, index=IX_Artist_NameNormalized)` | +| Exact `MusicBrainzIdRaw` lookup | Timed out after `180 s` | `IndexSeek(table=Artist, index=IX_Artist_MusicBrainzIdRaw)` | + +The previous fixes were therefore insufficient. Selecting `IndexSeek` is +necessary, but it is not enough. + +## Constraints + +Do not solve this in Melodee by shelling out to the DecentDB CLI. + +Melodee must use native .NET bindings for DecentDB access and maintenance: + +- `DecentDB.AdoNet` +- `DecentDB.EntityFrameworkCore` +- `DecentDB.EntityFrameworkCore.NodaTime` +- `DecentDBMaintenance` + +Do not claim the issue is fixed only because `EXPLAIN` reports `IndexSeek`. +That has already been proven and is still too slow. + +Do not accept a DecentDB change that causes broad performance regressions. +A previous DecentDB attempt degraded benchmark performance by orders of +magnitude. Any proposed fix must include targeted regression tests and must +pass the DecentDB quality and benchmark checks listed below. + +## Required DecentDB Work + +Work in `/home/steven/src/github/decentdb`. + +Investigate why checkpointed large-table indexed equality remains +multi-second even when `EXPLAIN` reports `IndexSeek`. + +Create DecentDB-side tests that reproduce the behavior without requiring the +full Melodee repository. The tests should cover at least these cases: + +1. A large table with an indexed string column and millions of rows or a + scaled-down fixture that preserves the same execution bug. +2. A highly selective exact string equality lookup that returns one row. +3. A lookup with `ORDER BY` and `LIMIT`. +4. A lookup without `ORDER BY` to isolate sort cost from index and row fetch + cost. +5. A checkpoint and reopen cycle before timing the query. +6. Verification that the selected plan is `IndexSeek`. +7. Verification that execution time is request-safe and does not regress. + +If a full millions-of-rows unit test is too expensive for normal CI, create: + +- a small deterministic regression test that fails before the fix; and +- a larger ignored/manual performance test or benchmark that can be run before + release. + +The large/manual test is still required before publishing a package that +Melodee will treat as a candidate fix. + +## Investigation Areas + +Start with these hypotheses. They may be wrong, but the final answer must +explain which were ruled out and why. + +- `IndexSeek` may find matching keys quickly but perform expensive row + materialization for each candidate. +- `IndexSeek` may scan a much larger portion of the index than the predicate + implies. +- Equality on string keys may be doing extra decode, collation, comparison, + allocation, or normalization work per row. +- The `Sort(...)` above `IndexSeek` may force materialization of many rows + before `LIMIT`, even when the equality predicate is highly selective. +- `Project(*)` may fetch large row payloads inefficiently from checkpointed + files. +- Row lookup by row id from an index entry may be slow after checkpoint or + reopen. +- WAL-backed and checkpointed files may use different access paths or caching + behavior. +- The `Artist` table may expose a different problem than `ArtistAlias` because + of row count, row width, index layout, page locality, or payload size. +- The `.coord` or checkpoint metadata path may add overhead when resolving + index entries to rows. +- The query executor may not stop early enough for `LIMIT 1` after an + equality predicate. + +The final DecentDB fix should identify the actual root cause, not only adjust +one Melodee-specific query. + +## Reproduction From Melodee + +Use this only as the final integration proof. The DecentDB repository should +also get its own focused tests. + +From `/home/steven/src/github/melodee`, with DecentDB packages updated to the +candidate package: + +```bash +dotnet restore Melodee.sln +dotnet build Melodee.sln --no-restore +``` + +Run a clean real-file import probe: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks --no-restore -- \ + musicbrainz-import-probe \ + --storage /mnt/fileserver_db_storage/melodee/search-engine-storage/musicbrainz \ + --db .tmp/decentdb-validation/musicbrainz.ddb \ + --output .tmp/decentdb-validation/musicbrainz-import-probe.json \ + --clean \ + --sample-interval-ms 10000 +``` + +Run a query probe before checkpoint: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks --no-restore -- \ + musicbrainz-query-probe \ + --db .tmp/decentdb-validation/musicbrainz.ddb \ + --output .tmp/decentdb-validation/musicbrainz-query-probe-wal-backed.json +``` + +Checkpoint through the native .NET binding. Do not use the DecentDB CLI for +Melodee validation: + +```csharp +await DecentDBMaintenance.CheckpointAsync(databasePath, cancellationToken); +``` + +Run the checkpointed query probe: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks --no-restore -- \ + musicbrainz-query-probe \ + --db .tmp/decentdb-validation/musicbrainz.ddb \ + --output .tmp/decentdb-validation/musicbrainz-query-probe-checkpointed.json +``` + +Extract timings: + +```bash +jq -r '.Measurements[] | + [.Pass, .Name, (.ElapsedMilliseconds | tostring), (.RowCount | tostring), + (.PlanDiagnostics.Lines | join(" | "))] | @tsv' \ + .tmp/decentdb-validation/musicbrainz-query-probe-checkpointed.json +``` + +## DecentDB Quality Gate + +Before claiming the DecentDB issue is fixed, run DecentDB's normal quality +checks from `/home/steven/src/github/decentdb`: + +```bash +python ./scripts/do-pre-commit-checks.py +``` + +Also run the relevant DecentDB test suites and targeted benchmarks for the +changed runtime/index/storage path. + +Because an earlier DecentDB attempt caused severe performance regressions, +also run the DecentDB benchmark coverage that can catch broad execution +slowdowns, including the Rust baseline benchmarks when applicable: + +```bash +cd /home/steven/src/github/decentdb/benchmarks/rust-baseline +``` + +Use the repository's benchmark instructions from there. Record the before and +after results or explain why a benchmark could not be run. + +## Acceptance Criteria + +The DecentDB work is not complete until all of these are true: + +- DecentDB has a focused regression test that fails before the fix and passes + after the fix. +- DecentDB has coverage for checkpoint and reopen behavior. +- DecentDB proves `IndexSeek` is still selected for the relevant queries. +- DecentDB proves the actual execution time is request-safe, not merely that + plan generation is fast. +- DecentDB quality checks pass, including + `python ./scripts/do-pre-commit-checks.py`. +- DecentDB benchmarks do not show broad performance regression. +- Melodee consumes the candidate DecentDB package through NuGet. +- Melodee restore and build pass. +- Melodee's fresh real-file import, native checkpoint, and checkpointed query + probe show DDB-002 and DDB-003 are request-safe. +- `design/DECENTDB_IMPROVEMENTS.md` is updated with the new package version, + exact timings, plan output, and the final status. +- `docs/pages/changelog.md` is updated after Melodee validation. + +## What Not To Do + +Do not close this as fixed based only on a small in-memory or uncheckpointed +database test. + +Do not close this as fixed based only on `EXPLAIN`. + +Do not close this as fixed based only on planner changes. + +Do not add a Melodee workaround that shells out to the DecentDB CLI. + +Do not add an unbounded fallback query in Melodee. + +Do not trade correctness or broad DecentDB performance for this one query +shape. + +## Desired Final Report + +When the DecentDB agent finishes, provide a report with: + +- root cause; +- files changed; +- tests added; +- benchmarks run; +- before and after DecentDB timings; +- before and after Melodee real-file timings; +- any remaining risks; +- exact DecentDB NuGet package version that Melodee should consume. + +If the root cause cannot be fixed in one pass, document the blocker precisely +and keep DDB-002 and DDB-003 open. diff --git a/design/docs/decentdb-large-text-search-strategy.md b/design/docs/decentdb-large-text-search-strategy.md new file mode 100644 index 000000000..a495f661f --- /dev/null +++ b/design/docs/decentdb-large-text-search-strategy.md @@ -0,0 +1,100 @@ +## DecentDB Large-Text Search Strategy + +**Date**: 2026-06-15 +**Status**: Needed and adopted for Melodee DecentDB-backed request paths + +## Decision + +Melodee will not use substring scans over large text or blob-like columns on +normal DecentDB-backed request paths. Large text search must use one of these +bounded shapes: + +- exact equality against indexed normalized columns +- exact equality against dedicated normalized lookup tables +- small, explicitly bounded fallback queries that are kept off hot request + paths +- a separate full-text or token index if fuzzy search becomes a product + requirement + +This is a Melodee schema and query-shape decision. DecentDB provider full-text +support may be useful in the future, but current Melodee scale work must not +depend on provider substring optimization for request safety. + +This document is still needed because `design/DECENTDB_IMPROVEMENTS.md` uses it +as the adopted guidance for DDB-009. + +## Rationale + +Real-file MusicBrainz probes showed that substring scans over alternate-name +text are not viable on large request paths. The MusicBrainz search path follows +the safer schema shape by using normalized artist names and the `ArtistAlias` +lookup table instead of scanning an `AlternateNames` string. + +The published DecentDB `2.13.0` real-file equality timings were still too slow +for hot request paths on the large `Artist` table. DDB-002 and DDB-003 consumed +the published planner/provider fixes, and DecentDB `EXPLAIN` reported +`IndexSeek`, but checkpointed warm `Artist.NameNormalized` and +`Artist.MusicBrainzIdRaw` targeted probes timed out after `180 s`. + +DecentDB `2.13.1` has since validated the indexed equality paths through the +published NuGet packages. This strategy document still defines the Melodee +query shape that should be preserved for future package updates and +regression checks. + +## Query Rules + +- Prefer normalized exact-match columns such as `NameNormalized` and + `MusicBrainzIdRaw`. +- Store alternate names as one normalized row per searchable value when the + dataset can become large. +- Apply `OrderBy` before `Take`, `Skip`, `First`, or `FirstOrDefault` when the + result shape is not naturally unique. +- Load related data only after a bounded candidate set is known. +- Do not add automatic `Contains(...)` fallback over large text fields. +- If a fuzzy fallback is required, make it opt-in, bounded, instrumented, and + absent from normal ingestion or page-render paths. + +## Diagnostics + +Use the manual MusicBrainz query probe for DecentDB-backed query validation: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks \ + -- musicbrainz-query-probe \ + --db /path/to/musicbrainz.ddb \ + --output /tmp/musicbrainz-query-probe.json +``` + +The report includes cold and warm timings, row counts, generated SQL, DecentDB +`EXPLAIN` output for the fixed probe shapes, configured index metadata, and the +sample values used for the exact-name, exact-alias, and exact-MBID probes. +The broad `ordered-first-row-existence` measurement is excluded by default; use +`--include-row-existence` only when investigating broad table access outside +normal request-path validation. + +Use the package-upgrade gate when validating a new DecentDB NuGet version: + +```bash +./scripts/run-decentdb-package-upgrade-gate.sh \ + --db /path/to/musicbrainz.ddb \ + --label 2.13.1 \ + --output-dir .tmp/decentdb-upgrade \ + --name "$MUSICBRAINZ_SAMPLE_NAME" \ + --alias "$MUSICBRAINZ_SAMPLE_ALIAS" \ + --mbid "$MUSICBRAINZ_SAMPLE_MBID" +``` + +Use the real-file import probe when validating import-scale behavior: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks \ + -- musicbrainz-import-probe \ + --storage /path/to/musicbrainz-storage \ + --db /tmp/musicbrainz.ddb \ + --output /tmp/musicbrainz-import-probe.json \ + --clean +``` + +The import report captures phase timings, imported row counts reported by the +streaming importer, process memory, Linux `VmRSS` and `VmHWM` when available, +CPU time, and DecentDB file/WAL growth samples. diff --git a/design/docs/decentdb-package-upgrade-validation-runbook.md b/design/docs/decentdb-package-upgrade-validation-runbook.md new file mode 100644 index 000000000..f67a68067 --- /dev/null +++ b/design/docs/decentdb-package-upgrade-validation-runbook.md @@ -0,0 +1,147 @@ +## Melodee DecentDB Package-Upgrade Validation Runbook + +The runbook is Melodee-only and exists to validate candidate DecentDB +package upgrades against real MusicBrainz data without modifying service code. + +## Scope + +This runbook supports: + +- Revalidating DDB-002 and DDB-003 on an existing checkpointed MusicBrainz `.ddb`. +- Rebuilding a fresh checkpointed file and collecting import/query probes when + package changes require it. +- Producing artifacts that can be attached to upstream or downstream package + regression issues. + +Do **not** use the DecentDB CLI or local DecentDB repository changes in this +phase of validation. + +## Required inputs + +- `MUSICBRAINZ_DDB_PATH`: path to an existing checkpointed MusicBrainz file, + for example `.tmp/decentdb-validation/musicbrainz.ddb`. +- `OUTPUT_DIR`: writable folder for probe artifacts. +- `PACKAGE_LABEL`: short identifier for the DecentDB package version being tested. + +Optional inputs: + +- `MUSICBRAINZ_STAGING_PATH`: path to a staging MusicBrainz dump for fresh + import rebuild runs. +- `MUSICBRAINZ_SAMPLE_NAME`: normalized artist name from a prior probe. +- `MUSICBRAINZ_SAMPLE_ALIAS`: normalized artist alias from a prior probe. +- `MUSICBRAINZ_SAMPLE_MBID`: raw MusicBrainz artist ID from a prior probe. +- `MUSICBRAINZ_QUERY_JSON`: custom path for query probe output. +- `MUSICBRAINZ_IMPORT_JSON`: custom path for import probe output. + +## Scripted execution + +Use this helper: + +```bash +./scripts/run-decentdb-package-upgrade-gate.sh \ + --db /path/to/musicbrainz.ddb \ + --label 2.13.1 \ + --output-dir .tmp/decentdb-upgrade \ + --name "$MUSICBRAINZ_SAMPLE_NAME" \ + --alias "$MUSICBRAINZ_SAMPLE_ALIAS" \ + --mbid "$MUSICBRAINZ_SAMPLE_MBID" +``` + +Optional fresh rebuild path: + +```bash +./scripts/run-decentdb-package-upgrade-gate.sh \ + --db /tmp/musicbrainz.ddb \ + --storage /mnt/fileserver_db_storage/melodee/search-engine-storage/musicbrainz \ + --label 2.13.1-candidate \ + --output-dir .tmp/decentdb-upgrade +``` + +## Command sequence (manual equivalent) + +1. Build once before repeated probes: + +```bash +dotnet restore Melodee.sln +dotnet build Melodee.sln --no-restore +``` + +2. Run optional import probe for a fresh file (skip when using an existing + checkpointed `.ddb`): + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks -- \ + musicbrainz-import-probe \ + --storage /path/to/musicbrainz \ + --db /tmp/musicbrainz.ddb \ + --output /tmp/musicbrainz-import-probe.json \ + --clean \ + --sample-interval-ms 10000 +``` + +3. Run query probe on the checkpointed target: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks -- \ + musicbrainz-query-probe \ + --db /tmp/musicbrainz.ddb \ + --output /tmp/musicbrainz-query-probe-checkpointed.json \ + --name "$MUSICBRAINZ_SAMPLE_NAME" \ + --alias "$MUSICBRAINZ_SAMPLE_ALIAS" \ + --mbid "$MUSICBRAINZ_SAMPLE_MBID" +``` + +Use `--include-row-existence` only while investigating broad table access. The +DDB-002/DDB-003 gate intentionally excludes that measurement by default because +it is not a request-safe search shape. + +## Gate acceptance criteria + +Gate output must satisfy all of these for DDB-002 and DDB-003: + +| Item | Target | +| --- | --- | +| Targeted warm `Artist.NameNormalized` lookup | `IndexSeek` plan and non-timeout execution | +| Targeted warm `Artist.MusicBrainzIdRaw` lookup | `IndexSeek` plan and non-timeout execution | +| Warm execution time (both rows above) | Request-safe target (<= 1000 ms) | +| `Exact alias` behavior | Captured and materially faster than prior checkpointed regressions | +| Database open / probe report | Probe report is written and includes row counts | + +This runbook intentionally keeps DDB-002/DDB-003 acceptance focused on +request-safe behavior against **checkpointed real-file** data rather than +WAL-backed-only outputs. + +### Planned follow-up for non-hot-path search + +- Do not use automatic fuzzy or substring matches on the large `Artist` table in hot + request paths. +- Keep fuzzy/full-text behavior as explicit future work: + bounded candidate extraction, separate indexed schema, and explicit cost + guardrails. + +## Checkpoint/WAL scheduling notes + +- Perform checkpointing as a native maintenance step after import completion in + Melodee; checkpointing is required before asserting DDB-002/DDB-003 request + safety for production-scale data. +- Checkpointed artifacts should show expected WAL reduction and stable query timing + before declaring a package-level PASS. +- If `.wal` remains large, run the query probe anyway for investigation, but + require checkpointed/reopened artifacts for acceptance. +- Keep maintenance native to Melodee and DecentDB .NET bindings (`DecentDBMaintenance` + helpers); avoid external CLI or repository-level changes. + +Example maintenance contract: + +```csharp +await DecentDBMaintenance.CheckpointAsync(databasePath, cancellationToken); +``` + +## Failure handling + +If the gate fails: + +- Keep existing DecentDB package versions unchanged in Melodee. +- Attach the produced import/query JSON, command output, and `dotnet` logs to the + upstream ticket. +- Re-run the same named outputs on the next candidate package. diff --git a/design/docs/decentdb-provider-enhancements.md b/design/docs/decentdb-provider-enhancements.md new file mode 100644 index 000000000..aa0cd7064 --- /dev/null +++ b/design/docs/decentdb-provider-enhancements.md @@ -0,0 +1,215 @@ +## DecentDB Provider Enhancement Candidates + +**Date**: 2026-06-15 +**Status**: Updated after DecentDB 2.13.1 validation; DDB-002 and DDB-003 +completed + +## Context + +Melodee now references `DecentDB.AdoNet` `2.13.1`, +`DecentDB.EntityFrameworkCore` `2.13.1`, and +`DecentDB.EntityFrameworkCore.NodaTime` `2.13.1`. The large real-file import +and query timings in `design/DECENTDB_IMPROVEMENTS.md` were rerun against +DecentDB `2.13.0` and then revalidated against DecentDB `2.13.1` on +2026-06-15. + +The current DecentDB packages include the post-`2.9.0` fixes for Melodee's +original provider-follow-up items: ordered indexed string equality remains on +the native executor fast path, EF Core has regression coverage for Melodee's +`Where(...).OrderBy(...).Take(...)` query shape, and the ADO.NET bindings +expose native index rebuild helpers. The published `2.13.0` package still did +not satisfy DDB-002 or DDB-003 because checkpointed `Artist` table indexed +equality timed out even with captured `IndexSeek` plans. + +The DecentDB `2.13.1` package removes open-time eager all-index hydration and +adds a bounded process cache for deferred runtime B-tree indexes plus paged row +locators. Melodee validation against the real checkpointed MusicBrainz file now +shows warm `Artist.NameNormalized`, `ArtistAlias.NameNormalized`, and +`Artist.MusicBrainzIdRaw` equality paths under `1 ms` with `IndexSeek`. + +This document is still needed as provider context for future DecentDB package +regression checks. The Melodee side now has repeatable manual probes: + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks \ + -- musicbrainz-import-probe \ + --storage /path/to/musicbrainz-storage \ + --db /tmp/musicbrainz.ddb \ + --output /tmp/musicbrainz-import-probe.json \ + --clean +``` + +```bash +dotnet run -c Release --project benchmarks/Melodee.Benchmarks \ + -- musicbrainz-query-probe \ + --db /tmp/musicbrainz.ddb \ + --output /tmp/musicbrainz-query-probe.json +``` + +## Melodee Package-Upgrade Validation Gate + +Use the new internal runbook and helper for package-upgrade checks: + +- `design/docs/decentdb-package-upgrade-validation-runbook.md` +- `scripts/run-decentdb-package-upgrade-gate.sh` + +The helper performs: + +- optional fresh import probe capture (from staging data) +- checkpointed query probe capture against an existing `.ddb` +- DDB-002/DDB-003 warm-query gate checks for `IndexSeek` + request-safe timings +- optional explicit sample reuse (`--name`, `--alias`, `--mbid`) so package + validation can avoid broad sampling work +- consistent output layout for attachment to future provider regressions + +This is a Melodee-only gating helper; it does not alter production workloads +or depend on DecentDB CLI usage. + +These probes give reproducible inputs for provider issues without requiring +ad hoc scripts. + +## Melodee Warm-Up And Request-Path Guardrails + +Melodee now warms the large MusicBrainz DecentDB indexes through native .NET +queries after Blazor startup and after a successful MusicBrainz database +promotion. The warm-up is opportunistic and non-fatal; if it cannot complete, +search remains available and the same indexes warm on demand. + +The warmed query shapes intentionally match request-safe repository behavior: + +- exact `Artist.NameNormalized` equality +- exact `Artist.MusicBrainzIdRaw` equality +- exact `ArtistAlias.NameNormalized` equality +- bounded albums by `MusicBrainzArtistId` + +The broad `ordered-first-row-existence` measurement remains available for +investigation with `musicbrainz-query-probe --include-row-existence`, but it is +not part of normal package gate acceptance. + +## Enhancement List + +Items marked published below are available to Melodee through the DecentDB +`2.13.1` .NET bindings. + +### Native .NET Checkpoint And Maintenance APIs + +Status: + +- Published in DecentDB `2.9.0`: checkpoint, vacuum, compact, and WAL-status + helpers. +- Published in DecentDB `2.10.0`: binding-native index rebuild helpers. +- Melodee now checkpoints imported MusicBrainz databases through + `DecentDBMaintenance.CheckpointAsync(...)` and does not execute an external + DecentDB process. + +Useful API shapes added upstream: + +```csharp +await DecentDBMaintenance.CheckpointAsync(databasePath, cancellationToken); +await DecentDBMaintenance.VacuumAsync(databasePath, cancellationToken); +await DecentDBMaintenance.CompactAsync(sourcePath, targetPath, cancellationToken); +await DecentDBMaintenance.RebuildIndexAsync(databasePath, indexName, cancellationToken); +await DecentDBMaintenance.RebuildIndexesAsync(databasePath, cancellationToken); +``` + +The rebuild helpers give Melodee a native maintenance surface if future +validation proves that a targeted rebuild workflow is useful, but they did not +make DDB-002 or DDB-003 complete by themselves. + +### WAL Checkpoint Visibility + +Current observation: + +- The DecentDB `2.13.0` real-file import finished with the main `.ddb` file at + `8 KB` and the WAL at roughly `5.0 GiB`. +- Native ADO.NET checkpoint reduced the WAL to `32 B` and grew the main `.ddb` + file to roughly `2.4 GiB`, but the checkpoint took about `636 s`. + +Requested enhancement: + +- Implemented in DecentDB `2.9.0`: `DecentDBMaintenance.GetWalStatus(...)` + and checkpoint results expose before/after WAL sizes through ADO.NET. +- Document `WalAutoCheckpoint` behavior and the connection string settings that + affect large import workloads. + +### Indexed String Equality Performance + +Current observation from the published DecentDB `2.13.0` rebuilt and +checkpointed MusicBrainz file: + +- `Artist.NameNormalized == value` targeted probe: timed out after `180 s`. +- `Artist.MusicBrainzIdRaw == value` targeted probe: timed out after `180 s`. +- The affected columns have EF model indexes, generated SQL is captured by the + query probe, and DecentDB `EXPLAIN` reports `IndexSeek` for the tested + equality shapes. +- The remaining package issue is not index selection: the probe captures + `IndexSeek` plans while execution does not complete in a request-safe window + on the checkpointed large `Artist` table. + +DecentDB `2.13.1` package observation: + +- `Database.CanConnectAsync()`: `16.57 ms` cold. +- `Artist.NameNormalized`: `9,996 ms` cold first-use hydration and `0.66 ms` + warm. +- `ArtistAlias.NameNormalized`: `780 ms` cold and `0.80 ms` warm. +- `Artist.MusicBrainzIdRaw`: `11.04 ms` cold and `0.56 ms` warm. +- All three indexed equality plans use `IndexSeek`. + +Requested enhancement: + +- Published in DecentDB `2.9.0`: .NET provider regression coverage for indexed + string equality translation. +- Published in DecentDB `2.10.0`: the native executor accepts simple + `ORDER BY`, `LIMIT`, and `OFFSET` clauses for indexed equality projections, + and EF Core regression coverage executes Melodee's + `Where(...).OrderBy(...).Take(1)` shape. +- Published in DecentDB `2.13.1`: DecentDB keeps deferred-table open lazy, + hydrates only the requested secondary index on first use, and reuses that + runtime index plus paged row locators across short-lived connections in the + same process. + +### Query Plan And Index Diagnostics + +Current observation: + +- `ToQueryString()` gives useful SQL shape for EF queries. +- Melodee's manual query probe now records both EF-generated SQL and DecentDB + `EXPLAIN` output for the fixed probe shapes. + +Requested enhancement: + +- Implemented in DecentDB `2.9.0`: ADO.NET exposes an opt-in query-plan helper. +- Implemented in Melodee: `musicbrainz-query-probe` records generated SQL, + elapsed time, row counts, sample values, EF model index metadata, and + structured plan diagnostics from the DecentDB ADO.NET query-plan helper. +- Keep plan diagnostics opt-in and safe for app-level probe output. + +### Large Text Search Guidance Or Full-Text Support + +Current observation: + +- Melodee removed substring scans from large request paths and uses normalized + lookup tables instead. +- This is safe, but it pushes fuzzy search responsibility to application + schema design. + +Requested enhancement: + +- Document DecentDB's recommended pattern for large substring or fuzzy text + workloads. +- If provider support is planned, expose token/full-text indexes with clear + query-shape guidance. + +### Database Format Compatibility Diagnostics + +Current observation: + +- An existing MusicBrainz database failed to open with + `unsupported database format version: 11` after the DecentDB `2.8.0` upgrade. + +Requested enhancement: + +- DecentDB `2.9.0` improves the ADO.NET open failure message for unsupported + file format versions. +- Provide clearer upgrade guidance for old file format versions. +- Expose a supported upgrade or migration path when possible. diff --git a/docs/VERSION b/docs/VERSION index 7ec1d6db4..ac2cdeba0 100644 --- a/docs/VERSION +++ b/docs/VERSION @@ -1 +1 @@ -2.1.0 +2.1.3 diff --git a/docs/_config.yml b/docs/_config.yml index 7df690ef8..1a32f82bf 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -47,11 +47,13 @@ version_params: # Allow these versions to be searched search_versions: + - 2.1.0 - 2.0.0 - 1.8.0 - 1.7.0 - latest: 2.0.0 + latest: 2.1.0 versions: + - 2.1.0 - 2.0.0 - 1.8.0 - 1.7.0 diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index 14cca752e..796955b14 100644 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -1,6 +1,6 @@ - title: About url: about -- title: Documentation - url: docs - title: Homelab url: homelab +- title: News + url: news diff --git a/docs/_data/toc.yml b/docs/_data/toc.yml index 76a3270f4..585854a3f 100644 --- a/docs/_data/toc.yml +++ b/docs/_data/toc.yml @@ -20,6 +20,8 @@ url: "hardware" - title: "Backup & Recovery" url: "backup" + - title: "DecentDB Usage & Migration" + url: "decentdb" - title: Core Concepts links: diff --git a/docs/_posts/2026-05-24-melodee-2-1-0-released.md b/docs/_posts/2026-05-24-melodee-2-1-0-released.md index 0b32079e6..478278340 100644 --- a/docs/_posts/2026-05-24-melodee-2-1-0-released.md +++ b/docs/_posts/2026-05-24-melodee-2-1-0-released.md @@ -13,7 +13,7 @@ Melodee 2.1.0 is here! This release focuses on performance, developer experience ## What's New in 2.1.0 ### SkiaSharp Image Processing -All image processing has migrated from `SixLabors.ImageSharp` to `SkiaSharp`, replacing the legacy library with a more performant and actively maintained alternative. +All image processing has migrated to `SkiaSharp`, replacing the legacy library with a more performant and actively maintained alternative. - New `IImageProcessor` abstraction for decode, encode, resize, format detection, and hash generation - Clean dependency injection throughout services, Blazor components, CLI commands, and tests diff --git a/docs/pages/changelog.md b/docs/pages/changelog.md index 4c52e6612..61ef6e834 100644 --- a/docs/pages/changelog.md +++ b/docs/pages/changelog.md @@ -21,20 +21,167 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [Unreleased] + +### Added + +- Added public DecentDB usage and migration documentation covering Melodee's + generated search databases and rebuild steps for unsupported file-format + errors. + +### Fixed + +- Admin dashboard and login health warnings now open-check MusicBrainz and + ArtistSearch DecentDB files and link to the migration guide when unsupported + DecentDB file-format versions are detected. + +## [2.1.3] - 2026-06-15 + +### Added + +- Added MusicBrainz DecentDB index warm-up after Blazor startup and after + successful MusicBrainz database promotion, using native .NET queries against + the request-path indexed lookup shapes. +- Added an internal DecentDB package-upgrade validation gate and runbook for + repeatable DDB-002/DDB-003 checks against checkpointed MusicBrainz data. + +### Changed + +- Upgraded `DecentDB.AdoNet`, `DecentDB.EntityFrameworkCore`, and + `DecentDB.EntityFrameworkCore.NodaTime` to `2.13.1`. +- MusicBrainz query probes now keep the broad ordered first-row existence + measurement opt-in via `--include-row-existence` so default validation stays + focused on request-safe indexed lookups. +- Replaced the legacy IdSharp metadata fallback with Melodee's native ID3 tag + reader, removing the obsolete transitive image-processing dependency path. +- Added explicit private `MessagePack` references for NBomber consumers so the + test and benchmark graphs resolve the non-vulnerable package version. + +### Fixed + +- Validated DecentDB `2.13.1` NuGet packages against the checkpointed real + MusicBrainz query probe, completing DDB-002 and DDB-003 with `IndexSeek` + plans and sub-millisecond warm indexed equality timings. +- MusicBrainz DecentDB startup warm-up no longer runs the alias-by-artist + bounded query shape that DecentDB `2.13.1` rejects with a missing-parameter + error. + +## [2.1.2] - 2026-06-10 + +### Added + +- Added DecentDB MusicBrainz import and query probes for real-file performance diagnostics, + including JSON phase timings, row counts, memory samples, WAL growth, SQL shape, + DecentDB `EXPLAIN` output, and cold/warm lookup timings. +- Added internal DecentDB search strategy and provider enhancement notes covering ADO.NET + maintenance APIs, WAL visibility, query diagnostics, large indexed string equality, and + large-text search guidance. + +### Changed + +- Upgraded `DecentDB.AdoNet`, `DecentDB.EntityFrameworkCore`, and + `DecentDB.EntityFrameworkCore.NodaTime` to `2.13.0`. +- Recorded DecentDB `2.13.0` real-file MusicBrainz validation. The package + provides indexed equality `EXPLAIN` plans for the tested query shapes, but + DDB-002 and DDB-003 remain provider follow-up because checkpointed large + `Artist` table equality probes still time out. +- Validated a local DecentDB worktree fix for DDB-002 and DDB-003 through + direct local binding binaries; the items remain pending until a published + DecentDB NuGet package reproduces the real-file probe results. +- MusicBrainz DecentDB imports now return materialized row counts and keep final full-table + verification counts opt-in, avoiding redundant full-table counts during normal imports. +- Local artist cache lookups now use staged exact identifier, normalized name, and normalized + alias queries, with database-side paging for artist list requests. +- Local artist aliases now use a normalized lookup table that is backfilled on startup for + existing cache data and synchronized when cached artists change. + +### Fixed + +- Release editing no longer triggers EasyMDE's Font Awesome CDN stylesheet load, + preventing Content Security Policy violations in the browser console. +- The Blazor shell now loads the EasyMDE script only once. +- Admin dashboard doctor checks no longer emit Entity Framework warnings for + unordered row-limiting probes. +- Exact MusicBrainz ID lookups now apply deterministic ordering before row limiting. +- DecentDB improvement tracking now separates completed Melodee changes from provider + enhancement candidates. +- DecentDB improvement tracking now distinguishes the DecentDB `2.13.0` + planner/provider fix from the remaining large-file runtime/storage follow-up. +- MusicBrainz database imports now checkpoint through the DecentDB + `DecentDBMaintenance.CheckpointAsync(...)` API instead of an external process. + +## [2.1.1] - 2026-05-25 + +### Changed + +- Updated the docs release dropdown so the latest documentation track points to `2.1.0`, while patch releases continue to use the `2.1.x` application version line. +- Replaced the top-navbar `Documentation` link with `News` so release posts are easier to find from the docs site. +- Expanded `mcli library scan` storage-transfer reporting to separate ready albums, newly moved albums, albums merged with existing storage, duplicate-prefixed staging directories, failed metadata loads, and albums left in staging with their validation reason counts. +- Added `mcli library scan` performance reporting for artist lookup cache behavior, conversion time, copy time, revalidation skips, and DecentDB artist-search persistence retry counts. +- `mcli library scan` now shows live progress messages and item counts for inbound processing, staging revalidation, storage transfer, and database insert work instead of leaving active steps at an apparent 0%. +- `mcli library scan` now suppresses ATL library stack traces during progress rendering and reports non-fatal inbound processing errors as scan warnings instead of letting raw exception text corrupt the TUI. +- Artist search database read/open errors, including non-retryable DecentDB provider failures, are now counted in full-scan performance output and reported as scan warnings. +- `mcli doctor` now validates DecentDB files by checking file presence, opening the configured database, inspecting expected schema tables, and running read queries instead of relying on shallow connection checks. +- Bounded concurrent media conversions during inbound processing to reduce CPU and disk saturation on large batch scans. +- Staging artist revalidation now uses a staging-local `.melodee-revalidation.ddb` retry state database so repeated scans defer recently failed albums instead of re-querying every invalid staged release on every run; the state database is recreated automatically if missing or corrupt. +- Staging artist revalidation now logs the retry state database path and row counts, and `mcli library scan` reports artist lookup attempts and no-match counts for revalidation work. +- MusicBrainz DecentDB artist searches now report phase timings and use a compact release/alias loading path for one-result ingestion lookups. + +### Fixed + +- Prevent inbound processing from deleting release directories through directory event scripts; releases now remain available for conversion, normalization, validation, and staging according to the documented ingestion pipeline. +- Preserve copied cover images in staged album metadata after inbound processing renames images to Melodee's normalized `i-##-Type.jpg` filenames. +- Revalidation during the full scan workflow now bypasses same-run negative artist lookup cache entries, allowing albums to become valid when artist metadata is discovered later in the ingestion pass. +- iTunes artist matches now count as trusted artist identities alongside Spotify and MusicBrainz matches, allowing iTunes-only artist lookups to validate staged albums and participate in storage insert matching. +- Staging revalidation now clears stale invalid-artist statuses when an album already contains a trusted artist identity, and iTunes-only artist IDs can be used for storage directory naming. +- Dashboard loading now updates the layout spinner through the layout notification event and clears the global spinner when leaving the page. +- Inbound move-mode processing now removes source sidecar metadata files such as `.sfv`, `.nfo`, `.m3u`, `.cue`, and Blackbeard provenance after albums are staged, including metadata-only directories left by earlier runs. +- Inbound staging now tracks media files converted during processing, so NFO-derived albums that start as FLAC are staged from the converted MP3 files instead of leaving converted songs behind in `inbound`. +- Storage-transfer chaining now treats albums merged into existing storage directories as handled work, so the next ingestion step can continue after merge-only batches. +- Full-scan artist lookup work now shares one run-scoped cache across inbound processing and staging revalidation, including forced revalidation lookups. +- Artist search persistence now retries transient DecentDB transaction conflicts while surfacing non-retryable DecentDB provider errors without retrying them. +- Compound release artists such as `Artist One feat. Artist Two` now get conservative fallback artist lookups when the fallback candidate has trusted identity data and matching release evidence. +- Staging revalidation now skips albums whose artist metadata is blank or obviously unsearchable instead of repeatedly calling external artist providers. +- Forced staging revalidation no longer expands compound artist names into multiple fallback provider searches, preventing `mcli library scan` from appearing hung on batches of invalid collaboration artists. +- Move-mode inbound cleanup now removes source residue files such as release artwork and `.txt` notes only after media files are gone, allowing empty inbound release directories to be removed without deleting unprocessed media. +- Inbound processing now defers directories whose files are still changing instead of partially staging releases while the source copy is still in progress. +- Media conversion now accepts a valid generated MP3 when ffmpeg produced usable output but ATL reports an unexpected format label, preventing converted tracks from being stranded in inbound. +- External artist provider searches now honor bounded requested result limits instead of requesting unbounded provider result sets during forced lookups. +- iTunes artist searches now deserialize large Apple artist, collection, AMG, and genre identifiers without failing, and artist image searches correctly send the requested result limit. +- NFO parsing now ignores malformed track lines and missing artist metadata without emitting parser stack traces. +- Inbound staging now reports missing staged files once per album when skipping tag updates instead of generating repeated per-song update warnings. + ## [2.1.0] - 2026-05-24 ### Changed -- **Replaced `SixLabors.ImageSharp` with `SkiaSharp`** for all image processing operations. A new `IImageProcessor` abstraction centralizes decode, encode, resize, format identification, and average-hash computation. `ImageHasher`, `ImageConvertor`, and `ImageValidator` now receive `IImageProcessor` via dependency injection rather than using static library calls. All services, Blazor components, CLI commands, and test constructors were updated consistently. SkiaSharp native assets are included conditionally (`SkiaSharp.NativeAssets.Linux` on Linux) so builds work across platforms without extra runtime dependencies. +- **Replaced the legacy image processing library with `SkiaSharp`** for all image processing operations. + A new `IImageProcessor` abstraction centralizes decode, encode, resize, format + identification, and average-hash computation. `ImageHasher`, `ImageConvertor`, and + `ImageValidator` now receive `IImageProcessor` via dependency injection rather than + using static library calls. All services, Blazor components, CLI commands, and test + constructors were updated consistently. SkiaSharp native assets are included + conditionally (`SkiaSharp.NativeAssets.Linux` on Linux) so builds work across + platforms without extra runtime dependencies. - Set min-width on album detail action column for layout stability - Remove unnecessary EnsureArtistAliasTableAsync call in MusicBrainz repository - Dashboard data loading moved from `OnInitializedAsync` to `OnAfterRenderAsync` so skeleton placeholders render immediately instead of blocking the initial page render. - Bulk delete operations in `SongService`, `AlbumService`, and `ArtistService` now batch-load entities in a single query instead of executing N+1 queries per item, significantly improving performance for large deletions. - Quartz job scheduling extracted into `QuartzSchedulerExtensions.ScheduleJobIfConfigured` helper method, reducing `Program.cs` from 1,046 to ~860 lines and eliminating ~150 lines of repetitive scheduling code. -- **Squashed 55 EF Core migrations into a single `InitialBaseline` migration.** The migration history (107 files spanning Feb 2025 – Jan 2026) has been consolidated into one baseline file that generates the complete current schema. This reduces repository size, speeds up CI builds, and eliminates fragile migration chains. Existing databases that have already applied the latest migration are unaffected; new setups will apply only the single baseline. +- **Squashed 55 EF Core migrations into a single `InitialBaseline` migration.** The + migration history (107 files spanning Feb 2025 – Jan 2026) has been consolidated into + one baseline file that generates the complete current schema. This reduces repository + size, speeds up CI builds, and eliminates fragile migration chains. Existing databases + that have already applied the latest migration are unaffected; new setups will apply + only the single baseline. - Added `.kilo/` project configuration with slash commands (`/build`, `/test`, `/test-mql`, `/lint`, `/migrate`, `/coverage`) and a project-aware `melodee-developer` agent for consistent developer workflows. - Dropped JavaScript/TypeScript from CodeQL analysis — the repository contains only minimal JS files (jQuery, lunr.js in docs site), and scanning them wasted ~5–10 minutes per CI run with no security value. -- **Refactored `PartyModeService` to call domain services directly instead of making HTTP requests to the same application.** Replaced `HttpClient` with `PartySessionService`, `PartyQueueService`, `PartyPlaybackService`, and `PartySessionEndpointRegistryService` via dependency injection. User identity resolved through `IAuthService.CurrentUser` rather than cookie auth. Eliminates ~20 HTTP round-trips per user interaction (create, join, leave, queue, playback, endpoints) in party mode Blazor components. +- **Refactored `PartyModeService` to call domain services directly instead of making HTTP + requests to the same application.** Replaced `HttpClient` with `PartySessionService`, + `PartyQueueService`, `PartyPlaybackService`, and `PartySessionEndpointRegistryService` + via dependency injection. User identity resolved through `IAuthService.CurrentUser` + rather than cookie auth. Eliminates ~20 HTTP round-trips per user interaction (create, + join, leave, queue, playback, endpoints) in party mode Blazor components. ### Security diff --git a/docs/pages/cli/doctor.md b/docs/pages/cli/doctor.md index 3ff005992..6e2132e50 100644 --- a/docs/pages/cli/doctor.md +++ b/docs/pages/cli/doctor.md @@ -29,8 +29,8 @@ mcli doctor [OPTIONS] - Validates required connection strings exist 2. **Database connectivity** - Postgres (`DefaultConnection`) - - MusicBrainz SQLite (`MusicBrainzConnection`) - - ArtistSearchEngine SQLite (`ArtistSearchEngineConnection`) + - MusicBrainz DecentDB (`MusicBrainzConnection`) + - ArtistSearchEngine DecentDB (`ArtistSearchEngineConnection`) 3. **Library paths** - Ensures each configured library path exists - Optionally validates write access with `--write-test` @@ -60,8 +60,8 @@ mcli doctor [OPTIONS] ``` ✓ Configuration ✓ Database: Postgres -✓ Database: MusicBrainz (SQLite) -✓ Database: ArtistSearchEngine (SQLite) +✓ Database: MusicBrainz (DecentDB) +✓ Database: ArtistSearchEngine (DecentDB) ✓ Libraries All checks passed. diff --git a/docs/pages/decentdb.md b/docs/pages/decentdb.md new file mode 100644 index 000000000..1080d3311 --- /dev/null +++ b/docs/pages/decentdb.md @@ -0,0 +1,218 @@ +--- +title: DecentDB Usage & Migration +description: How Melodee uses DecentDB, how to diagnose compatibility issues, and how to rebuild generated DecentDB search databases. +permalink: /decentdb/ +tags: + - decentdb + - migration + - troubleshooting +--- + +## Overview + +Melodee uses DecentDB for local, generated search databases. These files are +separate from the primary Melodee PostgreSQL database and can be rebuilt from +source data when necessary. + +DecentDB is currently used for: + +- **MusicBrainz search data**: the local MusicBrainz artist, alias, relation, + and album lookup database. +- **Artist search cache**: the local artist search repository used to speed up + artist matching and enrichment. + +Melodee does not store user accounts, playlists, play history, ratings, or +library metadata in these DecentDB files. Those records live in the primary +PostgreSQL database. + +## Configuration + +The DecentDB databases are configured with these connection strings: + +| Connection string | Purpose | +|-------------------|---------| +| `MusicBrainzConnection` | Local MusicBrainz lookup database | +| `ArtistSearchEngineConnection` | Local artist search cache database | + +In container deployments, the same values can be supplied with environment +variables: + +```bash +ConnectionStrings__MusicBrainzConnection="Data Source=/app/storage/_search-engines/musicbrainz/musicbrainz.ddb" +ConnectionStrings__ArtistSearchEngineConnection="Data Source=/app/storage/_search-engines/artistSearchEngine.ddb" +``` + +Use your configured paths as the source of truth. Older installations may use +`.db` filenames while newer examples use `.ddb`; the connection string path is +what matters. + +## Doctor Compatibility Checks + +Doctor opens both DecentDB files with the current Melodee DecentDB provider. If +the file was created by a newer or incompatible DecentDB engine, Doctor reports +an issue similar to: + +```text +MusicBrainz DecentDB database uses a file format that is not supported by the current DecentDB provider. +Provider error: unsupported DecentDB file format version 11 +``` + +This means the current Melodee process cannot safely read that generated search +database. Search and enrichment may continue in degraded mode, but the affected +database should be rebuilt or the Melodee/DecentDB package version should be +updated. + +## Migration Strategy + +For Melodee's generated DecentDB files, migration usually means rebuilding the +affected generated database with the current Melodee version. This is safer than +trying to manually edit a DecentDB file created by an incompatible provider. + +Use this order: + +1. Upgrade Melodee to the intended version. +2. Stop Melodee so no process is writing to the DecentDB files. +3. Back up the affected `.ddb` file and any companion WAL or shared-memory + files. +4. Move the incompatible generated database out of the active path. +5. Start Melodee. +6. Rebuild the affected database from Melodee. +7. Run Doctor again and confirm the DecentDB checks pass. + +## Backup Before Rebuild + +Back up the main database and generated DecentDB files before changing anything. +The example below preserves common DecentDB companion file names. + +```bash +#!/usr/bin/env bash +set -euo pipefail + +backup_root="$HOME/melodee-decentdb-backup-$(date +%Y%m%d-%H%M%S)" +mkdir -p "$backup_root" + +musicbrainz_db="/path/to/search-engine-storage/musicbrainz/musicbrainz.ddb" +artist_search_db="/path/to/search-engine-storage/artistSearchEngine.ddb" + +backup_decentdb_file() { + local db_path="$1" + local db_name + db_name="$(basename "$db_path")" + + for suffix in "" ".wal" "-wal" ".shm" "-shm"; do + if [ -f "${db_path}${suffix}" ]; then + cp -a "${db_path}${suffix}" "$backup_root/${db_name}${suffix}" + fi + done +} + +backup_decentdb_file "$musicbrainz_db" +backup_decentdb_file "$artist_search_db" + +echo "Backed up DecentDB files to $backup_root" +``` + +Also back up PostgreSQL before a Melodee upgrade. See +[Backup & Recovery](/backup/) for full backup guidance. + +## Example: Rebuild Incompatible DecentDB Files + +This example renames incompatible generated DecentDB files so Melodee can create +fresh files with the current provider. + +```bash +#!/usr/bin/env bash +set -euo pipefail + +stamp="$(date +%Y%m%d-%H%M%S)" + +musicbrainz_db="/path/to/search-engine-storage/musicbrainz/musicbrainz.ddb" +artist_search_db="/path/to/search-engine-storage/artistSearchEngine.ddb" + +move_decentdb_file_aside() { + local db_path="$1" + + for suffix in "" ".wal" "-wal" ".shm" "-shm"; do + if [ -f "${db_path}${suffix}" ]; then + mv "${db_path}${suffix}" "${db_path}${suffix}.unsupported-$stamp" + fi + done +} + +# Stop Melodee before moving active database files. +# podman compose down +# docker compose down + +move_decentdb_file_aside "$musicbrainz_db" +move_decentdb_file_aside "$artist_search_db" + +# Start Melodee again. +# podman compose up -d +# docker compose up -d +``` + +After the files are moved aside, rebuild the generated databases. + +### MusicBrainz + +Use one of these options: + +- In the web UI, go to **Admin > Doctor** and use **Generate MusicBrainz + Database**. +- In the web UI, go to **Admin > Jobs** and run + `MusicBrainzUpdateDatabaseJob`. +- From the CLI, run: + +```bash +./mcli job musicbrainz-update +``` + +MusicBrainz rebuilds can take a long time because Melodee downloads and imports +the MusicBrainz dump into a local DecentDB file. + +### Artist Search Cache + +Use one of these options: + +- In the web UI, go to **Admin > Jobs** and run + `ArtistSearchEngineRepositoryHousekeepingJob`. +- From the CLI, run: + +```bash +./mcli job artistsearchengine-refresh +``` + +The artist search cache is generated from Melodee's library data and configured +artist search providers. It can be rebuilt after the incompatible file is moved +aside. + +## Verify The Migration + +Run Doctor after rebuilding: + +```bash +./mcli doctor --verbose +``` + +Or open **Admin > Doctor** in the web UI. + +The following checks should pass: + +- `MusicBrainzDatabase` +- `ArtistSearchEngineDatabase` + +If Doctor still reports an unsupported DecentDB file format after rebuilding, +confirm that: + +- Melodee is running the version you expect. +- The active connection strings point to the rebuilt files. +- No old `.wal`, `-wal`, `.shm`, or `-shm` companion files remain beside the + rebuilt database. +- The app container or service was restarted after the rebuild. + +## What Not To Delete + +Do not delete the primary PostgreSQL database when resolving DecentDB search +cache compatibility issues. The unsupported DecentDB file-format warning applies +to generated local search databases, not the primary Melodee database. + diff --git a/docs/pages/scripting.md b/docs/pages/scripting.md index 31d265723..842578713 100644 --- a/docs/pages/scripting.md +++ b/docs/pages/scripting.md @@ -79,7 +79,7 @@ is displayed to the user explaining why the action was denied. | `settingKey` | string | The settings key, e.g. `script.userLoginStart` | | `timeoutMs` | number | The configured timeout in milliseconds | | `maxStatements` | number | The configured statement limit | -| `onDeny` | string | Host action when result is `false` (`skip`, `delete`, `quarantine`) | +| `onDeny` | string | Host action when result is `false` (`skip` or `quarantine` for ingestion events) | | `isOverride` | boolean | Whether an override (library/path) matched | | `libraryId` | number\|null | The matched override library ID (directory events only) | | `pathPrefix` | string\|null | The matched override path prefix (directory events only) | @@ -109,7 +109,7 @@ Each `script.` setting value is a JSON document. The conceptual schem "enabled": true, "libraryId": 1, "pathPrefix": "Incoming/", - "onDeny": "delete", + "onDeny": "skip", "body": "function check(ctx, scriptConfig) { return ctx.mediaFilesCount >= 3; }" } ] @@ -144,11 +144,12 @@ Melodee treats scripts as untrusted code (defense in depth): - Statement limit (`maxStatements`) - Failures default to allow and are logged using the settings key and script hash (not the full script body). -### Directory deletion and dry-run +### Directory actions and dry-run -Directory deletion is constrained to safe roots. You can also enable dry-run mode: +Inbound ingestion does not physically delete release directories through event scripting. Directory scripts can skip a +candidate directory, and quarantine handlers are still subject to dry-run mode: -- `script.dryRun.enabled = true` prevents deletion/quarantine from actually modifying the filesystem. +- `script.dryRun.enabled = true` prevents quarantine actions from modifying the filesystem. ## Event reference @@ -156,8 +157,8 @@ This section lists the supported events and the `ctx` fields available to script ### `directoryProcessingStart` -Runs before processing each candidate directory. If it returns `false`, Melodee applies `onDeny` (`skip`, `delete`, or -`quarantine`). +Runs before processing each candidate directory. If it returns `false`, Melodee applies `onDeny`. For ingestion safety, +`delete` is treated as `skip`. Context: `DirectoryProcessingContext` @@ -176,8 +177,8 @@ Context: `DirectoryProcessingContext` ### `directoryProcessingDelete` -Runs when `directoryProcessingStart` returns `false` and `onDeny` is `delete`. If this script returns `true`, deletion -proceeds; if `false`, deletion is skipped (directory is not processed but also not deleted). +Legacy event name retained for existing settings compatibility. Inbound ingestion no longer evaluates this event and does +not delete release directories through event scripting. Context: `DirectoryProcessingContext` (same as above). @@ -303,20 +304,19 @@ Recommended override config example: "enabled": true, "libraryId": 1, "pathPrefix": "Incoming/", - "onDeny": "delete", + "onDeny": "skip", "body": "function check(ctx, scriptConfig) { return ctx.mediaFilesCount >= 3; }" } ] } ``` -### Example: add an extra safety check before deletion +### Example: skip directories with track number gaps -Event: `directoryProcessingDelete` +Event: `directoryProcessingStart` ```javascript function check(ctx, scriptConfig) { - // Only allow deletion if there are no track number gaps. return ctx.hasTrackNumberGaps === false; } ``` diff --git a/scripts/run-decentdb-package-upgrade-gate.sh b/scripts/run-decentdb-package-upgrade-gate.sh new file mode 100755 index 000000000..ddbe6cc5b --- /dev/null +++ b/scripts/run-decentdb-package-upgrade-gate.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + ./scripts/run-decentdb-package-upgrade-gate.sh [--options] + +This helper runs Melodee's existing MusicBrainz import/query probes and evaluates +DDB-002/DDB-003 acceptance for an existing checkpointed database. + +Required: + --db PATH Path to checkpointed MusicBrainz .ddb file. + +Options: + --storage PATH Path to MusicBrainz staging dump for optional import rebuild. + --label TEXT Identifier for output subdirectory (default: candidate). + --output-dir PATH Output root (default: .tmp/decentdb-upgrade-validation). + --max-warm-name-ms N Max warm Artist.NameNormalized query time in ms (default: 1000). + --max-warm-mbid-ms N Max warm Artist.MusicBrainzIdRaw query time in ms (default: 1000). + --max-warm-alias-ms N Max warm ArtistAlias.NameNormalized query time in ms (default: 2000). + --name TEXT Explicit normalized artist sample for the query probe. + --alias TEXT Explicit normalized alias sample for the query probe. + --mbid TEXT Explicit raw MusicBrainz ID sample for the query probe. + --include-row-existence + Include the broad first-row existence probe for investigation. + --skip-build Skip restore/build step and run probes directly. + --help Show this help text. + +Examples: + ./scripts/run-decentdb-package-upgrade-gate.sh \ + --db /tmp/musicbrainz.ddb \ + --label 2.13.1 \ + --output-dir .tmp/decentdb-upgrade + + ./scripts/run-decentdb-package-upgrade-gate.sh \ + --db /tmp/musicbrainz.ddb \ + --storage /mnt/fileserver_db_storage/melodee/search-engine-storage/musicbrainz \ + --label 2.13.1-candidate \ + --output-dir .tmp/decentdb-upgrade +USAGE +} + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +DB_PATH="" +STORAGE_PATH="" +LABEL="candidate" +OUTPUT_DIR=".tmp/decentdb-upgrade-validation" +MAX_NAME_MS=1000 +MAX_MBID_MS=1000 +MAX_ALIAS_MS=2000 +SAMPLE_NAME="" +SAMPLE_ALIAS="" +SAMPLE_MBID="" +INCLUDE_ROW_EXISTENCE=false +SKIP_BUILD=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --db) + DB_PATH="$2" + shift 2 + ;; + --storage) + STORAGE_PATH="$2" + shift 2 + ;; + --label) + LABEL="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --max-warm-name-ms) + MAX_NAME_MS="$2" + shift 2 + ;; + --max-warm-mbid-ms) + MAX_MBID_MS="$2" + shift 2 + ;; + --max-warm-alias-ms) + MAX_ALIAS_MS="$2" + shift 2 + ;; + --name) + SAMPLE_NAME="$2" + shift 2 + ;; + --alias) + SAMPLE_ALIAS="$2" + shift 2 + ;; + --mbid) + SAMPLE_MBID="$2" + shift 2 + ;; + --include-row-existence) + INCLUDE_ROW_EXISTENCE=true + shift + ;; + --skip-build) + SKIP_BUILD=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "$DB_PATH" ]]; then + echo "Error: --db is required." >&2 + usage + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required for acceptance checks." >&2 + exit 1 +fi + +if ! command -v dotnet >/dev/null 2>&1; then + echo "Error: dotnet is required to run benchmark probes." >&2 + exit 1 +fi + +RUN_DIR="${OUTPUT_DIR%/}/${LABEL}" +IMPORT_REPORT="${RUN_DIR}/musicbrainz-import-probe.json" +CHECKPOINTED_REPORT="${RUN_DIR}/musicbrainz-query-probe-checkpointed.json" +mkdir -p "$RUN_DIR" + +if [[ "$SKIP_BUILD" != true ]]; then + echo "Restoring and building benchmark project..." + dotnet restore Melodee.sln + dotnet build Melodee.sln --no-restore +fi + +if [[ -n "$STORAGE_PATH" ]]; then + if [[ ! -d "$STORAGE_PATH" ]]; then + echo "Error: --storage path does not exist: $STORAGE_PATH" >&2 + exit 1 + fi + + if [[ ! -d "$STORAGE_PATH/staging/mbdump" ]]; then + echo "Error: expected MusicBrainz dump under $STORAGE_PATH/staging/mbdump." >&2 + exit 1 + fi + + echo "Running import probe for fresh real-file rebuild..." + dotnet run -c Release --project benchmarks/Melodee.Benchmarks -- \ + musicbrainz-import-probe \ + --storage "$STORAGE_PATH" \ + --db "$DB_PATH" \ + --output "$IMPORT_REPORT" \ + --clean \ + --sample-interval-ms 10000 + + echo "Import probe complete -> $IMPORT_REPORT" +fi + +echo "Running checkpointed query probe..." +QUERY_PROBE_ARGS=( + musicbrainz-query-probe + --db "$DB_PATH" + --output "$CHECKPOINTED_REPORT" +) + +if [[ -n "$SAMPLE_NAME" ]]; then + QUERY_PROBE_ARGS+=(--name "$SAMPLE_NAME") +fi + +if [[ -n "$SAMPLE_ALIAS" ]]; then + QUERY_PROBE_ARGS+=(--alias "$SAMPLE_ALIAS") +fi + +if [[ -n "$SAMPLE_MBID" ]]; then + QUERY_PROBE_ARGS+=(--mbid "$SAMPLE_MBID") +fi + +if [[ "$INCLUDE_ROW_EXISTENCE" == true ]]; then + QUERY_PROBE_ARGS+=(--include-row-existence) +fi + +dotnet run -c Release --project benchmarks/Melodee.Benchmarks -- \ + "${QUERY_PROBE_ARGS[@]}" + +if [[ ! -f "$CHECKPOINTED_REPORT" ]]; then + echo "Error: query probe report not created: $CHECKPOINTED_REPORT" >&2 + exit 1 +fi + +check_query_ms() { + local query_name="$1" + local max_ms="$2" + local report_path="$3" + + local warm_ms + warm_ms=$(jq -r --arg name "$query_name" --arg pass "warm" \ + '.Measurements[] | select(.Name == $name and .Pass == $pass) | .ElapsedMilliseconds' "$report_path" | head -n 1) + + if [[ -z "$warm_ms" || "$warm_ms" == "null" ]]; then + echo "$query_name: missing warm timing" + return 1 + fi + + local plan + plan=$(jq -r --arg name "$query_name" --arg pass "warm" \ + '.Measurements[] | select(.Name == $name and .Pass == $pass) | .PlanDiagnostics.Lines | join(" ")' "$report_path") + + if ! echo "$plan" | grep -q "IndexSeek"; then + echo "$query_name: warm $warm_ms ms -> FAIL (plan missing IndexSeek)" + return 1 + fi + + if awk -v ms="$warm_ms" -v limit="$max_ms" 'BEGIN { exit !(ms <= limit) }'; then + echo "$query_name: warm $warm_ms ms (PASS <= ${max_ms}ms, plan captured)" + return 0 + fi + + echo "$query_name: warm $warm_ms ms (FAIL > ${max_ms}ms)" + return 1 +} + +FAILURES=0 + +if ! check_query_ms "exact-normalized-name" "$MAX_NAME_MS" "$CHECKPOINTED_REPORT"; then + FAILURES=1 +fi + +if ! check_query_ms "exact-musicbrainz-id-raw" "$MAX_MBID_MS" "$CHECKPOINTED_REPORT"; then + FAILURES=1 +fi + +if ! check_query_ms "exact-normalized-alias" "$MAX_ALIAS_MS" "$CHECKPOINTED_REPORT"; then + echo "Alias shape check failed but does not block DDB-002/DDB-003." >&2 +fi + +echo "Query probe report: $CHECKPOINTED_REPORT" + +if [[ "$FAILURES" -ne 0 ]]; then + echo "DDB-002 / DDB-003 gate failed. See report output for details." >&2 + exit 1 +fi + +echo "DDB-002 / DDB-003 gate passed." diff --git a/src/Melodee.Blazor/Components/App.razor b/src/Melodee.Blazor/Components/App.razor index c6aad6e7b..69118eec9 100644 --- a/src/Melodee.Blazor/Components/App.razor +++ b/src/Melodee.Blazor/Components/App.razor @@ -71,7 +71,6 @@ - diff --git a/src/Melodee.Blazor/Components/Components/ImageSearchUpload.razor b/src/Melodee.Blazor/Components/Components/ImageSearchUpload.razor index 68f6fa04b..096e3a6f6 100644 --- a/src/Melodee.Blazor/Components/Components/ImageSearchUpload.razor +++ b/src/Melodee.Blazor/Components/Components/ImageSearchUpload.razor @@ -7,6 +7,7 @@ @inject ArtistImageSearchEngineService ArtistImageSearchEngineService @inject MainLayoutProxyService MainLayoutProxyService @inject ILogger Logger +@implements IDisposable Image Search @@ -90,10 +91,13 @@ bool _isLoading; int _maxResults = 10; bool _doDeleteExistingImages = true; + int _searchRequestId; + bool _disposed; private ImageSearchResult[] _results = []; private ImageSearchResult? _selectedResult; + private CancellationTokenSource? _searchCancellationTokenSource; [Parameter] public SearchValue[] SearchValues { get; set; } = []; @@ -127,12 +131,13 @@ private async Task SelectOrCancelClicked(bool clickedSelected) { - if (!clickedSelected) + if (!clickedSelected || _selectedResult is null) { await OnUpdateCallback.InvokeAsync(null); + return; } - if (OnUpdateCallback.HasDelegate && _selectedResult != null) + if (OnUpdateCallback.HasDelegate) { await OnUpdateCallback.InvokeAsync(_selectedResult); } @@ -148,53 +153,104 @@ private async Task DoSearchAsync() { - if (_isLoading) + if (_disposed || SearchType == ImageSearchType.Profile) { return; } + var searchValues = SearchValues.OrderBy(x => x.SortOrder).ToArray(); + if (searchValues.Length == 0) + { + _results = []; + return; + } + + var previousSearch = _searchCancellationTokenSource; + var searchTokenSource = new CancellationTokenSource(); + _searchCancellationTokenSource = searchTokenSource; + var searchRequestId = Interlocked.Increment(ref _searchRequestId); + previousSearch?.Cancel(); + _isLoading = true; + _selectedResult = null; + _results = []; + await InvokeAsync(StateHasChanged); + try { if (SearchType == ImageSearchType.Artist) { - _results = (await ArtistImageSearchEngineService.DoSearchAsync(new ArtistQuery + var results = (await ArtistImageSearchEngineService.DoSearchAsync(new ArtistQuery { - Name = SearchValues.First().Value - }, _maxResults)) + Name = searchValues.First().Value + }, _maxResults, searchTokenSource.Token)) .Data .Where(x => x.ThumbnailUrlValue.Nullify() != null) .Where(x => x.Width >= _minimumImageSize && x.Height >= _minimumImageSize) .OrderByDescending(x => x.Width * x.Height) .Select(x => new ImageSearchResult(x.ThumbnailUrlValue, x.MediaUrl, x.Title ?? string.Empty, _doDeleteExistingImages, x.Width, x.Height)) .ToArray(); + + if (searchRequestId == _searchRequestId && !searchTokenSource.IsCancellationRequested) + { + _results = results; + } } - else if (SearchType != ImageSearchType.Profile) + else if (searchValues.Length >= 3) { - _results = (await AlbumImageSearchEngineService.DoSearchAsync(new AlbumQuery + var results = (await AlbumImageSearchEngineService.DoSearchAsync(new AlbumQuery { - Artist = SearchValues.OrderBy(x => x.SortOrder).First().Value, - Year = SafeParser.ToNumber(SearchValues.OrderBy(x => x.SortOrder).Skip(1).First().Value), - Name = SearchValues.OrderBy(x => x.SortOrder).Skip(2).First().Value - }, _maxResults)) + Artist = searchValues[0].Value, + Year = SafeParser.ToNumber(searchValues[1].Value), + Name = searchValues[2].Value + }, _maxResults, searchTokenSource.Token)) .Data .Where(x => x.Width >= _minimumImageSize && x.Height >= _minimumImageSize) .OrderByDescending(x => x.Width * x.Height) .Select(x => new ImageSearchResult(x.ThumbnailUrlValue, x.MediaUrl, x.Title ?? string.Empty, _doDeleteExistingImages, x.Width, x.Height)) .ToArray(); + + if (searchRequestId == _searchRequestId && !searchTokenSource.IsCancellationRequested) + { + _results = results; + } } } + catch (OperationCanceledException) when (searchTokenSource.IsCancellationRequested) + { + // A newer search or dialog close superseded this request. + } catch (Exception e) { - Logger.Error(e, "For search [{Value}]", SearchValues.FirstOrDefault()?.Value); + Logger.Error(e, "For search [{Value}]", searchValues.FirstOrDefault()?.Value); } finally { - _isLoading = false; - await InvokeAsync(StateHasChanged); + if (searchRequestId == _searchRequestId) + { + _isLoading = false; + if (!_disposed) + { + await InvokeAsync(StateHasChanged); + } + } + + if (ReferenceEquals(_searchCancellationTokenSource, searchTokenSource)) + { + _searchCancellationTokenSource = null; + } + + searchTokenSource.Dispose(); } } + public void Dispose() + { + _disposed = true; + Interlocked.Increment(ref _searchRequestId); + _searchCancellationTokenSource?.Cancel(); + } + private Task CancelClicked() { diff --git a/src/Melodee.Blazor/Components/Layout/MainLayout.razor b/src/Melodee.Blazor/Components/Layout/MainLayout.razor index cb4eff229..a25717982 100644 --- a/src/Melodee.Blazor/Components/Layout/MainLayout.razor +++ b/src/Melodee.Blazor/Components/Layout/MainLayout.razor @@ -228,7 +228,7 @@ .Select(x => new MediaLibraryInfo(x.Name, $"/media/library/{x.ApiKey}")) .ToArray(); - MainLayoutProxyService.ShowSpinner = false; + MainLayoutProxyService.SetSpinnerVisible(false); await CheckActivityAsync(); _activityTimer = new Timer(async _ => await CheckActivityAsync(), null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1)); diff --git a/src/Melodee.Blazor/Components/Pages/Admin/Dashboard.razor b/src/Melodee.Blazor/Components/Pages/Admin/Dashboard.razor index 40846327d..61e34aaab 100644 --- a/src/Melodee.Blazor/Components/Pages/Admin/Dashboard.razor +++ b/src/Melodee.Blazor/Components/Pages/Admin/Dashboard.razor @@ -54,6 +54,35 @@ + @if (!_isCheckingHealth && _systemHealthIssues.Length > 0) + { + + + @L("Dashboard.IssuesDetected") + @foreach (var issue in _systemHealthIssues) + { + + @FormatHealthIssue(issue) + + } + @if (DashboardHealthWarningEvaluator.HasUnsupportedDecentDbIssue(_systemHealthIssues)) + { + + @L("Dashboard.DecentDbMigrationGuide") + + } + + + + } + @* Quick Links *@ @@ -502,6 +531,7 @@ private string _melodeeVersion = "---"; private string _apiVersion = "---"; private string _osDescription = "---"; + private Melodee.Common.Services.Doctor.DoctorCheckResult[] _systemHealthIssues = []; private bool _systemHealthy = true; private bool _isCheckingHealth = true; @@ -648,10 +678,19 @@ try { var results = await DoctorService.RunAllChecksAsync(); - _systemHealthy = results.Checks.All(c => c.Success); + _systemHealthIssues = results.Checks.Where(c => !c.Success).Take(5).ToArray(); + _systemHealthy = _systemHealthIssues.Length == 0; } catch { + _systemHealthIssues = + [ + new Melodee.Common.Services.Doctor.DoctorCheckResult( + "Doctor", + false, + "Doctor checks failed before returning results", + TimeSpan.Zero) + ]; _systemHealthy = false; } finally @@ -661,6 +700,43 @@ } } + private string FormatHealthIssue(Melodee.Common.Services.Doctor.DoctorCheckResult issue) + { + var name = LocalizeHealthIssueName(issue.Name); + return string.IsNullOrWhiteSpace(issue.Details) + ? name + : $"{name}: {issue.Details}"; + } + + private string LocalizeHealthIssueName(string checkName) + { + return checkName switch + { + "Configuration" => L("AdminDoctor.Check.Configuration"), + "PostgresDatabase" => L("AdminDoctor.Check.PostgresDatabase"), + "MusicBrainzDatabase" => L("AdminDoctor.Check.MusicBrainzDatabase"), + "ArtistSearchEngineDatabase" => L("AdminDoctor.Check.ArtistSearchEngineDatabase"), + "LibraryPaths" => L("AdminDoctor.Check.LibraryPaths"), + "SerilogLogging" => L("AdminDoctor.Check.SerilogLogging"), + "ConfigurableServices" => L("AdminDoctor.Check.ConfigurableServices"), + "DiskSpace" => L("AdminDoctor.Check.DiskSpace"), + "LibraryPathOverlap" => L("AdminDoctor.Check.LibraryPathOverlap"), + "SearchEngineApiKeys" => L("AdminDoctor.Check.SearchEngineApiKeys"), + "SmtpConfiguration" => L("AdminDoctor.Check.SmtpConfiguration"), + "JwtTokenStrength" => L("AdminDoctor.Check.JwtTokenStrength"), + "HttpsSecurity" => L("AdminDoctor.Check.HttpsSecurity"), + "AdminPassword" => L("AdminDoctor.Check.AdminPassword"), + "Scheduler" => L("AdminDoctor.Check.Scheduler"), + "FFmpeg" => L("AdminDoctor.Check.FFmpeg"), + "Memory" => L("AdminDoctor.Check.Memory"), + "TempDirectory" => L("AdminDoctor.Check.TempDirectory"), + "DatabaseLatency" => L("AdminDoctor.Check.DatabaseLatency"), + "JukeboxConfiguration" => L("AdminDoctor.Check.JukeboxConfiguration"), + "PodcastConfiguration" => L("AdminDoctor.Check.PodcastConfiguration"), + _ => checkName + }; + } + private void ClearCacheClicked() { CacheManager.Clear(); diff --git a/src/Melodee.Blazor/Components/Pages/Dashboard.razor b/src/Melodee.Blazor/Components/Pages/Dashboard.razor index 29d687f55..adcefc0ce 100644 --- a/src/Melodee.Blazor/Components/Pages/Dashboard.razor +++ b/src/Melodee.Blazor/Components/Pages/Dashboard.razor @@ -44,10 +44,26 @@ - + @L("Dashboard.IssuesDetected") + @foreach (var issue in _healthIssues.Take(3)) + { + + @FormatHealthIssue(issue) + + } + @if (DashboardHealthWarningEvaluator.HasUnsupportedDecentDbIssue(_healthIssues)) + { + + @L("Dashboard.DecentDbMigrationGuide") + + } @@ -350,6 +366,7 @@ private TopItemStat[] _recentlyPlayed = []; private SongDataInfo[] _recentlyPlayedSongs = []; private Request[] _unreadRequests = []; + private Melodee.Common.Services.Doctor.DoctorCheckResult[] _healthIssues = []; private bool _showHealthWarning; private bool _isAdmin; @@ -363,6 +380,7 @@ private bool _artistsLoading = true; private bool _albumsLoading = true; private bool _isLoading => _healthLoading || _pinsLoading || _kpisLoading || _playsLoading || _topSongsLoading || _recentlyPlayedLoading || _requestsLoading || _artistsLoading || _albumsLoading; + private bool _isDisposed; private short _latestPageSize; private LocalDate _rangeStart; @@ -431,7 +449,7 @@ var userApiKey = CurrentUser?.ToApiGuid() ?? Guid.Empty; var userId = CurrentUser?.UserId() ?? 0; - MainLayoutProxyService.ShowSpinner = true; + MainLayoutProxyService.SetSpinnerVisible(true); _ = LoadHealthAsync(); _ = LoadPinsAsync(userId); @@ -446,25 +464,43 @@ private void UpdateSpinner() { + if (_isDisposed) + { + MainLayoutProxyService.SetSpinnerVisible(false); + return; + } + var anyLoading = _healthLoading || _pinsLoading || _kpisLoading || _playsLoading || _topSongsLoading || _recentlyPlayedLoading || _requestsLoading || _artistsLoading || _albumsLoading; - MainLayoutProxyService.ShowSpinner = anyLoading; + MainLayoutProxyService.SetSpinnerVisible(anyLoading); + } + + private async Task NotifyDashboardChangedAsync() + { + if (_isDisposed) + { + return; + } + + await InvokeAsync(StateHasChanged); } private async Task LoadHealthAsync() { try { - _showHealthWarning = await DashboardHealthWarningEvaluator.ShouldShowAsync(CurrentUser, DoctorService); + _healthIssues = (await DashboardHealthWarningEvaluator.GetIssuesAsync(CurrentUser, DoctorService)).ToArray(); + _showHealthWarning = _healthIssues.Length > 0; } catch { + _healthIssues = []; _showHealthWarning = false; } finally { _healthLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -486,7 +522,7 @@ { _pinsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -505,7 +541,7 @@ { _kpisLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -524,7 +560,7 @@ { _playsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -543,7 +579,7 @@ { _topSongsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -599,7 +635,7 @@ { _recentlyPlayedLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -618,7 +654,7 @@ { _requestsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -644,7 +680,7 @@ { _artistsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -670,7 +706,7 @@ { _albumsLoading = false; UpdateSpinner(); - await InvokeAsync(StateHasChanged); + await NotifyDashboardChangedAsync(); } } @@ -679,15 +715,43 @@ _ = InvokeAsync(async () => { var authenticationState = await authenticationStateTask; + if (_isDisposed) + { + return; + } + _isAdmin = authenticationState.User.IsAdmin(); - _showHealthWarning = await DashboardHealthWarningEvaluator.ShouldShowAsync(authenticationState.User, DoctorService); + _healthIssues = (await DashboardHealthWarningEvaluator.GetIssuesAsync(authenticationState.User, DoctorService)).ToArray(); + _showHealthWarning = _healthIssues.Length > 0; StateHasChanged(); }); } + private string FormatHealthIssue(Melodee.Common.Services.Doctor.DoctorCheckResult issue) + { + var name = LocalizeHealthIssueName(issue.Name); + return string.IsNullOrWhiteSpace(issue.Details) + ? name + : $"{name}: {issue.Details}"; + } + + private string LocalizeHealthIssueName(string checkName) + { + return checkName switch + { + "Configuration" => L("AdminDoctor.Check.Configuration"), + "PostgresDatabase" => L("AdminDoctor.Check.PostgresDatabase"), + "MusicBrainzDatabase" => L("AdminDoctor.Check.MusicBrainzDatabase"), + "ArtistSearchEngineDatabase" => L("AdminDoctor.Check.ArtistSearchEngineDatabase"), + _ => checkName + }; + } + public void Dispose() { + _isDisposed = true; DashboardAuthenticationStateProvider.AuthenticationStateChanged -= OnAuthenticationStateChanged; + MainLayoutProxyService.SetSpinnerVisible(false); } } diff --git a/src/Melodee.Blazor/Components/Pages/DashboardHealthWarningEvaluator.cs b/src/Melodee.Blazor/Components/Pages/DashboardHealthWarningEvaluator.cs index 4b478d48c..c44f6632f 100644 --- a/src/Melodee.Blazor/Components/Pages/DashboardHealthWarningEvaluator.cs +++ b/src/Melodee.Blazor/Components/Pages/DashboardHealthWarningEvaluator.cs @@ -1,12 +1,15 @@ using System.Security.Claims; using Melodee.Blazor.Security.Extensions; -using Melodee.Blazor.Services; +using BlazorDoctorService = Melodee.Blazor.Services.IDoctorService; +using DoctorCheckResult = Melodee.Common.Services.Doctor.DoctorCheckResult; namespace Melodee.Blazor.Components.Pages; internal static class DashboardHealthWarningEvaluator { - public static Task ShouldShowAsync(ClaimsPrincipal? user, IDoctorService doctorService, CancellationToken cancellationToken = default) + public const string DecentDbMigrationGuideUrl = "https://melodee.org/decentdb/"; + + public static Task ShouldShowAsync(ClaimsPrincipal? user, BlazorDoctorService doctorService, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(doctorService); @@ -14,4 +17,32 @@ public static Task ShouldShowAsync(ClaimsPrincipal? user, IDoctorService d ? doctorService.NeedsAttentionAsync(cancellationToken) : Task.FromResult(false); } + + public static async Task> GetIssuesAsync( + ClaimsPrincipal? user, + BlazorDoctorService doctorService, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(doctorService); + + return user?.IsAdmin() == true + ? await doctorService.GetAttentionChecksAsync(cancellationToken) + : []; + } + + public static bool HasUnsupportedDecentDbIssue(IEnumerable issues) + { + ArgumentNullException.ThrowIfNull(issues); + + return issues.Any(issue => + IsDecentDbCheck(issue.Name) && + issue.Details.Contains("DecentDB", StringComparison.OrdinalIgnoreCase) && + issue.Details.Contains("file format", StringComparison.OrdinalIgnoreCase) && + issue.Details.Contains("not supported", StringComparison.OrdinalIgnoreCase)); + } + + private static bool IsDecentDbCheck(string checkName) + { + return checkName is "MusicBrainzDatabase" or "ArtistSearchEngineDatabase"; + } } diff --git a/src/Melodee.Blazor/Components/Pages/Data/AlbumDetail.razor b/src/Melodee.Blazor/Components/Pages/Data/AlbumDetail.razor index 7f6a19a5d..1b8d2da6f 100644 --- a/src/Melodee.Blazor/Components/Pages/Data/AlbumDetail.razor +++ b/src/Melodee.Blazor/Components/Pages/Data/AlbumDetail.razor @@ -1026,13 +1026,13 @@ { Key = "Year", Value = _album.ReleaseDate.Year.ToString(), - SortOrder = 1 + SortOrder = 2 }, new() { Key = "Title", Value = _album.Name, - SortOrder = 1 + SortOrder = 3 } ]; diff --git a/src/Melodee.Blazor/Components/Pages/Data/AlbumEdit.razor b/src/Melodee.Blazor/Components/Pages/Data/AlbumEdit.razor index b49c637c5..75fe408fd 100644 --- a/src/Melodee.Blazor/Components/Pages/Data/AlbumEdit.razor +++ b/src/Melodee.Blazor/Components/Pages/Data/AlbumEdit.razor @@ -133,7 +133,7 @@ - + @@ -209,7 +209,7 @@ - + @@ -403,4 +403,3 @@ } - diff --git a/src/Melodee.Blazor/Components/Pages/Media/AlbumDetail.razor b/src/Melodee.Blazor/Components/Pages/Media/AlbumDetail.razor index 02e17008d..5d15fb97b 100644 --- a/src/Melodee.Blazor/Components/Pages/Media/AlbumDetail.razor +++ b/src/Melodee.Blazor/Components/Pages/Media/AlbumDetail.razor @@ -793,13 +793,13 @@ { Key = "Year", Value = _album.AlbumYear()?.ToString() ?? string.Empty, - SortOrder = 1 + SortOrder = 2 }, new() { Key = "Title", Value = _album.AlbumTitle() ?? string.Empty, - SortOrder = 1 + SortOrder = 3 } ]; diff --git a/src/Melodee.Blazor/Components/Pages/Media/AlbumEdit.razor b/src/Melodee.Blazor/Components/Pages/Media/AlbumEdit.razor index f12a4625a..9ab25465f 100644 --- a/src/Melodee.Blazor/Components/Pages/Media/AlbumEdit.razor +++ b/src/Melodee.Blazor/Components/Pages/Media/AlbumEdit.razor @@ -372,13 +372,13 @@ { Key = "Year", Value = _album.Year.ToString(), - SortOrder = 1 + SortOrder = 2 }, new() { Key = "Title", Value = _album.Title ?? string.Empty, - SortOrder = 1 + SortOrder = 3 } ]; @@ -482,4 +482,3 @@ } - diff --git a/src/Melodee.Blazor/Components/Pages/Shared.razor b/src/Melodee.Blazor/Components/Pages/Shared.razor index 514c51bd5..22eea1d15 100644 --- a/src/Melodee.Blazor/Components/Pages/Shared.razor +++ b/src/Melodee.Blazor/Components/Pages/Shared.razor @@ -1,5 +1,4 @@ @page "/share/{ShareUniqueId}" -@using AuralizeBlazor @using Melodee.Common.Data.Models.Extensions @using Melodee.Common.Enums @@ -13,15 +12,32 @@
- -
- + @if (_selectedTrack is not null) + { +
+ +
+

@_selectedTrack.Title

+ +
- + } + + @if (_tracks.Length > 1) + { +
    + @foreach (var track in _tracks) + { +
  1. + +
    2. + } +
+ }
@@ -29,7 +45,8 @@ [Parameter] public required string ShareUniqueId { get; set; } [CascadingParameter] private HttpContext? HttpContext { get; set; } - private IAuralizerTrack[] _songInfos = [new AuralizerTrack(string.Empty, string.Empty, string.Empty)]; + private SharedTrack[] _tracks = []; + private SharedTrack? _selectedTrack; protected override async Task OnParametersSetAsync() { @@ -44,12 +61,12 @@ var songResult = await SongService.GetAsync(share.ShareId); if (songResult is { IsSuccess: true, Data: not null }) { - _songInfos = + SetTracks( [ - new AuralizerTrack(songResult.Data.ToApiStreamUrl(configuration, HttpContext), + new SharedTrack(songResult.Data.ToApiStreamUrl(configuration, HttpContext), $"{songResult.Data.Album.Name} - {songResult.Data.Title}", $"/images/{songResult.Data.ToCoverArtId()}/{ ImageSize.Large }") - ]; + ]); } break; @@ -57,11 +74,11 @@ var albumResult = await AlbumService.GetAsync(share.ShareId); if (albumResult is { IsSuccess: true, Data: not null }) { - _songInfos = albumResult.Data.Songs.Select(x => - new AuralizerTrack(x.ToApiStreamUrl(configuration, HttpContext), + SetTracks(albumResult.Data.Songs.Select(x => + new SharedTrack(x.ToApiStreamUrl(configuration, HttpContext), $"{albumResult.Data.Name} - {x.Title}", $"/images/{albumResult.Data.ToCoverArtId()}/{ ImageSize.Large }") - ).ToArray(); + ).ToArray()); } break; @@ -69,11 +86,11 @@ var playlistResult = await PlaylistService.GetAsync(share.ShareId); if (playlistResult is { IsSuccess: true, Data: not null }) { - _songInfos = playlistResult.Data.Songs.Select(x => - new AuralizerTrack(x.Song.ToApiStreamUrl(configuration, HttpContext), + SetTracks(playlistResult.Data.Songs.Select(x => + new SharedTrack(x.Song.ToApiStreamUrl(configuration, HttpContext), $"{x.Song.Album.Name} - {x.Song.Title}", $"/images/{x.Song.ToCoverArtId()}/{ ImageSize.Large }") - ).ToArray(); + ).ToArray()); } break; @@ -83,8 +100,18 @@ } } + private void SetTracks(SharedTrack[] tracks) + { + _tracks = tracks; + _selectedTrack = _tracks.FirstOrDefault(); + } -} + private void SelectTrack(SharedTrack track) + { + _selectedTrack = track; + } + private sealed record SharedTrack(string Url, string Title, string CoverArtUrl); +} diff --git a/src/Melodee.Blazor/Melodee.Blazor.csproj b/src/Melodee.Blazor/Melodee.Blazor.csproj index 4b10b406c..8e609f6bf 100644 --- a/src/Melodee.Blazor/Melodee.Blazor.csproj +++ b/src/Melodee.Blazor/Melodee.Blazor.csproj @@ -16,7 +16,7 @@ - 2.1.0 + 2.1.3 build$([System.DateTime]::UtcNow.ToString("yyyyMMddHHmmss")) $(VersionPrefix).0 @@ -57,7 +57,6 @@ - diff --git a/src/Melodee.Blazor/Program.cs b/src/Melodee.Blazor/Program.cs index 5e063cbba..b5f94eb3a 100644 --- a/src/Melodee.Blazor/Program.cs +++ b/src/Melodee.Blazor/Program.cs @@ -70,6 +70,7 @@ Trace.Listeners.Clear(); Trace.Listeners.Add(new ConsoleTraceListener()); +ATL.Settings.OutputStacktracesToConsole = false; builder.Host.UseSerilog((hostingContext, loggerConfiguration) => loggerConfiguration.ReadFrom.Configuration(hostingContext.Configuration)); @@ -165,6 +166,8 @@ { options.UseDecentDB(builder.Configuration.GetConnectionString("MusicBrainzConnection") ?? throw new Exception("Invalid Connection String"), x => x.UseNodaTime()); }); + + builder.Services.AddHostedService(); } builder.Services.AddApiVersioning(options => @@ -535,6 +538,7 @@ .AddSingleton() .AddSingleton() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() @@ -561,6 +565,7 @@ .AddSingleton() .AddSingleton() .AddScoped() + .AddSingleton() .AddScoped() .AddScoped() .AddScoped() diff --git a/src/Melodee.Blazor/Resources/SharedResources.ar-SA.resx b/src/Melodee.Blazor/Resources/SharedResources.ar-SA.resx index 5ccbe1ad5..07df84594 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.ar-SA.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.ar-SA.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.cs-CZ.resx b/src/Melodee.Blazor/Resources/SharedResources.cs-CZ.resx index c98653c48..43a429bfd 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.cs-CZ.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.cs-CZ.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.de-DE.resx b/src/Melodee.Blazor/Resources/SharedResources.de-DE.resx index 20dfd24a4..410d29468 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.de-DE.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.de-DE.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.es-ES.resx b/src/Melodee.Blazor/Resources/SharedResources.es-ES.resx index d5a63823c..a3993fc8a 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.es-ES.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.es-ES.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.fa-IR.resx b/src/Melodee.Blazor/Resources/SharedResources.fa-IR.resx index 70905151b..5fc234c39 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.fa-IR.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.fa-IR.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.fr-FR.resx b/src/Melodee.Blazor/Resources/SharedResources.fr-FR.resx index 15807cdff..e66d27cee 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.fr-FR.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.fr-FR.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.id-ID.resx b/src/Melodee.Blazor/Resources/SharedResources.id-ID.resx index 98afef905..12812e47c 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.id-ID.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.id-ID.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.it-IT.resx b/src/Melodee.Blazor/Resources/SharedResources.it-IT.resx index 56f855c6b..7bda1afd0 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.it-IT.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.it-IT.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.ja-JP.resx b/src/Melodee.Blazor/Resources/SharedResources.ja-JP.resx index e01855fc0..2e3550ea4 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.ja-JP.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.ja-JP.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.ko-KR.resx b/src/Melodee.Blazor/Resources/SharedResources.ko-KR.resx index 9b231c83d..db08d69dd 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.ko-KR.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.ko-KR.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.nl-NL.resx b/src/Melodee.Blazor/Resources/SharedResources.nl-NL.resx index 97004b65e..e42e71a06 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.nl-NL.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.nl-NL.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.pl-PL.resx b/src/Melodee.Blazor/Resources/SharedResources.pl-PL.resx index c4e95297e..368823f6c 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.pl-PL.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.pl-PL.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.pt-BR.resx b/src/Melodee.Blazor/Resources/SharedResources.pt-BR.resx index cf9c2fca2..a90fae212 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.pt-BR.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.pt-BR.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.resx b/src/Melodee.Blazor/Resources/SharedResources.resx index fed6a737b..41a93f8eb 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.resx @@ -6893,4 +6893,8 @@ Common fixes: Max JavaScript statements allowed per execution + + + Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.ru-RU.resx b/src/Melodee.Blazor/Resources/SharedResources.ru-RU.resx index 779bf8a8d..fe4a2c2ab 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.ru-RU.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.ru-RU.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.sv-SE.resx b/src/Melodee.Blazor/Resources/SharedResources.sv-SE.resx index 4d6bf2650..26f3f61a3 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.sv-SE.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.sv-SE.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.tr-TR.resx b/src/Melodee.Blazor/Resources/SharedResources.tr-TR.resx index 1767edf01..a2ef27bbd 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.tr-TR.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.tr-TR.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.uk-UA.resx b/src/Melodee.Blazor/Resources/SharedResources.uk-UA.resx index a7759d33b..5a17f0cbd 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.uk-UA.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.uk-UA.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.vi-VN.resx b/src/Melodee.Blazor/Resources/SharedResources.vi-VN.resx index 78df4ae71..f5685ec8a 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.vi-VN.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.vi-VN.resx @@ -6507,4 +6507,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Resources/SharedResources.zh-CN.resx b/src/Melodee.Blazor/Resources/SharedResources.zh-CN.resx index 22b58f7d1..67a9ba8e6 100644 --- a/src/Melodee.Blazor/Resources/SharedResources.zh-CN.resx +++ b/src/Melodee.Blazor/Resources/SharedResources.zh-CN.resx @@ -6856,4 +6856,8 @@ Common fixes: [NEEDS TRANSLATION] Max JavaScript statements allowed per execution + + + [NEEDS TRANSLATION] Open DecentDB migration guide + diff --git a/src/Melodee.Blazor/Services/DoctorService.cs b/src/Melodee.Blazor/Services/DoctorService.cs index 541ba52e2..60bf5635d 100644 --- a/src/Melodee.Blazor/Services/DoctorService.cs +++ b/src/Melodee.Blazor/Services/DoctorService.cs @@ -60,34 +60,46 @@ public sealed class DoctorService( private const int JobStalenessHours = 48; // Jobs should run within this period public async Task NeedsAttentionAsync(CancellationToken cancellationToken = default) + { + var checks = await GetAttentionChecksAsync(cancellationToken); + return checks.Count > 0; + } + + public async Task> GetAttentionChecksAsync(CancellationToken cancellationToken = default) { // Dashboard uses this fast path on first render, so keep it limited to - // cheap checks that still catch obviously unhealthy startup state. + // cheap checks that still catch obviously unhealthy startup state. The + // DecentDB open probes are intentionally included because file + // existence alone misses unsupported local database format versions. + var checks = new List(); + using (Operation.At(LogEventLevel.Debug).Time("[{Service}] HasMissingConnectionStrings", nameof(DoctorService))) { if (HasMissingConnectionStrings()) { - return true; + checks.Add(new DoctorCheckResult( + "Configuration", + false, + "One or more required connection strings are missing", + TimeSpan.Zero)); } } - // Fast-path: only verify the MusicBrainz file exists and is non-empty. - // Full probe (query) is deferred to RunAllChecksAsync. using (Operation.At(LogEventLevel.Debug).Time("[{Service}] HasMusicBrainzFileIssues", nameof(DoctorService))) { - if (!HasConfiguredFileBackedDatabase("MusicBrainzConnection")) + var musicBrainzCheck = await RunMusicBrainzAttentionCheckAsync(cancellationToken); + if (!musicBrainzCheck.Success) { - return true; + checks.Add(musicBrainzCheck); } } - // Fast-path: only verify the ArtistSearch file exists and is non-empty. - // Full probe (query) is deferred to RunAllChecksAsync. using (Operation.At(LogEventLevel.Debug).Time("[{Service}] HasArtistSearchFileIssues", nameof(DoctorService))) { - if (!HasConfiguredFileBackedDatabase("ArtistSearchEngineConnection")) + var artistSearchCheck = await RunArtistSearchAttentionCheckAsync(cancellationToken); + if (!artistSearchCheck.Success) { - return true; + checks.Add(artistSearchCheck); } } @@ -96,13 +108,26 @@ public async Task NeedsAttentionAsync(CancellationToken cancellationToken try { await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - return !await db.Database.CanConnectAsync(cancellationToken); + if (!await db.Database.CanConnectAsync(cancellationToken)) + { + checks.Add(new DoctorCheckResult( + "PostgresDatabase", + false, + "Unable to connect to the primary database", + TimeSpan.Zero)); + } } - catch + catch (Exception ex) { - return true; + checks.Add(new DoctorCheckResult( + "PostgresDatabase", + false, + $"Unable to connect to the primary database: {FirstMessageLine(ex)}", + TimeSpan.Zero)); } } + + return checks; } public async Task IsMusicBrainzDatabaseEmptyAsync(CancellationToken cancellationToken = default) @@ -478,6 +503,103 @@ private bool HasConfiguredFileBackedDatabase(string connectionStringName) return HasNonEmptyFileBackedDatabase(configuration.GetConnectionString(connectionStringName)); } + private async Task RunMusicBrainzAttentionCheckAsync(CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + var connectionString = configuration.GetConnectionString("MusicBrainzConnection") ?? ""; + var fileInfo = DescribeFileDatabasePath(connectionString); + if (!HasNonEmptyFileBackedDatabase(connectionString)) + { + return new DoctorCheckResult( + "MusicBrainzDatabase", + false, + $"MusicBrainz DecentDB database file is missing or empty; {fileInfo}", + sw.Elapsed); + } + + try + { + await using var db = await _musicBrainzDbContextFactory.CreateDbContextAsync(cancellationToken); + var canConnect = await db.Database.CanConnectAsync(cancellationToken); + return new DoctorCheckResult( + "MusicBrainzDatabase", + canConnect, + canConnect + ? $"OK; {fileInfo}" + : $"MusicBrainz DecentDB database cannot be opened by the current DecentDB provider; {fileInfo}", + sw.Elapsed); + } + catch (Exception ex) + { + return new DoctorCheckResult( + "MusicBrainzDatabase", + false, + FormatDecentDbOpenFailure("MusicBrainz", ex, fileInfo), + sw.Elapsed); + } + } + + private async Task RunArtistSearchAttentionCheckAsync(CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + var connectionString = configuration.GetConnectionString("ArtistSearchEngineConnection") ?? ""; + var fileInfo = DescribeFileDatabasePath(connectionString); + if (!HasNonEmptyFileBackedDatabase(connectionString)) + { + return new DoctorCheckResult( + "ArtistSearchEngineDatabase", + false, + $"ArtistSearch DecentDB database file is missing or empty; {fileInfo}", + sw.Elapsed); + } + + try + { + await using var db = await _artistSearchEngineDbContextFactory.CreateDbContextAsync(cancellationToken); + var canConnect = await db.Database.CanConnectAsync(cancellationToken); + return new DoctorCheckResult( + "ArtistSearchEngineDatabase", + canConnect, + canConnect + ? $"OK; {fileInfo}" + : $"ArtistSearch DecentDB database cannot be opened by the current DecentDB provider; {fileInfo}", + sw.Elapsed); + } + catch (Exception ex) + { + return new DoctorCheckResult( + "ArtistSearchEngineDatabase", + false, + FormatDecentDbOpenFailure("ArtistSearch", ex, fileInfo), + sw.Elapsed); + } + } + + private static string FormatDecentDbOpenFailure(string databaseName, Exception exception, string fileInfo) + { + var message = FirstMessageLine(exception); + if (IsUnsupportedDecentDbFormat(message)) + { + return $"{databaseName} DecentDB database uses a file format that is not supported by the current DecentDB provider. Rebuild the database or upgrade Melodee/DecentDB. Provider error: {message}; {fileInfo}"; + } + + return $"Unable to open {databaseName} DecentDB database: {message}; {fileInfo}"; + } + + private static bool IsUnsupportedDecentDbFormat(string message) + { + return message.Contains("unsupported", StringComparison.OrdinalIgnoreCase) && + message.Contains("format", StringComparison.OrdinalIgnoreCase); + } + + private static string FirstMessageLine(Exception exception) + { + var message = exception.GetBaseException().Message; + return string.IsNullOrWhiteSpace(message) + ? exception.GetType().Name + : message.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? message; + } + private async Task HasMusicBrainzConnectionIssuesAsync(CancellationToken cancellationToken) { if (!HasConfiguredFileBackedDatabase("MusicBrainzConnection")) @@ -538,10 +660,12 @@ CREATE INDEX IF NOT EXISTS "IX_Artists_IsLocked_LastRefreshed" _ = await db.Artists .AsNoTracking() + .OrderBy(x => x.Id) .Select(x => x.Id) .FirstOrDefaultAsync(cancellationToken); _ = await db.Albums .AsNoTracking() + .OrderBy(x => x.Id) .Select(x => x.Id) .FirstOrDefaultAsync(cancellationToken); @@ -561,6 +685,7 @@ CREATE INDEX IF NOT EXISTS "IX_Artists_IsLocked_LastRefreshed" await using var db = await _musicBrainzDbContextFactory.CreateDbContextAsync(cancellationToken); var firstArtistId = await db.Artists .AsNoTracking() + .OrderBy(x => x.Id) .Select(x => x.Id) .FirstOrDefaultAsync(cancellationToken); @@ -1602,8 +1727,8 @@ private async Task RunDatabaseLatencyCheckAsync(CancellationT await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - // Simple query to measure latency - _ = await db.Settings.Take(1).CountAsync(cancellationToken); + // Simple existence query to measure latency. + _ = await db.Settings.AsNoTracking().AnyAsync(cancellationToken); queryStart.Stop(); var latencyMs = queryStart.Elapsed.TotalMilliseconds; diff --git a/src/Melodee.Blazor/Services/IDoctorService.cs b/src/Melodee.Blazor/Services/IDoctorService.cs index 273fa2a45..e17d0f545 100644 --- a/src/Melodee.Blazor/Services/IDoctorService.cs +++ b/src/Melodee.Blazor/Services/IDoctorService.cs @@ -19,6 +19,11 @@ public interface IDoctorService : Common.Services.Doctor.IDoctorService /// Task NeedsAttentionAsync(CancellationToken cancellationToken = default); + /// + /// Returns dashboard-safe health issues that should be shown to administrators immediately after login. + /// + Task> GetAttentionChecksAsync(CancellationToken cancellationToken = default); + /// /// Checks if the MusicBrainz database is empty or not properly initialized. /// diff --git a/src/Melodee.Blazor/Services/MainLayoutProxyService.cs b/src/Melodee.Blazor/Services/MainLayoutProxyService.cs index 6cdd9144d..4005a2d32 100644 --- a/src/Melodee.Blazor/Services/MainLayoutProxyService.cs +++ b/src/Melodee.Blazor/Services/MainLayoutProxyService.cs @@ -5,7 +5,7 @@ namespace Melodee.Blazor.Services; /// public sealed class MainLayoutProxyService { - public bool ShowSpinner { get; set; } + public bool ShowSpinner { get; private set; } public string Header { get; set; } = "Loading..."; @@ -22,7 +22,17 @@ public void SetHeader(string header) public void ToggleSpinnerVisible(bool? forceState = null) { - ShowSpinner = forceState ?? !ShowSpinner; + SetSpinnerVisible(forceState ?? !ShowSpinner); + } + + public void SetSpinnerVisible(bool isVisible) + { + if (ShowSpinner == isVisible) + { + return; + } + + ShowSpinner = isVisible; SpinnerVisibleChanged?.Invoke(this, EventArgs.Empty); } diff --git a/src/Melodee.Blazor/Services/MusicBrainzDecentDbWarmupHostedService.cs b/src/Melodee.Blazor/Services/MusicBrainzDecentDbWarmupHostedService.cs new file mode 100644 index 000000000..5310b8891 --- /dev/null +++ b/src/Melodee.Blazor/Services/MusicBrainzDecentDbWarmupHostedService.cs @@ -0,0 +1,40 @@ +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; + +namespace Melodee.Blazor.Services; + +internal sealed class MusicBrainzDecentDbWarmupHostedService( + IServiceScopeFactory scopeFactory, + Serilog.ILogger logger) : BackgroundService +{ + private static readonly TimeSpan StartupDelay = TimeSpan.FromSeconds(5); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await Task.Delay(StartupDelay, stoppingToken).ConfigureAwait(false); + + using var scope = scopeFactory.CreateScope(); + var warmupService = scope.ServiceProvider.GetRequiredService(); + var result = await warmupService.WarmHotIndexesAsync(stoppingToken).ConfigureAwait(false); + if (result.Succeeded || result.Skipped) + { + return; + } + + logger.Warning( + "MusicBrainz DecentDB startup warm-up did not complete: {Message}", + result.Message); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Normal shutdown path. + } + catch (Exception ex) + { + logger.Warning( + ex, + "MusicBrainz DecentDB startup warm-up failed. Search remains available; indexes will warm on demand."); + } + } +} diff --git a/src/Melodee.Cli/Command/CommandBase.cs b/src/Melodee.Cli/Command/CommandBase.cs index 8b26e5a99..0744b291d 100644 --- a/src/Melodee.Cli/Command/CommandBase.cs +++ b/src/Melodee.Cli/Command/CommandBase.cs @@ -100,6 +100,7 @@ protected ServiceProvider CreateServiceProvider() }); services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddSingleton(opt => new MemoryCacheManager(opt.GetRequiredService(), @@ -120,6 +121,7 @@ protected ServiceProvider CreateServiceProvider() services.AddSingleton(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Melodee.Cli/Command/DoctorCommand.cs b/src/Melodee.Cli/Command/DoctorCommand.cs index 3208e0842..04c3b1f21 100644 --- a/src/Melodee.Cli/Command/DoctorCommand.cs +++ b/src/Melodee.Cli/Command/DoctorCommand.cs @@ -1,6 +1,7 @@ using System.Data.Common; using System.Diagnostics; using System.Text.Json; +using DecentDB.AdoNet; using Melodee.Cli.CommandSettings; using Melodee.Common.Configuration; using Melodee.Common.Data; @@ -216,6 +217,19 @@ private static void RenderSummary(CliDoctorCheckResults results, TimeSpan elapse public sealed class CliDoctorService : DoctorServiceBase { + private static readonly string[] RequiredMusicBrainzTables = + [ + "Artist", + "Album", + "ArtistAlias" + ]; + + private static readonly string[] RequiredArtistSearchTables = + [ + "Artists", + "Albums" + ]; + private readonly Serilog.ILogger _logger; private readonly IDbContextFactory _dbContextFactory; private readonly IDbContextFactory _musicBrainzDbContextFactory; @@ -273,15 +287,22 @@ await RunCheckAsync(progress, checks, "Configuration", async () => await RunCheckAsync(progress, checks, "Database: MusicBrainz (DecentDB)", async () => { var checkSw = Stopwatch.StartNew(); - var (canQuery, _, error) = await ProbeMusicBrainzDatabaseAsync(cancellationToken); var cs = GetConnectionString("MusicBrainzConnection"); var fileInfo = DescribeFileDatabasePath(cs); - var details = canQuery - ? $"OK; {fileInfo}" - : string.IsNullOrWhiteSpace(error) - ? $"Unable to query; {fileInfo}" - : $"{error}; {fileInfo}"; - return new DoctorCheckResult("Database: MusicBrainz (DecentDB)", canQuery, details, checkSw.Elapsed); + var probe = await ProbeDecentDbDatabaseAsync( + cs, + RequiredMusicBrainzTables, + [ + """SELECT "Id" FROM "Artist" LIMIT 1""", + """SELECT "Id" FROM "Album" LIMIT 1""" + ], + requireRowsForReadQueries: true, + cancellationToken) + .ConfigureAwait(false); + var details = probe.Success + ? $"{probe.Details}; {fileInfo}" + : $"{probe.Details}; {fileInfo}"; + return new DoctorCheckResult("Database: MusicBrainz (DecentDB)", probe.Success, details, checkSw.Elapsed); }); await RunCheckAsync(progress, checks, "Database: ArtistSearchEngine (DecentDB)", async () => @@ -309,13 +330,20 @@ await RunCheckAsync(progress, checks, "Database: ArtistSearchEngine (DecentDB)", checkSw.Elapsed); } - var (canQuery, error) = await ProbeArtistSearchDatabaseAsync(cancellationToken); - var details = canQuery - ? $"OK; {fileInfo}" - : string.IsNullOrWhiteSpace(error) - ? $"Unable to query; {fileInfo}" - : $"{error}; {fileInfo}"; - return new DoctorCheckResult("Database: ArtistSearchEngine (DecentDB)", canQuery, details, checkSw.Elapsed); + var probe = await ProbeDecentDbDatabaseAsync( + cs, + RequiredArtistSearchTables, + [ + """SELECT "Id" FROM "Artists" LIMIT 1""", + """SELECT "Id" FROM "Albums" LIMIT 1""" + ], + requireRowsForReadQueries: false, + cancellationToken) + .ConfigureAwait(false); + var details = probe.Success + ? $"{probe.Details}; {fileInfo}" + : $"{probe.Details}; {fileInfo}"; + return new DoctorCheckResult("Database: ArtistSearchEngine (DecentDB)", probe.Success, details, checkSw.Elapsed); }); await RunCheckAsync(progress, checks, "Library Paths", async () => @@ -433,56 +461,86 @@ private static bool HasNonEmptyFileBackedDatabase(string connectionString) } } - private async Task<(bool CanQuery, string? Error)> ProbeArtistSearchDatabaseAsync( - CancellationToken cancellationToken) + public static async Task ProbeDecentDbDatabaseAsync( + string connectionString, + IReadOnlyCollection requiredTables, + IReadOnlyCollection readQueries, + bool requireRowsForReadQueries, + CancellationToken cancellationToken = default) { try { - await using var db = await _artistSearchEngineDbContextFactory.CreateDbContextAsync(cancellationToken); - await db.Database.EnsureCreatedAsync(cancellationToken); - if (db.Database.IsRelational()) + var dataSource = GetDataSourceFromConnectionString(connectionString); + if (string.IsNullOrWhiteSpace(dataSource)) + { + return DecentDbDoctorProbeResult.Fail("Data Source is empty or invalid"); + } + + if (!File.Exists(dataSource)) + { + return DecentDbDoctorProbeResult.Fail($"Database file does not exist: {dataSource}"); + } + + var databaseFile = new FileInfo(dataSource); + if (databaseFile.Length == 0) { - await db.Database.ExecuteSqlRawAsync( - """ - CREATE INDEX IF NOT EXISTS "IX_Artists_IsLocked_LastRefreshed" - ON "Artists" ("IsLocked", "LastRefreshed") - """, - cancellationToken); + return DecentDbDoctorProbeResult.Fail($"Database file is empty: {dataSource}"); } - _ = await db.Artists - .AsNoTracking() - .Select(x => x.Id) - .FirstOrDefaultAsync(cancellationToken); - _ = await db.Albums - .AsNoTracking() - .Select(x => x.Id) - .FirstOrDefaultAsync(cancellationToken); + await using var connection = new DecentDBConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + + var tables = ParseTableNames(connection.ListTablesJson()); + var missingTables = requiredTables + .Where(required => !tables.Contains(required, StringComparer.OrdinalIgnoreCase)) + .ToArray(); + if (missingTables.Length > 0) + { + return DecentDbDoctorProbeResult.Fail($"Missing required table(s): {string.Join(", ", missingTables)}"); + } + + foreach (var table in requiredTables) + { + var ddl = connection.GetTableDdl(table); + if (string.IsNullOrWhiteSpace(ddl)) + { + return DecentDbDoctorProbeResult.Fail($"Required table [{table}] has no DDL from schema introspection"); + } + } - return (true, null); + foreach (var query in readQueries) + { + await using var command = connection.CreateCommand(); + command.CommandText = query; + var scalar = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + if (requireRowsForReadQueries && (scalar is null || scalar == DBNull.Value)) + { + return DecentDbDoctorProbeResult.Fail($"Read query returned no rows: {query}"); + } + } + + return DecentDbDoctorProbeResult.Ok($"OK; opened; tables={tables.Length}; readQueries={readQueries.Count}"); } catch (Exception ex) { - return (false, ex.Message); + return DecentDbDoctorProbeResult.Fail($"Unable to open/query DecentDB: {ex.Message}"); } } - private async Task<(bool CanQuery, bool HasArtistData, string? Error)> ProbeMusicBrainzDatabaseAsync( - CancellationToken cancellationToken) + private static string[] ParseTableNames(string tablesJson) { - try + if (string.IsNullOrWhiteSpace(tablesJson)) { - await using var db = await _musicBrainzDbContextFactory.CreateDbContextAsync(cancellationToken); - var firstArtistId = await db.Artists - .AsNoTracking() - .Select(x => x.Id) - .FirstOrDefaultAsync(cancellationToken); + return []; + } - return (true, firstArtistId != 0, null); + try + { + return JsonSerializer.Deserialize(tablesJson) ?? []; } - catch (Exception ex) + catch { - return (false, false, ex.Message); + return []; } } } @@ -497,3 +555,10 @@ public sealed class CliDoctorCheckResults public int IssuesCount => Checks.Count(c => !c.Success); } + +public sealed record DecentDbDoctorProbeResult(bool Success, string Details) +{ + public static DecentDbDoctorProbeResult Ok(string details) => new(true, details); + + public static DecentDbDoctorProbeResult Fail(string details) => new(false, details); +} diff --git a/src/Melodee.Cli/Command/JobRunCommand.cs b/src/Melodee.Cli/Command/JobRunCommand.cs index 191492cdd..65c8457c0 100644 --- a/src/Melodee.Cli/Command/JobRunCommand.cs +++ b/src/Melodee.Cli/Command/JobRunCommand.cs @@ -289,7 +289,8 @@ private static JobBase CreateJob(Type jobType, IServiceProvider sp) sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService>(), - sp.GetRequiredService()); + sp.GetRequiredService(), + sp.GetRequiredService()); } if (jobType == typeof(NowPlayingCleanupJob)) diff --git a/src/Melodee.Cli/Command/JobRunMusicBrainzUpdateDatabaseJobCommand.cs b/src/Melodee.Cli/Command/JobRunMusicBrainzUpdateDatabaseJobCommand.cs index 4f6d26ad3..a21bdd6b4 100644 --- a/src/Melodee.Cli/Command/JobRunMusicBrainzUpdateDatabaseJobCommand.cs +++ b/src/Melodee.Cli/Command/JobRunMusicBrainzUpdateDatabaseJobCommand.cs @@ -23,7 +23,8 @@ protected override async Task ExecuteAsync(CommandContext context, JobSetti scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService>(), - scope.ServiceProvider.GetRequiredService() + scope.ServiceProvider.GetRequiredService(), + scope.ServiceProvider.GetRequiredService() ); await job.Execute(new MelodeeJobExecutionContext(cancellationToken)); return 1; diff --git a/src/Melodee.Cli/Command/LibraryScanCommand.cs b/src/Melodee.Cli/Command/LibraryScanCommand.cs index 91d86e555..9981ea213 100644 --- a/src/Melodee.Cli/Command/LibraryScanCommand.cs +++ b/src/Melodee.Cli/Command/LibraryScanCommand.cs @@ -4,8 +4,10 @@ using Melodee.Common.Configuration; using Melodee.Common.Data; using Melodee.Common.Jobs; +using Melodee.Common.Models; using Melodee.Common.Serialization; using Melodee.Common.Services; +using Melodee.Common.Services.Models; using Melodee.Common.Services.Scanning; using Melodee.Common.Services.SearchEngines; using Microsoft.EntityFrameworkCore; @@ -33,11 +35,160 @@ namespace Melodee.Cli.Command; /// public class LibraryScanCommand : CommandBase { - private static string FormatNumber(int number) + private sealed class ScanProgressState + { + private readonly object _lock = new(); + private int _current; + private int _max; + private string _message = "Starting..."; + + public void SetMessage(string message) + { + lock (_lock) + { + _message = message; + } + } + + public void Update(int current, int max, string message) + { + lock (_lock) + { + _current = Math.Max(0, current); + _max = Math.Max(0, max); + _message = message; + } + } + + public (int Current, int Max, string Message) Snapshot() + { + lock (_lock) + { + return (_current, _max, _message); + } + } + } + + private static string FormatNumber(long number) { return number.ToString("N0"); } + private static string FormatDurationMs(long milliseconds) + { + return TimeSpan.FromMilliseconds(milliseconds).ToString(@"hh\:mm\:ss"); + } + + private static string TrimProgressMessage(string message) + { + var singleLine = message + .ReplaceLineEndings(" ") + .Trim(); + + return singleLine.Length > 90 ? singleLine[..87] + "..." : singleLine; + } + + private static void ApplyProcessingEvent(ScanProgressState? progress, ProcessingEvent processingEvent) + { + progress?.Update( + processingEvent.Current, + processingEvent.Max, + processingEvent.Message); + } + + private static void UpdateProgressTask(ProgressTask stepTask, string stepName, ScanProgressState? progress) + { + if (progress is null) + { + return; + } + + var (current, max, message) = progress.Snapshot(); + if (max > 0) + { + var maxValue = Math.Max(1, max); + stepTask.IsIndeterminate = false; + stepTask.MaxValue = maxValue; + stepTask.Value = Math.Min(current, maxValue); + } + else + { + stepTask.IsIndeterminate = true; + } + + stepTask.Description = $"[green]{Markup.Escape(stepName)}[/] [dim]{Markup.Escape(TrimProgressMessage(message))}[/]"; + } + + private static async Task ExecuteStepWithProgressAsync( + string stepName, + Func> execute, + ProgressTask stepTask, + CancellationToken cancellationToken) + { + var progress = new ScanProgressState(); + var executionTask = execute(progress); + + while (!executionTask.IsCompleted) + { + UpdateProgressTask(stepTask, stepName, progress); + await Task.Delay(200, cancellationToken).ConfigureAwait(false); + } + + UpdateProgressTask(stepTask, stepName, progress); + + return await executionTask.ConfigureAwait(false); + } + + private static ScanStepResult AddStepResult(ScanStepResult summary, ScanStepResult result) + { + return summary with + { + NewArtistsCount = summary.NewArtistsCount + result.NewArtistsCount, + NewAlbumsCount = summary.NewAlbumsCount + result.NewAlbumsCount, + NewSongsCount = summary.NewSongsCount + result.NewSongsCount, + InboundProcessingErrors = summary.InboundProcessingErrors + result.InboundProcessingErrors, + AlbumsRevalidated = summary.AlbumsRevalidated + result.AlbumsRevalidated, + AlbumsNowValid = summary.AlbumsNowValid + result.AlbumsNowValid, + AlbumsRevalidationLookupsAttempted = summary.AlbumsRevalidationLookupsAttempted + result.AlbumsRevalidationLookupsAttempted, + AlbumsRevalidationNoMatch = summary.AlbumsRevalidationNoMatch + result.AlbumsRevalidationNoMatch, + AlbumsSkippedRevalidation = summary.AlbumsSkippedRevalidation + result.AlbumsSkippedRevalidation, + AlbumsDeferredRevalidation = summary.AlbumsDeferredRevalidation + result.AlbumsDeferredRevalidation, + AlbumsReadyToMove = summary.AlbumsReadyToMove + result.AlbumsReadyToMove, + AlbumsMoved = summary.AlbumsMoved + result.AlbumsMoved, + AlbumsMergedWithExisting = summary.AlbumsMergedWithExisting + result.AlbumsMergedWithExisting, + AlbumsSkippedByStatus = summary.AlbumsSkippedByStatus + result.AlbumsSkippedByStatus, + AlbumsSkippedAsDuplicateDirectory = summary.AlbumsSkippedAsDuplicateDirectory + result.AlbumsSkippedAsDuplicateDirectory, + AlbumsFailedToLoad = summary.AlbumsFailedToLoad + result.AlbumsFailedToLoad, + ArtistsInserted = summary.ArtistsInserted + result.ArtistsInserted, + AlbumsInserted = summary.AlbumsInserted + result.AlbumsInserted, + SongsInserted = summary.SongsInserted + result.SongsInserted, + AlbumsSkippedByReason = MergeSkippedReasonCounts(summary.AlbumsSkippedByReason, result.AlbumsSkippedByReason) + }; + } + + private static IReadOnlyDictionary? MergeSkippedReasonCounts( + IReadOnlyDictionary? existing, + IReadOnlyDictionary? incoming) + { + if ((existing?.Count ?? 0) == 0 && (incoming?.Count ?? 0) == 0) + { + return null; + } + + var merged = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var item in existing ?? Enumerable.Empty>()) + { + merged[item.Key] = item.Value; + } + + foreach (var item in incoming ?? Enumerable.Empty>()) + { + merged[item.Key] = merged.GetValueOrDefault(item.Key) + item.Value; + } + + return merged; + } + protected override async Task ExecuteAsync(CommandContext context, LibraryScanSettings settings, CancellationToken cancellationToken) { using var scope = CreateServiceProvider().CreateScope(); @@ -70,6 +221,7 @@ protected override async Task ExecuteAsync(CommandContext context, LibraryS var directoryProcessor = scope.ServiceProvider.GetRequiredService(); var albumDiscoveryService = scope.ServiceProvider.GetRequiredService(); var artistSearchEngineService = scope.ServiceProvider.GetRequiredService(); + var revalidationStateStore = scope.ServiceProvider.GetRequiredService(); var serializer = scope.ServiceProvider.GetRequiredService(); var fileSystemService = scope.ServiceProvider.GetRequiredService(); var dbContextFactory = scope.ServiceProvider.GetRequiredService>(); @@ -79,45 +231,155 @@ protected override async Task ExecuteAsync(CommandContext context, LibraryS var summary = new ScanStepResult(); var errors = new List(); + var warnings = new List(); + using var scanRunContext = new DirectoryRunContext(); - var steps = new (string Name, Func> Execute)[] + var steps = new (string Name, Func> Execute)[] { - ("Processing inbound files", async () => + ("Processing inbound files", async progress => { + progress?.SetMessage("Discovering inbound directories..."); var job = new LibraryInboundProcessJob(logger, configFactory, libraryService, directoryProcessor, schedulerFactory); var jobContext = new MelodeeJobExecutionContext(cancellationToken); jobContext.Put(MelodeeJobExecutionContext.ForceMode, settings.ForceMode); jobContext.Put(MelodeeJobExecutionContext.Verbose, settings.Verbose); - await job.Execute(jobContext); - return jobContext.Result as ScanStepResult; + jobContext.Put(MelodeeJobExecutionContext.DirectoryRunContext, scanRunContext); + var totalDirectories = 0; + var processedDirectories = 0; + + void OnProcessingStart(object? sender, int count) + { + totalDirectories = count; + progress?.Update(0, totalDirectories, $"Found [{count:N0}] directories to process"); + } + + void OnDirectoryProcessed(object? sender, FileSystemDirectoryInfo directoryInfo) + { + var current = Interlocked.Increment(ref processedDirectories); + progress?.Update(current, totalDirectories, $"Processed [{directoryInfo.Name}]"); + } + + void OnProcessingEvent(object? sender, string message) + { + progress?.Update(processedDirectories, totalDirectories, message); + } + + if (progress is not null) + { + directoryProcessor.OnProcessingStart += OnProcessingStart; + directoryProcessor.OnDirectoryProcessed += OnDirectoryProcessed; + directoryProcessor.OnProcessingEvent += OnProcessingEvent; + } + + try + { + await job.Execute(jobContext); + return jobContext.Result as ScanStepResult; + } + finally + { + if (progress is not null) + { + directoryProcessor.OnProcessingStart -= OnProcessingStart; + directoryProcessor.OnDirectoryProcessed -= OnDirectoryProcessed; + directoryProcessor.OnProcessingEvent -= OnProcessingEvent; + } + } }), - ("Revalidating staging albums", async () => + ("Revalidating staging albums", async progress => { - var job = new StagingAlbumRevalidationJob(logger, configFactory, libraryService, albumDiscoveryService, artistSearchEngineService, serializer, fileSystemService); + progress?.SetMessage("Discovering staged albums..."); + var job = new StagingAlbumRevalidationJob(logger, configFactory, libraryService, albumDiscoveryService, artistSearchEngineService, serializer, fileSystemService, revalidationStateStore); var jobContext = new MelodeeJobExecutionContext(cancellationToken); jobContext.Put(MelodeeJobExecutionContext.ForceMode, settings.ForceMode); - await job.Execute(jobContext); - return jobContext.Result as ScanStepResult; + jobContext.Put(MelodeeJobExecutionContext.DirectoryRunContext, scanRunContext); + + void OnProcessingEvent(object? sender, ProcessingEvent processingEvent) + { + ApplyProcessingEvent(progress, processingEvent); + } + + if (progress is not null) + { + job.OnProcessingEvent += OnProcessingEvent; + } + + try + { + await job.Execute(jobContext); + return jobContext.Result as ScanStepResult; + } + finally + { + if (progress is not null) + { + job.OnProcessingEvent -= OnProcessingEvent; + } + } }), - ("Moving approved albums to storage", async () => + ("Moving approved albums to storage", async progress => { + progress?.SetMessage("Finding approved staging albums..."); var job = new StagingAutoMoveJob(logger, configFactory, libraryService, schedulerFactory); var jobContext = new MelodeeJobExecutionContext(cancellationToken); - await job.Execute(jobContext); - return jobContext.Result as ScanStepResult; + + void OnProcessingEvent(object? sender, ProcessingEvent processingEvent) + { + ApplyProcessingEvent(progress, processingEvent); + } + + if (progress is not null) + { + libraryService.OnProcessingProgressEvent += OnProcessingEvent; + } + + try + { + await job.Execute(jobContext); + return jobContext.Result as ScanStepResult; + } + finally + { + if (progress is not null) + { + libraryService.OnProcessingProgressEvent -= OnProcessingEvent; + } + } }), - ("Inserting albums into database", async () => + ("Inserting albums into database", async progress => { + progress?.SetMessage("Discovering library metadata..."); var job = new LibraryInsertJob(logger, configFactory, libraryService, serializer, dbContextFactory, artistService, albumService, albumDiscoveryService, directoryProcessor, bus); var jobContext = new MelodeeJobExecutionContext(cancellationToken); jobContext.Put(MelodeeJobExecutionContext.ForceMode, settings.ForceMode); jobContext.Put(MelodeeJobExecutionContext.Verbose, settings.Verbose); - await job.Execute(jobContext); - return jobContext.Result as ScanStepResult; + + void OnProcessingEvent(object? sender, ProcessingEvent processingEvent) + { + ApplyProcessingEvent(progress, processingEvent); + } + + if (progress is not null) + { + job.OnProcessingEvent += OnProcessingEvent; + } + + try + { + await job.Execute(jobContext); + return jobContext.Result as ScanStepResult; + } + finally + { + if (progress is not null) + { + job.OnProcessingEvent -= OnProcessingEvent; + } + } }) }; - var stepResults = new Dictionary(); + var stepResults = new Dictionary(); if (isSilent) { @@ -126,27 +388,20 @@ protected override async Task ExecuteAsync(CommandContext context, LibraryS var stepStartTime = Stopwatch.GetTimestamp(); try { - var result = await execute(); + var result = await execute(null); if (result is not null) { - summary = summary with + summary = AddStepResult(summary, result); + if (result.HasWarnings) { - NewArtistsCount = summary.NewArtistsCount + result.NewArtistsCount, - NewAlbumsCount = summary.NewAlbumsCount + result.NewAlbumsCount, - NewSongsCount = summary.NewSongsCount + result.NewSongsCount, - AlbumsRevalidated = summary.AlbumsRevalidated + result.AlbumsRevalidated, - AlbumsNowValid = summary.AlbumsNowValid + result.AlbumsNowValid, - AlbumsMoved = summary.AlbumsMoved + result.AlbumsMoved, - ArtistsInserted = summary.ArtistsInserted + result.ArtistsInserted, - AlbumsInserted = summary.AlbumsInserted + result.AlbumsInserted, - SongsInserted = summary.SongsInserted + result.SongsInserted - }; + warnings.Add($"{name}: {result.NonFatalErrorCount:N0} non-fatal processing error(s); see log for details."); + } } - stepResults[name] = (true, Stopwatch.GetElapsedTime(stepStartTime)); + stepResults[name] = (true, result?.HasWarnings ?? false, Stopwatch.GetElapsedTime(stepStartTime)); } catch (Exception ex) { - stepResults[name] = (false, Stopwatch.GetElapsedTime(stepStartTime)); + stepResults[name] = (false, false, Stopwatch.GetElapsedTime(stepStartTime)); errors.Add($"{name}: {ex.Message}"); logger.Error(ex, "Error during {StepName}", name); } @@ -180,38 +435,37 @@ await AnsiConsole.Progress() try { - var result = await execute(); + var result = await ExecuteStepWithProgressAsync( + name, + execute, + stepTask, + cancellationToken).ConfigureAwait(false); if (result is not null) { - summary = summary with + summary = AddStepResult(summary, result); + if (result.HasWarnings) { - NewArtistsCount = summary.NewArtistsCount + result.NewArtistsCount, - NewAlbumsCount = summary.NewAlbumsCount + result.NewAlbumsCount, - NewSongsCount = summary.NewSongsCount + result.NewSongsCount, - AlbumsRevalidated = summary.AlbumsRevalidated + result.AlbumsRevalidated, - AlbumsNowValid = summary.AlbumsNowValid + result.AlbumsNowValid, - AlbumsMoved = summary.AlbumsMoved + result.AlbumsMoved, - ArtistsInserted = summary.ArtistsInserted + result.ArtistsInserted, - AlbumsInserted = summary.AlbumsInserted + result.AlbumsInserted, - SongsInserted = summary.SongsInserted + result.SongsInserted - }; + warnings.Add($"{name}: {result.NonFatalErrorCount:N0} non-fatal processing error(s); see log for details."); + } } var elapsed = Stopwatch.GetElapsedTime(stepStartTime); - stepTask.Description = $"[green]✓ {name}[/] [dim]({elapsed:mm\\:ss})[/]"; - stepTask.Value = 100; + stepTask.Description = result?.HasWarnings == true + ? $"[yellow]! {name}[/] [dim]({elapsed:mm\\:ss}, warnings)[/]" + : $"[green]✓ {name}[/] [dim]({elapsed:mm\\:ss})[/]"; stepTask.MaxValue = 100; + stepTask.Value = 100; stepTask.IsIndeterminate = false; - stepResults[name] = (true, elapsed); + stepResults[name] = (true, result?.HasWarnings ?? false, elapsed); } catch (Exception ex) { var elapsed = Stopwatch.GetElapsedTime(stepStartTime); stepTask.Description = $"[red]✗ {name}[/] [dim]({elapsed:mm\\:ss})[/]"; - stepTask.Value = 100; stepTask.MaxValue = 100; + stepTask.Value = 100; stepTask.IsIndeterminate = false; - stepResults[name] = (false, elapsed); + stepResults[name] = (false, false, elapsed); errors.Add($"{name}: {ex.Message}"); logger.Error(ex, "Error during {StepName}", name); } @@ -225,6 +479,15 @@ await AnsiConsole.Progress() } var totalElapsed = Stopwatch.GetElapsedTime(overallStartTime); + var performanceSummary = scanRunContext.GetPerformanceSummary(); + if (performanceSummary.ArtistSearchReadErrors > 0) + { + warnings.Add($"Artist search: {performanceSummary.ArtistSearchReadErrors:N0} read error(s); see log for details."); + } + if (performanceSummary.ArtistSearchReadCorruptions > 0) + { + warnings.Add($"Artist search DecentDB open/read failure reported {performanceSummary.ArtistSearchReadCorruptions:N0} time(s); verify the configured database path, WAL state, and .NET provider logs."); + } if (settings.Json) { @@ -237,6 +500,7 @@ await AnsiConsole.Progress() { name = s.Key, success = s.Value.Success, + warnings = s.Value.HasWarnings, durationSeconds = s.Value.Elapsed.TotalSeconds }), summary = new @@ -245,16 +509,28 @@ await AnsiConsole.Progress() { newArtists = summary.NewArtistsCount, newAlbums = summary.NewAlbumsCount, - newSongs = summary.NewSongsCount + newSongs = summary.NewSongsCount, + processingErrors = summary.InboundProcessingErrors }, stagingRevalidation = new { albumsRevalidated = summary.AlbumsRevalidated, - albumsNowValid = summary.AlbumsNowValid + albumsNowValid = summary.AlbumsNowValid, + albumsRevalidationLookupsAttempted = summary.AlbumsRevalidationLookupsAttempted, + albumsRevalidationNoMatch = summary.AlbumsRevalidationNoMatch, + albumsSkippedRevalidation = summary.AlbumsSkippedRevalidation, + albumsDeferredRevalidation = summary.AlbumsDeferredRevalidation }, storageTransfer = new { - albumsMoved = summary.AlbumsMoved + albumsReadyToMove = summary.AlbumsReadyToMove, + albumsMoved = summary.AlbumsMoved, + albumsMergedWithExisting = summary.AlbumsMergedWithExisting, + albumsHandled = summary.AlbumsHandledByStorageTransfer, + albumsSkippedByStatus = summary.AlbumsSkippedByStatus, + albumsSkippedAsDuplicateDirectory = summary.AlbumsSkippedAsDuplicateDirectory, + albumsFailedToLoad = summary.AlbumsFailedToLoad, + albumsSkippedByReason = summary.AlbumsSkippedByReason }, databaseInsert = new { @@ -263,6 +539,41 @@ await AnsiConsole.Progress() songsInserted = summary.SongsInserted } }, + warnings, + performance = new + { + runtimeMs = performanceSummary.RuntimeMs, + directoriesProcessed = performanceSummary.DirectoriesProcessed, + pluginTimeMs = performanceSummary.PluginTimeMs, + albumProcessingTimeMs = performanceSummary.AlbumProcessingTimeMs, + enrichmentTimeMs = performanceSummary.EnrichmentTimeMs, + conversionTimeMs = performanceSummary.ConversionTimeMs, + conversionFilesProcessed = performanceSummary.ConversionFilesProcessed, + copyTimeMs = performanceSummary.CopyTimeMs, + artistSearchPersistenceRetries = performanceSummary.ArtistSearchPersistenceRetries, + artistSearchPersistenceConflicts = performanceSummary.ArtistSearchPersistenceConflicts, + artistSearchPersistenceNonRetryableErrors = performanceSummary.ArtistSearchPersistenceCorruptions, + artistSearchReadErrors = performanceSummary.ArtistSearchReadErrors, + artistSearchReadOpenFailures = performanceSummary.ArtistSearchReadCorruptions, + albumsSkippedRevalidation = performanceSummary.AlbumsSkippedRevalidation, + albumsDeferredRevalidation = performanceSummary.AlbumsDeferredRevalidation, + artistSearchCache = new + { + entries = performanceSummary.ArtistSearchCache.TotalEntries, + hits = performanceSummary.ArtistSearchCache.Hits, + misses = performanceSummary.ArtistSearchCache.Misses, + coalesced = performanceSummary.ArtistSearchCache.CoalescedRequests, + hitRate = performanceSummary.ArtistSearchCache.HitRate + }, + forcedArtistSearchCache = new + { + entries = performanceSummary.ForcedArtistSearchCache.TotalEntries, + hits = performanceSummary.ForcedArtistSearchCache.Hits, + misses = performanceSummary.ForcedArtistSearchCache.Misses, + coalesced = performanceSummary.ForcedArtistSearchCache.CoalescedRequests, + hitRate = performanceSummary.ForcedArtistSearchCache.HitRate + } + }, errors = errors }; Console.WriteLine(JsonSerializer.Serialize(jsonOutput, new JsonSerializerOptions { WriteIndented = true })); @@ -285,7 +596,12 @@ await AnsiConsole.Progress() AnsiConsole.WriteLine(); var hasActivity = summary.NewArtistsCount > 0 || summary.NewAlbumsCount > 0 || summary.NewSongsCount > 0 || - summary.AlbumsRevalidated > 0 || summary.AlbumsMoved > 0 || + summary.InboundProcessingErrors > 0 || + summary.AlbumsRevalidated > 0 || summary.AlbumsRevalidationLookupsAttempted > 0 || + summary.AlbumsRevalidationNoMatch > 0 || summary.AlbumsSkippedRevalidation > 0 || + summary.AlbumsDeferredRevalidation > 0 || summary.AlbumsReadyToMove > 0 || + summary.AlbumsHandledByStorageTransfer > 0 || summary.AlbumsSkippedByStatus > 0 || + summary.AlbumsSkippedAsDuplicateDirectory > 0 || summary.AlbumsFailedToLoad > 0 || summary.ArtistsInserted > 0 || summary.AlbumsInserted > 0 || summary.SongsInserted > 0; if (hasActivity) @@ -298,7 +614,8 @@ await AnsiConsole.Progress() summaryTable.AddColumn("Category"); summaryTable.AddColumn(new TableColumn("Count").RightAligned()); - if (summary.NewArtistsCount > 0 || summary.NewAlbumsCount > 0 || summary.NewSongsCount > 0) + if (summary.NewArtistsCount > 0 || summary.NewAlbumsCount > 0 || + summary.NewSongsCount > 0 || summary.InboundProcessingErrors > 0) { summaryTable.AddRow("[bold]Inbound Processing[/]", ""); if (summary.NewArtistsCount > 0) @@ -307,21 +624,54 @@ await AnsiConsole.Progress() summaryTable.AddRow(" New albums discovered", FormatNumber(summary.NewAlbumsCount)); if (summary.NewSongsCount > 0) summaryTable.AddRow(" New songs discovered", FormatNumber(summary.NewSongsCount)); + if (summary.InboundProcessingErrors > 0) + summaryTable.AddRow(" Processing errors", $"[yellow]{FormatNumber(summary.InboundProcessingErrors)}[/]"); } - if (summary.AlbumsRevalidated > 0 || summary.AlbumsNowValid > 0) + if (summary.AlbumsRevalidated > 0 || summary.AlbumsNowValid > 0 || + summary.AlbumsRevalidationLookupsAttempted > 0 || summary.AlbumsRevalidationNoMatch > 0 || + summary.AlbumsSkippedRevalidation > 0 || summary.AlbumsDeferredRevalidation > 0) { summaryTable.AddRow("[bold]Staging Revalidation[/]", ""); if (summary.AlbumsRevalidated > 0) summaryTable.AddRow(" Albums revalidated", FormatNumber(summary.AlbumsRevalidated)); if (summary.AlbumsNowValid > 0) summaryTable.AddRow(" Albums now valid", $"[green]{FormatNumber(summary.AlbumsNowValid)}[/]"); + if (summary.AlbumsRevalidationLookupsAttempted > 0) + summaryTable.AddRow(" Artist lookups attempted", FormatNumber(summary.AlbumsRevalidationLookupsAttempted)); + if (summary.AlbumsRevalidationNoMatch > 0) + summaryTable.AddRow(" Artist lookups with no match", $"[yellow]{FormatNumber(summary.AlbumsRevalidationNoMatch)}[/]"); + if (summary.AlbumsSkippedRevalidation > 0) + summaryTable.AddRow(" Albums skipped", $"[yellow]{FormatNumber(summary.AlbumsSkippedRevalidation)}[/]"); + if (summary.AlbumsDeferredRevalidation > 0) + summaryTable.AddRow(" Albums deferred", $"[yellow]{FormatNumber(summary.AlbumsDeferredRevalidation)}[/]"); } - if (summary.AlbumsMoved > 0) + if (summary.AlbumsReadyToMove > 0 || + summary.AlbumsHandledByStorageTransfer > 0 || + summary.AlbumsSkippedByStatus > 0 || + summary.AlbumsSkippedAsDuplicateDirectory > 0 || + summary.AlbumsFailedToLoad > 0) { summaryTable.AddRow("[bold]Storage Transfer[/]", ""); - summaryTable.AddRow(" Albums moved to storage", FormatNumber(summary.AlbumsMoved)); + if (summary.AlbumsReadyToMove > 0) + summaryTable.AddRow(" Albums ready to move", FormatNumber(summary.AlbumsReadyToMove)); + if (summary.AlbumsMoved > 0) + summaryTable.AddRow(" New albums moved to storage", FormatNumber(summary.AlbumsMoved)); + if (summary.AlbumsMergedWithExisting > 0) + summaryTable.AddRow(" Albums merged with existing storage", FormatNumber(summary.AlbumsMergedWithExisting)); + if (summary.AlbumsHandledByStorageTransfer > 0) + summaryTable.AddRow(" Albums handled by storage transfer", FormatNumber(summary.AlbumsHandledByStorageTransfer)); + if (summary.AlbumsSkippedByStatus > 0) + summaryTable.AddRow(" Albums left in staging", $"[yellow]{FormatNumber(summary.AlbumsSkippedByStatus)}[/]"); + if (summary.AlbumsSkippedAsDuplicateDirectory > 0) + summaryTable.AddRow(" Duplicate-prefixed staging dirs skipped", FormatNumber(summary.AlbumsSkippedAsDuplicateDirectory)); + if (summary.AlbumsFailedToLoad > 0) + summaryTable.AddRow(" Albums failed to load", $"[red]{FormatNumber(summary.AlbumsFailedToLoad)}[/]"); + foreach (var skippedReason in summary.AlbumsSkippedByReason?.OrderBy(x => x.Key) ?? Enumerable.Empty>()) + { + summaryTable.AddRow($" {Markup.Escape(skippedReason.Key)}", $"[yellow]{FormatNumber(skippedReason.Value)}[/]"); + } } if (summary.ArtistsInserted > 0 || summary.AlbumsInserted > 0 || summary.SongsInserted > 0) @@ -342,6 +692,78 @@ await AnsiConsole.Progress() AnsiConsole.MarkupLine("[dim]No new content processed during this scan.[/]"); } + var hasPerformanceActivity = performanceSummary.ArtistSearchCache.Hits > 0 || + performanceSummary.ArtistSearchCache.Misses > 0 || + performanceSummary.ArtistSearchCache.CoalescedRequests > 0 || + performanceSummary.ForcedArtistSearchCache.Hits > 0 || + performanceSummary.ForcedArtistSearchCache.Misses > 0 || + performanceSummary.ForcedArtistSearchCache.CoalescedRequests > 0 || + performanceSummary.ConversionFilesProcessed > 0 || + performanceSummary.ArtistSearchPersistenceRetries > 0 || + performanceSummary.ArtistSearchPersistenceConflicts > 0 || + performanceSummary.ArtistSearchPersistenceCorruptions > 0 || + performanceSummary.ArtistSearchReadErrors > 0 || + performanceSummary.ArtistSearchReadCorruptions > 0 || + performanceSummary.AlbumsSkippedRevalidation > 0 || + performanceSummary.AlbumsDeferredRevalidation > 0; + if (hasPerformanceActivity) + { + AnsiConsole.WriteLine(); + var performanceTable = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Grey) + .Title("[yellow]Scan Performance[/]"); + + performanceTable.AddColumn("Metric"); + performanceTable.AddColumn(new TableColumn("Value").RightAligned()); + + performanceTable.AddRow("Directories processed", FormatNumber(performanceSummary.DirectoriesProcessed)); + performanceTable.AddRow("Artist cache hits", FormatNumber(performanceSummary.ArtistSearchCache.Hits)); + performanceTable.AddRow("Artist cache misses", FormatNumber(performanceSummary.ArtistSearchCache.Misses)); + performanceTable.AddRow("Artist cache coalesced", FormatNumber(performanceSummary.ArtistSearchCache.CoalescedRequests)); + performanceTable.AddRow("Forced artist cache hits", FormatNumber(performanceSummary.ForcedArtistSearchCache.Hits)); + performanceTable.AddRow("Forced artist cache misses", FormatNumber(performanceSummary.ForcedArtistSearchCache.Misses)); + performanceTable.AddRow("Forced artist cache coalesced", FormatNumber(performanceSummary.ForcedArtistSearchCache.CoalescedRequests)); + performanceTable.AddRow("Artist lookup time", FormatDurationMs(performanceSummary.EnrichmentTimeMs)); + performanceTable.AddRow("Conversion files", FormatNumber(performanceSummary.ConversionFilesProcessed)); + performanceTable.AddRow("Conversion time", FormatDurationMs(performanceSummary.ConversionTimeMs)); + performanceTable.AddRow("Copy time", FormatDurationMs(performanceSummary.CopyTimeMs)); + if (performanceSummary.ArtistSearchPersistenceConflicts > 0 || + performanceSummary.ArtistSearchPersistenceRetries > 0 || + performanceSummary.ArtistSearchPersistenceCorruptions > 0) + { + performanceTable.AddRow("DecentDB conflicts", FormatNumber(performanceSummary.ArtistSearchPersistenceConflicts)); + performanceTable.AddRow("DecentDB retries", FormatNumber(performanceSummary.ArtistSearchPersistenceRetries)); + performanceTable.AddRow("DecentDB non-retryable errors", FormatNumber(performanceSummary.ArtistSearchPersistenceCorruptions)); + } + if (performanceSummary.ArtistSearchReadErrors > 0 || + performanceSummary.ArtistSearchReadCorruptions > 0) + { + performanceTable.AddRow("Artist search read errors", FormatNumber(performanceSummary.ArtistSearchReadErrors)); + performanceTable.AddRow("Artist search open/read failures", FormatNumber(performanceSummary.ArtistSearchReadCorruptions)); + } + if (performanceSummary.AlbumsSkippedRevalidation > 0) + { + performanceTable.AddRow("Revalidation skipped", FormatNumber(performanceSummary.AlbumsSkippedRevalidation)); + } + if (performanceSummary.AlbumsDeferredRevalidation > 0) + { + performanceTable.AddRow("Revalidation deferred", FormatNumber(performanceSummary.AlbumsDeferredRevalidation)); + } + + AnsiConsole.Write(performanceTable); + } + + if (warnings.Count > 0) + { + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine($"[yellow]Warnings encountered: {warnings.Count}[/]"); + foreach (var warning in warnings) + { + AnsiConsole.MarkupLine($" [yellow]• {Markup.Escape(warning)}[/]"); + } + } + if (errors.Count > 0) { AnsiConsole.WriteLine(); diff --git a/src/Melodee.Cli/Melodee.Cli.csproj b/src/Melodee.Cli/Melodee.Cli.csproj index 8079a7d7f..f8ca485a6 100644 --- a/src/Melodee.Cli/Melodee.Cli.csproj +++ b/src/Melodee.Cli/Melodee.Cli.csproj @@ -9,7 +9,7 @@ false $(NoWarn);NU1507 - 2.1.0 + 2.1.3 build$([System.DateTime]::UtcNow.ToString("yyyyMMddHHmmss")) $(VersionPrefix).0 diff --git a/src/Melodee.Cli/Program.cs b/src/Melodee.Cli/Program.cs index ec4a1dbb4..c68da3527 100644 --- a/src/Melodee.Cli/Program.cs +++ b/src/Melodee.Cli/Program.cs @@ -9,6 +9,8 @@ public static class Program { public static int Main(string[] args) { + ATL.Settings.OutputStacktracesToConsole = false; + // Support "mcli help [command path]" as an alias for "--help" / " --help". // (Spectre.Console.Cli supports --help, but users commonly expect a help subcommand.) if (args.Length > 0 && string.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Melodee.Common/Constants/SettingRegistry.cs b/src/Melodee.Common/Constants/SettingRegistry.cs index 8f21f0601..c3e8a3aca 100644 --- a/src/Melodee.Common/Constants/SettingRegistry.cs +++ b/src/Melodee.Common/Constants/SettingRegistry.cs @@ -72,6 +72,7 @@ public static class SettingRegistry public const string PlaylistDynamicPlaylistsDisabled = "playlist.dynamicPlaylist.disabled"; public const string PlaylistMaximumAllowedPageSize = "playlist.maximumAllowedPageSize"; + public const string PluginEnabledBlackbeard = "plugin.blackbeard.enabled"; public const string PluginEnabledCueSheet = "plugin.cueSheet.enabled"; public const string PluginEnabledM3u = "plugin.m3u.enabled"; public const string PluginEnabledNfo = "plugin.nfo.enabled"; diff --git a/src/Melodee.Common/Jobs/LibraryInboundProcessJob.cs b/src/Melodee.Common/Jobs/LibraryInboundProcessJob.cs index 6c73504c2..d5a782cbe 100644 --- a/src/Melodee.Common/Jobs/LibraryInboundProcessJob.cs +++ b/src/Melodee.Common/Jobs/LibraryInboundProcessJob.cs @@ -99,11 +99,14 @@ public override async Task Execute(IJobExecutionContext context) dataMap[JobMapNameRegistry.ScanStatus] = ScanStatus.InProcess.ToString(); await directoryProcessorToStagingService.InitializeAsync(null, context.CancellationToken) .ConfigureAwait(false); + var runContext = context.MergedJobDataMap.ContainsKey(MelodeeJobExecutionContext.DirectoryRunContext) + ? context.MergedJobDataMap[MelodeeJobExecutionContext.DirectoryRunContext] as DirectoryRunContext + : null; var result = await directoryProcessorToStagingService.ProcessDirectoryAsync(new FileSystemDirectoryInfo { Path = directoryInbound, Name = directoryInbound - }, inboundLibrary.LastScanAt, null, context.CancellationToken).ConfigureAwait(false); + }, inboundLibrary.LastScanAt, null, runContext, context.CancellationToken).ConfigureAwait(false); if (!result.IsSuccess) { @@ -129,7 +132,8 @@ await directoryProcessorToStagingService.InitializeAsync(null, context.Cancellat context.Result = new ScanStepResult( NewArtistsCount: result.Data.NewArtistsCount, NewAlbumsCount: result.Data.NewAlbumsCount, - NewSongsCount: result.Data.NewSongsCount); + NewSongsCount: result.Data.NewSongsCount, + InboundProcessingErrors: result.Errors?.Count() ?? 0); // Chain to StagingAutoMoveJob if this was a scheduled run (not manual) and we processed something, // or if ChainOnComplete flag is set (for programmatic triggers like after upload) diff --git a/src/Melodee.Common/Jobs/LibraryInsertJob.cs b/src/Melodee.Common/Jobs/LibraryInsertJob.cs index 01bf77f64..7c92f53f2 100644 --- a/src/Melodee.Common/Jobs/LibraryInsertJob.cs +++ b/src/Melodee.Common/Jobs/LibraryInsertJob.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Diagnostics; -using IdSharp.Common.Utils; using Melodee.Common.Configuration; using Melodee.Common.Constants; using Melodee.Common.Data; @@ -439,7 +438,7 @@ private async Task ProcessAlbumsAsync( var artistNormalizedName = artistName.ToNormalizedString() ?? artistName; var dbArtistResult = await artistService.FindArtistAsync(melodeeAlbum.Artist.ArtistDbId, melodeeAlbum.Artist.Id, artistNormalizedName, melodeeAlbum.Artist.MusicBrainzId, - melodeeAlbum.Artist.SpotifyId, cancellationToken).ConfigureAwait(false); + melodeeAlbum.Artist.SpotifyId, melodeeAlbum.Artist.ItunesId, cancellationToken).ConfigureAwait(false); var dbArtistId = dbArtistResult.Data?.Id; var dbArtist = dbArtistId == null ? null @@ -556,7 +555,7 @@ await bus.SendLocal(new MelodeeAlbumReprocessEvent(melodeeAlbum.Directory.FullNa break; } - var mediaFileHash = CRC32.Calculate(mediaFile); + var mediaFileHash = Crc32.Calculate(mediaFile); var songTitle = song.Title()?.CleanStringAsIs() ?? throw new Exception("Song title is required."); var s = new dbModels.Song @@ -790,11 +789,14 @@ await contextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals .Select(x => x.MusicBrainzId!.Value).ToArray(); var artistSpotifyIds = artists.Where(x => !string.IsNullOrEmpty(x.SpotifyId)) .Select(x => x.SpotifyId!).ToArray(); + var artistItunesIds = artists.Where(x => !string.IsNullOrEmpty(x.ItunesId)) + .Select(x => x.ItunesId!).ToArray(); var existingArtists = await scopedContext.Artists .Where(x => artistNormalizedNames.Contains(x.NameNormalized) || (x.MusicBrainzId.HasValue && artistMusicBrainzIds.Contains(x.MusicBrainzId.Value)) || - (x.SpotifyId != null && artistSpotifyIds.Contains(x.SpotifyId))) + (x.SpotifyId != null && artistSpotifyIds.Contains(x.SpotifyId)) || + (x.ItunesId != null && artistItunesIds.Contains(x.ItunesId))) .ToListAsync(cancellationToken) .ConfigureAwait(false); @@ -807,6 +809,8 @@ await contextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals existingArtistLookup.TryAdd($"mb:{existingArtist.MusicBrainzId.Value}", existingArtist); if (!string.IsNullOrEmpty(existingArtist.SpotifyId)) existingArtistLookup.TryAdd($"sp:{existingArtist.SpotifyId}", existingArtist); + if (!string.IsNullOrEmpty(existingArtist.ItunesId)) + existingArtistLookup.TryAdd($"it:{existingArtist.ItunesId}", existingArtist); } var dbArtistsToAdd = new List(); @@ -825,6 +829,11 @@ await contextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals { // Found by Spotify ID } + else if (!string.IsNullOrEmpty(artist.ItunesId) && + existingArtistLookup.TryGetValue($"it:{artist.ItunesId}", out existingArtist)) + { + // Found by iTunes ID + } else if (existingArtistLookup.TryGetValue($"name:{artist.NameNormalized}", out existingArtist)) { // Found by normalized name @@ -874,6 +883,8 @@ await contextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals existingArtistLookup.TryAdd($"mb:{artist.MusicBrainzId.Value}", newArtist); if (!string.IsNullOrEmpty(artist.SpotifyId)) existingArtistLookup.TryAdd($"sp:{artist.SpotifyId}", newArtist); + if (!string.IsNullOrEmpty(artist.ItunesId)) + existingArtistLookup.TryAdd($"it:{artist.ItunesId}", newArtist); } } diff --git a/src/Melodee.Common/Jobs/MelodeeJobExecutionContext.cs b/src/Melodee.Common/Jobs/MelodeeJobExecutionContext.cs index f13a0ff0d..eb16090a6 100644 --- a/src/Melodee.Common/Jobs/MelodeeJobExecutionContext.cs +++ b/src/Melodee.Common/Jobs/MelodeeJobExecutionContext.cs @@ -199,6 +199,7 @@ public class MelodeeJobExecutionContext(CancellationToken cancellation) : IJobEx public const string ScanJustDirectory = "ScanJustDirectory"; public const string Verbose = "Verbose"; public const string ChainOnComplete = "ChainOnComplete"; + public const string DirectoryRunContext = "DirectoryRunContext"; private readonly Dictionary _dataMap = new(); @@ -218,6 +219,11 @@ public void Put(object key, object objectValue) { _dataMap[key] = objectValue; } + + if (key is string stringKey) + { + MergedJobDataMap[stringKey] = objectValue; + } } public object? Get(object key) @@ -232,7 +238,7 @@ public void Put(object key, object objectValue) public bool Recovering { get; } = false; public TriggerKey RecoveringTriggerKey { get; } = null!; public int RefireCount { get; } = 0; - public JobDataMap MergedJobDataMap { get; } = null!; + public JobDataMap MergedJobDataMap { get; } = new(); public IJobDetail JobDetail { get; } = new JobDetailImpl(); public IJob JobInstance { get; } = null!; public DateTimeOffset FireTimeUtc { get; } = DateTimeOffset.MinValue; diff --git a/src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs b/src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs index 21e2aaeba..db8997e69 100644 --- a/src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs +++ b/src/Melodee.Common/Jobs/MusicBrainzUpdateDatabaseJob.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Text; using System.Text.Json; +using DecentDB.AdoNet; using ICSharpCode.SharpZipLib.BZip2; using ICSharpCode.SharpZipLib.Tar; using Melodee.Common.Configuration; @@ -79,7 +80,8 @@ public class MusicBrainzUpdateDatabaseJob( SettingService settingService, IHttpClientFactory httpClientFactory, IDbContextFactory dbContextFactory, - IMusicBrainzRepository repository) : JobBase(logger, configurationFactory) + IMusicBrainzRepository repository, + MusicBrainzDecentDbWarmupService warmupService) : JobBase(logger, configurationFactory) { private const string StageInitialize = "Initialize"; private const string StageDownloadMbDump = "Download mbdump.tar.bz2"; @@ -90,7 +92,16 @@ public class MusicBrainzUpdateDatabaseJob( private const int ImportStageScale = 1000; private const string ImportingDatabaseSuffix = ".importing"; private const string BackupDatabaseSuffix = ".backup"; - private const string DecentDbCliPathEnvironmentVariable = "DECENTDB_CLI_PATH"; + + private static readonly string[] DatabaseArtifactSuffixes = + [ + string.Empty, + ".wal", + "-wal", + ".shm", + "-shm", + ".coord" + ]; private static readonly string[] RequiredArchiveEntries = [ @@ -528,6 +539,17 @@ await settingService DeleteDatabaseArtifacts(dbName); MoveDatabaseArtifacts(importDbName, dbName, overwrite: true); + progress?.UpdateProgress("Warming MusicBrainz DecentDB indexes..."); + var warmupResult = await warmupService.WarmHotIndexesAsync(dbName, context.CancellationToken) + .ConfigureAwait(false); + if (!warmupResult.Succeeded && !warmupResult.Skipped) + { + Logger.Warning( + "[{JobName}] MusicBrainz DecentDB warm-up did not complete after promotion: {Message}", + nameof(MusicBrainzUpdateDatabaseJob), + warmupResult.Message); + } + if (tempDbName != null) { progress?.UpdateProgress("Deleting backup database..."); @@ -915,7 +937,7 @@ private static bool IsProcessRunning(int processId) private static void DeleteDatabaseArtifacts(string dbPath) { - foreach (var path in new[] { dbPath, $"{dbPath}.wal", $"{dbPath}.shm" }) + foreach (var path in DatabaseArtifactSuffixes.Select(suffix => $"{dbPath}{suffix}")) { if (File.Exists(path)) { @@ -926,7 +948,7 @@ private static void DeleteDatabaseArtifacts(string dbPath) private static void MoveDatabaseArtifacts(string sourceDbPath, string destinationDbPath, bool overwrite) { - foreach (var suffix in new[] { string.Empty, ".wal", ".shm" }) + foreach (var suffix in DatabaseArtifactSuffixes) { var sourcePath = $"{sourceDbPath}{suffix}"; if (!File.Exists(sourcePath)) @@ -946,82 +968,27 @@ private static void MoveDatabaseArtifacts(string sourceDbPath, string destinatio private async Task CheckpointImportedDatabaseAsync(string databasePath, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (!File.Exists(databasePath)) { throw new FileNotFoundException("Cannot checkpoint imported DecentDB database because the database file does not exist.", databasePath); } - var walPath = $"{databasePath}.wal"; - var walBytesBefore = File.Exists(walPath) ? new FileInfo(walPath).Length : 0; - var executablePath = ResolveDecentDbExecutablePath(); + var walStatusBefore = DecentDBMaintenance.GetWalStatus(databasePath); - if (!File.Exists(executablePath)) - { - Logger.Warning("[{JobName}] DecentDB CLI tool not found at '{ExecutablePath}'. Skipping WAL checkpoint. The import will still function correctly, but the WAL file may remain until the database is next opened by a process that supports checkpointing.", nameof(MusicBrainzUpdateDatabaseJob), executablePath); - return; - } - - var processInfo = new ProcessStartInfo - { - FileName = executablePath, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - processInfo.ArgumentList.Add("checkpoint"); - processInfo.ArgumentList.Add("--db"); - processInfo.ArgumentList.Add(databasePath); - - Logger.Information("[{JobName}] Running DecentDB checkpoint for imported database. WAL before checkpoint: {WalSize}", + Logger.Information("[{JobName}] Running DecentDB checkpoint through ADO.NET binding. WAL before checkpoint: {WalSize}", nameof(MusicBrainzUpdateDatabaseJob), - FormatBytes(walBytesBefore)); - - var checkpointStartTicks = Stopwatch.GetTimestamp(); - using var process = Process.Start(processInfo) - ?? throw new InvalidOperationException($"Failed to start DecentDB checkpoint executable [{executablePath}]."); - var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken); - var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken); + FormatBytes(walStatusBefore.TotalWalBytes)); - try - { - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - TryTerminateProcess(process); - throw; - } - - var stdout = await stdoutTask.ConfigureAwait(false); - var stderr = await stderrTask.ConfigureAwait(false); - if (process.ExitCode != 0) - { - throw new InvalidOperationException( - $"DecentDB checkpoint failed with exit code {process.ExitCode}. " + - $"stdout=[{TruncateProcessOutput(stdout)}] stderr=[{TruncateProcessOutput(stderr)}]"); - } - - var walBytesAfter = File.Exists(walPath) ? new FileInfo(walPath).Length : 0; + var checkpointResult = await DecentDBMaintenance + .CheckpointAsync(databasePath, cancellationToken) + .ConfigureAwait(false); Logger.Information( "[{JobName}] DecentDB checkpoint complete in {Elapsed:F1}s. WAL after checkpoint: {WalSize}", nameof(MusicBrainzUpdateDatabaseJob), - Stopwatch.GetElapsedTime(checkpointStartTicks).TotalSeconds, - FormatBytes(walBytesAfter)); - } - - private static string ResolveDecentDbExecutablePath() - { - var configuredPath = Environment.GetEnvironmentVariable(DecentDbCliPathEnvironmentVariable); - return string.IsNullOrWhiteSpace(configuredPath) - ? "decentdb" - : configuredPath; - } - - private static string TruncateProcessOutput(string output) - { - output = output.Trim(); - return output.Length <= 1000 ? output : string.Concat(output.AsSpan(0, 1000), "..."); + checkpointResult.Duration.TotalSeconds, + FormatBytes(checkpointResult.After.TotalWalBytes)); } private static bool HasAllRequiredExtractedFiles(string mbDumpDirectory) @@ -1177,7 +1144,7 @@ await settingService } var expectedImportPath = GetImportDatabaseFilePath(dbPath); - return File.Exists(expectedImportPath) || File.Exists($"{expectedImportPath}.wal") || File.Exists($"{expectedImportPath}.shm") + return DatabaseArtifactSuffixes.Any(suffix => File.Exists($"{expectedImportPath}{suffix}")) ? expectedImportPath : null; } diff --git a/src/Melodee.Common/Jobs/ScanStepResult.cs b/src/Melodee.Common/Jobs/ScanStepResult.cs index f4286bd05..f74bef35b 100644 --- a/src/Melodee.Common/Jobs/ScanStepResult.cs +++ b/src/Melodee.Common/Jobs/ScanStepResult.cs @@ -4,9 +4,27 @@ public sealed record ScanStepResult( int NewArtistsCount = 0, int NewAlbumsCount = 0, int NewSongsCount = 0, + int InboundProcessingErrors = 0, int AlbumsRevalidated = 0, int AlbumsNowValid = 0, + int AlbumsRevalidationLookupsAttempted = 0, + int AlbumsRevalidationNoMatch = 0, + int AlbumsSkippedRevalidation = 0, + int AlbumsDeferredRevalidation = 0, + int AlbumsReadyToMove = 0, int AlbumsMoved = 0, + int AlbumsMergedWithExisting = 0, + int AlbumsSkippedByStatus = 0, + int AlbumsSkippedAsDuplicateDirectory = 0, + int AlbumsFailedToLoad = 0, int ArtistsInserted = 0, int AlbumsInserted = 0, - int SongsInserted = 0); + int SongsInserted = 0, + IReadOnlyDictionary? AlbumsSkippedByReason = null) +{ + public int AlbumsHandledByStorageTransfer => AlbumsMoved + AlbumsMergedWithExisting; + + public int NonFatalErrorCount => InboundProcessingErrors; + + public bool HasWarnings => NonFatalErrorCount > 0; +} diff --git a/src/Melodee.Common/Jobs/StagingAlbumRevalidationJob.cs b/src/Melodee.Common/Jobs/StagingAlbumRevalidationJob.cs index be8757009..9ee95a3c1 100644 --- a/src/Melodee.Common/Jobs/StagingAlbumRevalidationJob.cs +++ b/src/Melodee.Common/Jobs/StagingAlbumRevalidationJob.cs @@ -11,6 +11,7 @@ using Melodee.Common.Services.Models; using Melodee.Common.Services.Scanning; using Melodee.Common.Services.SearchEngines; +using Melodee.Common.Utility; using Quartz; using Serilog; @@ -61,19 +62,55 @@ public class StagingAlbumRevalidationJob( AlbumDiscoveryService albumDiscoveryService, ArtistSearchEngineService artistSearchEngineService, ISerializer serializer, - IFileSystemService fileSystemService) : JobBase(logger, configurationFactory) + IFileSystemService fileSystemService, + IStagingAlbumRevalidationStateStore revalidationStateStore) : JobBase(logger, configurationFactory) { /// /// This is raised when a Log event happens to return activity to caller. /// public event EventHandler? OnProcessingEvent; + /// + /// Returns whether an album has enough useful artist information to spend a revalidation lookup on it. + /// + public static bool CanAttemptArtistRevalidation(Album album) + { + if (album.Artist.IsValid()) + { + return true; + } + + if (album.Artist.Name.Nullify() == null) + { + return false; + } + + if (album.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.ArtistNameHasUnwantedText)) + { + return false; + } + + return album.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists) || + album.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasUnknownArtist); + } + public override async Task Execute(IJobExecutionContext context) { var startTicks = Stopwatch.GetTimestamp(); + var albumsProcessed = 0; var albumsRevalidated = 0; var albumsNowValid = 0; + var albumsRevalidationLookupsAttempted = 0; + var albumsRevalidationNoMatch = 0; + var albumsSkippedRevalidation = 0; + var albumsDeferredRevalidation = 0; var dataMap = context.JobDetail.JobDataMap; + var forceMode = SafeParser.ToBoolean(context.Get(MelodeeJobExecutionContext.ForceMode)); + var sharedRunContext = context.MergedJobDataMap.ContainsKey(MelodeeJobExecutionContext.DirectoryRunContext) + ? context.MergedJobDataMap[MelodeeJobExecutionContext.DirectoryRunContext] as DirectoryRunContext + : null; + using var localRunContext = sharedRunContext is null ? new DirectoryRunContext() : null; + var activeRunContext = sharedRunContext ?? localRunContext!; try { @@ -130,6 +167,12 @@ public override async Task Execute(IJobExecutionContext context) nameof(StagingAlbumRevalidationJob), albumsNeedingRevalidation.Length); + await using var revalidationStateSession = await OpenRevalidationStateSessionAsync( + stagingLibrary.Path, + albumsNeedingRevalidation, + context.CancellationToken) + .ConfigureAwait(false); + OnProcessingEvent?.Invoke( this, new ProcessingEvent(ProcessingEventType.Start, @@ -147,83 +190,161 @@ public override async Task Execute(IJobExecutionContext context) try { - var searchRequest = album.Artist.ToArtistQuery([ - new KeyValue((album.AlbumYear() ?? 0).ToString(), - album.AlbumTitle().ToNormalizedString() ?? album.AlbumTitle()) - ]); + albumsProcessed++; + var now = DateTimeOffset.UtcNow; + var revalidationDecision = revalidationStateSession.GetDecision(album, now, forceMode); + if (!revalidationDecision.IsDue) + { + albumsDeferredRevalidation++; + activeRunContext.RecordAlbumDeferredRevalidation(); + Logger.Debug( + "[{JobName}] Deferring album [{Album}] artist revalidation until [{NextAttemptAt}] after [{AttemptCount}] attempts", + nameof(StagingAlbumRevalidationJob), + album.AlbumTitle(), + revalidationDecision.NextAttemptAt, + revalidationDecision.AttemptCount); + OnProcessingEvent?.Invoke( + this, + new ProcessingEvent(ProcessingEventType.Processing, + nameof(StagingAlbumRevalidationJob), + albumsNeedingRevalidation.Length, + albumsProcessed, + ProgressMessage(albumsProcessed, albumsNeedingRevalidation.Length, albumsRevalidated, albumsRevalidationLookupsAttempted, albumsRevalidationNoMatch, albumsSkippedRevalidation, albumsDeferredRevalidation))); + continue; + } + + if (!CanAttemptArtistRevalidation(album)) + { + albumsSkippedRevalidation++; + activeRunContext.RecordAlbumSkippedRevalidation(); + revalidationStateSession.RecordAttempt(album, now, "ArtistDataNotSearchable"); + await revalidationStateSession.SaveChangesAsync(context.CancellationToken).ConfigureAwait(false); + Logger.Debug( + "[{JobName}] Skipping album [{Album}] revalidation because artist data is not searchable: [{Reasons}]", + nameof(StagingAlbumRevalidationJob), + album.AlbumTitle(), + album.StatusReasons); + OnProcessingEvent?.Invoke( + this, + new ProcessingEvent(ProcessingEventType.Processing, + nameof(StagingAlbumRevalidationJob), + albumsNeedingRevalidation.Length, + albumsProcessed, + ProgressMessage(albumsProcessed, albumsNeedingRevalidation.Length, albumsRevalidated, albumsRevalidationLookupsAttempted, albumsRevalidationNoMatch, albumsSkippedRevalidation, albumsDeferredRevalidation))); + continue; + } - var artistSearchResult = await artistSearchEngineService.DoSearchAsync( - searchRequest, - 1, - context.CancellationToken).ConfigureAwait(false); + var shouldRevalidateAlbum = album.Artist.IsValid(); + var artistLookupAttempted = false; - if (artistSearchResult.IsSuccess && artistSearchResult.Data.Any()) + if (!shouldRevalidateAlbum) { - var artistFromSearch = artistSearchResult.Data.OrderByDescending(x => x.Rank).FirstOrDefault(); - if (artistFromSearch != null) + artistLookupAttempted = true; + albumsRevalidationLookupsAttempted++; + var searchRequest = album.Artist.ToArtistQuery([ + new KeyValue((album.AlbumYear() ?? 0).ToString(), + album.AlbumTitle().ToNormalizedString() ?? album.AlbumTitle()) + ]); + + var artistSearchResult = await artistSearchEngineService.DoSearchAsync( + searchRequest, + 1, + activeRunContext, + bypassNegativeCache: true, + context.CancellationToken).ConfigureAwait(false); + + if (artistSearchResult.IsSuccess && artistSearchResult.Data.Any()) { - album.Artist = album.Artist with + var artistFromSearch = artistSearchResult.Data.OrderByDescending(x => x.Rank).FirstOrDefault(); + if (artistFromSearch != null) { - AmgId = album.Artist.AmgId ?? artistFromSearch.AmgId, - ArtistDbId = album.Artist.ArtistDbId ?? artistFromSearch.Id, - DiscogsId = album.Artist.DiscogsId ?? artistFromSearch.DiscogsId, - ItunesId = album.Artist.ItunesId ?? artistFromSearch.ItunesId, - LastFmId = album.Artist.LastFmId ?? artistFromSearch.LastFmId, - MusicBrainzId = album.Artist.MusicBrainzId ?? artistFromSearch.MusicBrainzId, - Name = album.Artist.Name.Nullify() ?? artistFromSearch.Name, - NameNormalized = album.Artist.NameNormalized.Nullify() ?? - artistFromSearch.Name.ToNormalizedString() ?? - artistFromSearch.Name, - OriginalName = artistFromSearch.Name != album.Artist.Name ? album.Artist.Name : null, - SearchEngineResultUniqueId = album.Artist.SearchEngineResultUniqueId is null or < 1 - ? artistFromSearch.UniqueId - : album.Artist.SearchEngineResultUniqueId, - SortName = album.Artist.SortName.Nullify() ?? artistFromSearch.SortName, - SpotifyId = album.Artist.SpotifyId ?? artistFromSearch.SpotifyId, - WikiDataId = album.Artist.WikiDataId ?? artistFromSearch.WikiDataId - }; - - var validationResult = albumValidator.ValidateAlbum(album); - album.ValidationMessages = validationResult.Data.Messages ?? []; - album.Status = validationResult.Data.AlbumStatus; - album.StatusReasons = validationResult.Data.AlbumStatusReasons; - album.Modified = DateTimeOffset.UtcNow; - - var jsonPath = fileSystemService.CombinePath(album.Directory.FullName(), Album.JsonFileName); - var serialized = serializer.Serialize(album); - await fileSystemService.WriteAllBytesAsync( - jsonPath, - System.Text.Encoding.UTF8.GetBytes(serialized ?? string.Empty), - context.CancellationToken).ConfigureAwait(false); - - albumsRevalidated++; - - if (album.Status == AlbumStatus.Ok) - { - albumsNowValid++; - Logger.Information( - "[{JobName}] Album [{Album}] is now valid after artist revalidation", - nameof(StagingAlbumRevalidationJob), - album.AlbumTitle()); - } - else - { - Logger.Debug( - "[{JobName}] Album [{Album}] artist found but still invalid: [{Reasons}]", - nameof(StagingAlbumRevalidationJob), - album.AlbumTitle(), - album.StatusReasons); + album.Artist = album.Artist with + { + AmgId = album.Artist.AmgId ?? artistFromSearch.AmgId, + ArtistDbId = album.Artist.ArtistDbId ?? artistFromSearch.Id, + DiscogsId = album.Artist.DiscogsId ?? artistFromSearch.DiscogsId, + ItunesId = album.Artist.ItunesId ?? artistFromSearch.ItunesId, + LastFmId = album.Artist.LastFmId ?? artistFromSearch.LastFmId, + MusicBrainzId = album.Artist.MusicBrainzId ?? artistFromSearch.MusicBrainzId, + Name = album.Artist.Name.Nullify() ?? artistFromSearch.Name, + NameNormalized = album.Artist.NameNormalized.Nullify() ?? + artistFromSearch.Name.ToNormalizedString() ?? + artistFromSearch.Name, + OriginalName = artistFromSearch.Name != album.Artist.Name ? album.Artist.Name : null, + SearchEngineResultUniqueId = album.Artist.SearchEngineResultUniqueId is null or < 1 + ? artistFromSearch.UniqueId + : album.Artist.SearchEngineResultUniqueId, + SortName = album.Artist.SortName.Nullify() ?? artistFromSearch.SortName, + SpotifyId = album.Artist.SpotifyId ?? artistFromSearch.SpotifyId, + WikiDataId = album.Artist.WikiDataId ?? artistFromSearch.WikiDataId + }; + shouldRevalidateAlbum = true; } } } + if (shouldRevalidateAlbum) + { + var validationResult = albumValidator.ValidateAlbum(album); + album.ValidationMessages = validationResult.Data.Messages ?? []; + album.Status = validationResult.Data.AlbumStatus; + album.StatusReasons = validationResult.Data.AlbumStatusReasons; + album.Modified = now; + + var jsonPath = fileSystemService.CombinePath(album.Directory.FullName(), Album.JsonFileName); + var serialized = serializer.Serialize(album); + await fileSystemService.WriteAllBytesAsync( + jsonPath, + System.Text.Encoding.UTF8.GetBytes(serialized ?? string.Empty), + context.CancellationToken).ConfigureAwait(false); + + albumsRevalidated++; + + if (album.Status == AlbumStatus.Ok || !NeedsArtistRevalidation(album)) + { + revalidationStateSession.RecordSuccess(album); + } + else + { + revalidationStateSession.RecordAttempt(album, now, "StillInvalidAfterValidation"); + } + await revalidationStateSession.SaveChangesAsync(context.CancellationToken).ConfigureAwait(false); + + if (album.Status == AlbumStatus.Ok) + { + albumsNowValid++; + Logger.Information( + "[{JobName}] Album [{Album}] is now valid after artist revalidation", + nameof(StagingAlbumRevalidationJob), + album.AlbumTitle()); + } + else + { + Logger.Debug( + "[{JobName}] Album [{Album}] revalidated but still invalid: [{Reasons}]", + nameof(StagingAlbumRevalidationJob), + album.AlbumTitle(), + album.StatusReasons); + } + } + else if (artistLookupAttempted) + { + albumsRevalidationNoMatch++; + revalidationStateSession.RecordAttempt(album, now, "ArtistLookupNoMatch"); + await revalidationStateSession.SaveChangesAsync(context.CancellationToken).ConfigureAwait(false); + Logger.Debug( + "[{JobName}] Album [{Album}] artist lookup found no match during revalidation", + nameof(StagingAlbumRevalidationJob), + album.AlbumTitle()); + } + OnProcessingEvent?.Invoke( this, new ProcessingEvent(ProcessingEventType.Processing, nameof(StagingAlbumRevalidationJob), albumsNeedingRevalidation.Length, - albumsRevalidated, - $"Revalidated [{albumsRevalidated}/{albumsNeedingRevalidation.Length}]")); + albumsProcessed, + ProgressMessage(albumsProcessed, albumsNeedingRevalidation.Length, albumsRevalidated, albumsRevalidationLookupsAttempted, albumsRevalidationNoMatch, albumsSkippedRevalidation, albumsDeferredRevalidation))); } catch (Exception ex) { @@ -241,7 +362,11 @@ await fileSystemService.WriteAllBytesAsync( context.Result = new ScanStepResult( AlbumsRevalidated: albumsRevalidated, - AlbumsNowValid: albumsNowValid); + AlbumsNowValid: albumsNowValid, + AlbumsRevalidationLookupsAttempted: albumsRevalidationLookupsAttempted, + AlbumsRevalidationNoMatch: albumsRevalidationNoMatch, + AlbumsSkippedRevalidation: albumsSkippedRevalidation, + AlbumsDeferredRevalidation: albumsDeferredRevalidation); OnProcessingEvent?.Invoke( this, @@ -249,18 +374,86 @@ await fileSystemService.WriteAllBytesAsync( nameof(StagingAlbumRevalidationJob), albumsNeedingRevalidation.Length, albumsRevalidated, - $"Revalidated [{albumsRevalidated}] albums, [{albumsNowValid}] now valid")); + $"Revalidated [{albumsRevalidated}] albums, [{albumsNowValid}] now valid, lookup attempts [{albumsRevalidationLookupsAttempted}], no matches [{albumsRevalidationNoMatch}], skipped [{albumsSkippedRevalidation}], deferred [{albumsDeferredRevalidation}]")); Logger.Information( - "ℹ️ [{JobName}] Completed in [{Elapsed}]ms. Revalidated [{Revalidated}] albums, [{NowValid}] now valid and ready to move", + "[{JobName}] Completed in [{Elapsed}]ms. Revalidated [{Revalidated}] albums, [{NowValid}] now valid and ready to move, lookup attempts [{LookupAttempts}], no matches [{NoMatch}], skipped [{Skipped}], deferred [{Deferred}]", nameof(StagingAlbumRevalidationJob), elapsed.TotalMilliseconds, albumsRevalidated, - albumsNowValid); + albumsNowValid, + albumsRevalidationLookupsAttempted, + albumsRevalidationNoMatch, + albumsSkippedRevalidation, + albumsDeferredRevalidation); } catch (Exception e) { Logger.Error(e, "[{JobName}] Processing Exception", nameof(StagingAlbumRevalidationJob)); } } + + private async Task OpenRevalidationStateSessionAsync( + string stagingPath, + IReadOnlyCollection albumsNeedingRevalidation, + CancellationToken cancellationToken) + { + try + { + return await revalidationStateStore.OpenAsync(stagingPath, albumsNeedingRevalidation, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Logger.Warning( + ex, + "[{JobName}] Unable to open staging revalidation state store. Continuing without persistent revalidation backoff.", + nameof(StagingAlbumRevalidationJob)); + return new PassthroughRevalidationStateSession(); + } + } + + private static bool NeedsArtistRevalidation(Album album) + { + return album.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists) || + album.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasUnknownArtist); + } + + private static string ProgressMessage( + int processed, + int total, + int revalidated, + int lookupAttempts, + int noMatches, + int skipped, + int deferred) + { + return $"Processed [{processed}/{total}], revalidated [{revalidated}], lookups [{lookupAttempts}], no matches [{noMatches}], skipped [{skipped}], deferred [{deferred}]"; + } + + private sealed class PassthroughRevalidationStateSession : IStagingAlbumRevalidationStateSession + { + public StagingAlbumRevalidationDecision GetDecision(Album album, DateTimeOffset now, bool force) + { + return new StagingAlbumRevalidationDecision(true, Reason: "Passthrough"); + } + + public void RecordAttempt(Album album, DateTimeOffset now, string outcome) + { + } + + public void RecordSuccess(Album album) + { + } + + public Task SaveChangesAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } } diff --git a/src/Melodee.Common/Jobs/StagingAutoMoveJob.cs b/src/Melodee.Common/Jobs/StagingAutoMoveJob.cs index 3119f380e..d788c7dbb 100644 --- a/src/Melodee.Common/Jobs/StagingAutoMoveJob.cs +++ b/src/Melodee.Common/Jobs/StagingAutoMoveJob.cs @@ -87,12 +87,12 @@ public override async Task Execute(IJobExecutionContext context) return; } - var albumsMoved = 0; + ProcessingEventStatistics? finalStatistics = null; void OnProcessingProgress(object? sender, ProcessingEvent e) { if (e is { Type: ProcessingEventType.Stop, Statistics: not null }) { - albumsMoved = e.Statistics.AlbumsMoved; + finalStatistics = e.Statistics; } } @@ -108,17 +108,30 @@ void OnProcessingProgress(object? sender, ProcessingEvent e) if (moveResult.IsSuccess) { - context.Result = new ScanStepResult(AlbumsMoved: albumsMoved); + var albumsMoved = finalStatistics?.AlbumsMoved ?? 0; + var albumsMerged = finalStatistics?.AlbumsMergedWithExisting ?? 0; + var albumsHandled = albumsMoved + albumsMerged; + + context.Result = new ScanStepResult( + AlbumsReadyToMove: finalStatistics?.AlbumsReadyToMove ?? 0, + AlbumsMoved: albumsMoved, + AlbumsMergedWithExisting: albumsMerged, + AlbumsSkippedByStatus: finalStatistics?.AlbumsSkippedByStatus ?? 0, + AlbumsSkippedAsDuplicateDirectory: finalStatistics?.AlbumsSkippedAsDuplicateDirectory ?? 0, + AlbumsFailedToLoad: finalStatistics?.AlbumsFailedToLoad ?? 0, + AlbumsSkippedByReason: finalStatistics?.AlbumsSkippedByReason); + Logger.Information( - "[{JobName}] Completed staging auto-move: moved [{MovedCount}] albums from [{StagingLibrary}] to [{StorageLibrary}]", + "[{JobName}] Completed staging auto-move: moved [{MovedCount}] new albums and merged [{MergedCount}] existing albums from [{StagingLibrary}] to [{StorageLibrary}]", nameof(StagingAutoMoveJob), albumsMoved, + albumsMerged, stagingLibrary.Name, targetLibrary.Name); // Chain to LibraryInsertJob if this was a scheduled run (not manual) and we moved something, // or if ChainOnComplete flag is set (for programmatic triggers like after upload) - if (ShouldChainToNextJob(context) && albumsMoved > 0) + if (ShouldChainToNextJob(context) && albumsHandled > 0) { await TriggerNextJobAsync(context, JobKeyRegistry.LibraryProcessJobJobKey).ConfigureAwait(false); } diff --git a/src/Melodee.Common/Melodee.Common.csproj b/src/Melodee.Common/Melodee.Common.csproj index fd980c509..a85c844ac 100644 --- a/src/Melodee.Common/Melodee.Common.csproj +++ b/src/Melodee.Common/Melodee.Common.csproj @@ -9,7 +9,7 @@ $(NoWarn);NU1507 - 2.1.0 + 2.1.3 build$([System.DateTime]::UtcNow.ToString("yyyyMMddHHmmss")) $(VersionPrefix).0 $(VersionPrefix).0 @@ -22,6 +22,7 @@ + @@ -31,8 +32,6 @@ - - diff --git a/src/Melodee.Common/MessageBus/EventHandlers/AlbumAddEventHandler.cs b/src/Melodee.Common/MessageBus/EventHandlers/AlbumAddEventHandler.cs index fe20978c4..93d0f129b 100644 --- a/src/Melodee.Common/MessageBus/EventHandlers/AlbumAddEventHandler.cs +++ b/src/Melodee.Common/MessageBus/EventHandlers/AlbumAddEventHandler.cs @@ -1,4 +1,3 @@ -using IdSharp.Common.Utils; using JetBrains.Annotations; using Melodee.Common.Configuration; using Melodee.Common.Constants; @@ -246,7 +245,7 @@ await contextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals continue; } - var mediaFileHash = CRC32.Calculate(mediaFile); + var mediaFileHash = Crc32.Calculate(mediaFile); var s = new Song { AlbumId = newAlbum.Id, diff --git a/src/Melodee.Common/MessageBus/EventHandlers/AlbumRescanEventHandler.cs b/src/Melodee.Common/MessageBus/EventHandlers/AlbumRescanEventHandler.cs index 4e6582ad5..80cb560af 100644 --- a/src/Melodee.Common/MessageBus/EventHandlers/AlbumRescanEventHandler.cs +++ b/src/Melodee.Common/MessageBus/EventHandlers/AlbumRescanEventHandler.cs @@ -1,4 +1,3 @@ -using IdSharp.Common.Utils; using Melodee.Common.Configuration; using Melodee.Common.Constants; using Melodee.Common.Data; @@ -145,7 +144,7 @@ await libraryService.UpdateAggregatesAsync(dbAlbum.Artist.Library.Id, cancellati // Get all songs in directory for album, add any missing, remove any on album not in the directory foreach (var mediaFile in albumDirectory.AllMediaTypeFileInfos()) { - var mediaFileHash = CRC32.Calculate(mediaFile); + var mediaFileHash = Crc32.Calculate(mediaFile); var melodeeSong = melodeeAlbum.Songs?.FirstOrDefault(x => x.File.Name == mediaFile.Name); if (melodeeSong == null) { diff --git a/src/Melodee.Common/Models/Extensions/AlbumExtensions.cs b/src/Melodee.Common/Models/Extensions/AlbumExtensions.cs index 0ce4d6a37..90449173b 100644 --- a/src/Melodee.Common/Models/Extensions/AlbumExtensions.cs +++ b/src/Melodee.Common/Models/Extensions/AlbumExtensions.cs @@ -667,8 +667,13 @@ public static bool RenumberImages(this Album album, CancellationToken cancellati $"{ImageInfo.ImageFilePrefix}{(ii.i + 1).ToStringPadLeft(MelodeeConfiguration.ImageNameNumberPadding)}-{ii.x.PictureIdentifier.ToString()}.jpg"; if (ii.x.FileInfo?.Exists(album.Directory) ?? false) { - File.Move(ii.x.FileInfo.FullName(album.Directory), - Path.Combine(album.Directory.FullName(), newFilename), true); + var currentFileName = ii.x.FileInfo.FullName(album.Directory); + var newFullFileName = Path.Combine(album.Directory.FullName(), newFilename); + if (!string.Equals(currentFileName, newFullFileName, StringComparison.OrdinalIgnoreCase)) + { + File.Move(currentFileName, newFullFileName, true); + } + renumberedImages.Add(ii.x with { FileInfo = new FileSystemFileInfo diff --git a/src/Melodee.Common/Models/Extensions/ArtistExtensions.cs b/src/Melodee.Common/Models/Extensions/ArtistExtensions.cs index 404c0809d..0bad904a3 100644 --- a/src/Melodee.Common/Models/Extensions/ArtistExtensions.cs +++ b/src/Melodee.Common/Models/Extensions/ArtistExtensions.cs @@ -73,9 +73,11 @@ public static AlbumArtistType ArtistType(this Artist artist) public static bool IsValid(this Artist artist) { - // If the artist is known already to Melodee (via Dbid) or is a known MusicBrainz or Spotify artist then is ok. - // Musicbrainz and Spotify reliably return images for artists, other providers (looking at you LastFm) are spotty. - return (artist.ArtistDbId != null || artist.MusicBrainzId != null || artist.SpotifyId != null) && + // Melodee trusts MusicBrainz, Spotify, and iTunes artist identifiers as strong external identities. + return (artist.ArtistDbId != null || + artist.MusicBrainzId != null || + artist.SpotifyId.Nullify() != null || + artist.ItunesId.Nullify() != null) && artist.Name.Nullify() != null; } @@ -125,8 +127,9 @@ public static string ToDirectoryName(this Artist artist, int processingMaximumAr if (artistDirectoryId == null) { artistDirectoryId = SafeParser.Hash(artist.MusicBrainzId?.ToString() ?? - artist.SpotifyId ?? throw new Exception( - "Neither ArtistDbId, SearchEngineResultUniqueId, MusicBrainzId or SpotifyId is set.")) + artist.SpotifyId ?? + artist.ItunesId ?? throw new Exception( + "Neither ArtistDbId, SearchEngineResultUniqueId, MusicBrainzId, SpotifyId or ItunesId is set.")) .ToString(); } diff --git a/src/Melodee.Common/Models/Extensions/FileSystemFileInfoExtensions.cs b/src/Melodee.Common/Models/Extensions/FileSystemFileInfoExtensions.cs index 567ff1f28..7c85d73e5 100644 --- a/src/Melodee.Common/Models/Extensions/FileSystemFileInfoExtensions.cs +++ b/src/Melodee.Common/Models/Extensions/FileSystemFileInfoExtensions.cs @@ -1,3 +1,5 @@ +using Melodee.Common.Extensions; + namespace Melodee.Common.Models.Extensions; /// @@ -18,8 +20,15 @@ public static string FullName(this FileSystemFileInfo fileSystemFileInfo, FileSy public static bool Exists(this FileSystemFileInfo fileSystemFileInfo, FileSystemDirectoryInfo directoryInfo) { - return File.Exists(Path.Combine(directoryInfo.Path, - fileSystemFileInfo.OriginalName ?? fileSystemFileInfo.Name)); + if (File.Exists(fileSystemFileInfo.FullName(directoryInfo))) + { + return true; + } + + var originalName = fileSystemFileInfo.OriginalName.Nullify(); + return originalName is not null && + !string.Equals(originalName, fileSystemFileInfo.Name, StringComparison.OrdinalIgnoreCase) && + File.Exists(Path.Combine(directoryInfo.FullName(), originalName)); } public static string Extension(this FileSystemFileInfo fileSystemFileInfo, FileSystemDirectoryInfo directoryInfo) diff --git a/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/Artist.cs b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/Artist.cs index be5dcc245..1f83cdbd8 100644 --- a/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/Artist.cs +++ b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/Artist.cs @@ -49,6 +49,8 @@ public sealed class Artist public ICollection Albums { get; set; } = []; + public ICollection Aliases { get; set; } = []; + [NotMapped] public int Rank { get; set; } [NotMapped] public int AlbumCount { get; set; } diff --git a/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistAlias.cs b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistAlias.cs new file mode 100644 index 000000000..8f175c2c4 --- /dev/null +++ b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistAlias.cs @@ -0,0 +1,17 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.EntityFrameworkCore; + +namespace Melodee.Common.Models.SearchEngines.ArtistSearchEngineServiceData; + +[Index(nameof(NameNormalized))] +[Index(nameof(ArtistId), nameof(NameNormalized), IsUnique = true)] +public sealed class ArtistAlias +{ + [Key] public int Id { get; set; } + + public int ArtistId { get; set; } + + public Artist Artist { get; set; } = null!; + + [Required][MaxLength(2000)] public required string NameNormalized { get; set; } +} diff --git a/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistSearchEngineServiceDbContext.cs b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistSearchEngineServiceDbContext.cs index 19e79388f..0c3d952b4 100644 --- a/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistSearchEngineServiceDbContext.cs +++ b/src/Melodee.Common/Models/SearchEngines/ArtistSearchEngineServiceData/ArtistSearchEngineServiceDbContext.cs @@ -8,6 +8,8 @@ public class ArtistSearchEngineServiceDbContext(DbContextOptions Artists { get; set; } + public DbSet ArtistAliases { get; set; } + public DbSet Albums { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) @@ -25,5 +27,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasConversion(new DateTimeOffsetToBinaryConverter()); } } + + modelBuilder.Entity(entity => + { + entity.ToTable("ArtistAliases"); + entity.HasOne(x => x.Artist) + .WithMany(x => x.Aliases) + .HasForeignKey(x => x.ArtistId) + .OnDelete(DeleteBehavior.Cascade); + }); } } diff --git a/src/Melodee.Common/Plugins/Conversion/Media/MediaConvertor.cs b/src/Melodee.Common/Plugins/Conversion/Media/MediaConvertor.cs index 896405e4d..bc0443974 100644 --- a/src/Melodee.Common/Plugins/Conversion/Media/MediaConvertor.cs +++ b/src/Melodee.Common/Plugins/Conversion/Media/MediaConvertor.cs @@ -95,6 +95,16 @@ public async Task> ProcessFileAsync(FileSyst } catch (Exception ex) { + var fallbackFileInfo = new FileInfo(newFileName); + if (await IsUsableConvertedMp3Async(fallbackFileInfo, cancellationToken).ConfigureAwait(false)) + { + fileInfo = FinalizeConvertedFile(songFileInfo, fallbackFileInfo); + return new OperationResult + { + Data = fileInfo.ToFileSystemInfo() + }; + } + throw new Exception($"Unable to convert [{songFileInfo.FullName}] to MP3", ex); } @@ -109,22 +119,9 @@ public async Task> ProcessFileAsync(FileSyst throw new Exception($"Unable to convert [{songFileInfo.FullName}] to MP3"); } - var newAtl = new Track(newFileName); - if (string.Equals(newAtl.AudioFormat.ShortName, "mpeg", StringComparison.OrdinalIgnoreCase)) + if (await IsUsableConvertedMp3Async(newFileInfo, cancellationToken).ConfigureAwait(false)) { - var convertedRenamedExtension = - SafeParser.ToString(Configuration[SettingRegistry.ProcessingConvertedExtension]); - fileInfo = new FileInfo(newFileName); - if (SafeParser.ToBoolean(Configuration[SettingRegistry.ProcessingDoDeleteOriginal])) - { - songFileInfo.Delete(); - } - else if (convertedRenamedExtension.Nullify() != null) - { - var movedFileName = Path.Combine(songFileInfo.DirectoryName!, - $"{songFileInfo.Name}.{convertedRenamedExtension}"); - songFileInfo.MoveTo(movedFileName); - } + fileInfo = FinalizeConvertedFile(songFileInfo, newFileInfo); } else { @@ -140,6 +137,44 @@ public async Task> ProcessFileAsync(FileSyst }; } + public static async Task IsUsableConvertedMp3Async(FileInfo fileInfo, CancellationToken cancellationToken = default) + { + if (!fileInfo.Exists || fileInfo.Length == 0 || + !fileInfo.Extension.Equals(".mp3", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + try + { + return await FileFormatDetector.DetectFormatAsync(fileInfo.FullName, cancellationToken).ConfigureAwait(false) == AudioFormat.MP3; + } + catch + { + return false; + } + } + + private FileInfo FinalizeConvertedFile(FileInfo sourceFileInfo, FileInfo convertedFileInfo) + { + var convertedRenamedExtension = + SafeParser.ToString(Configuration[SettingRegistry.ProcessingConvertedExtension]); + + if (SafeParser.ToBoolean(Configuration[SettingRegistry.ProcessingDoDeleteOriginal]) && + sourceFileInfo.Exists) + { + sourceFileInfo.Delete(); + } + else if (convertedRenamedExtension.Nullify() != null && sourceFileInfo.Exists) + { + var movedFileName = Path.Combine(sourceFileInfo.DirectoryName!, + $"{sourceFileInfo.Name}.{convertedRenamedExtension}"); + sourceFileInfo.MoveTo(movedFileName); + } + + return convertedFileInfo; + } + private static bool ShouldMediaSongBeConverted(Track song) { if (song.AudioFormat == null || (song.AudioFormat?.MimeList?.Contains("image") ?? false)) diff --git a/src/Melodee.Common/Plugins/MetaData/Directory/Blackbeard/Blackbeard.cs b/src/Melodee.Common/Plugins/MetaData/Directory/Blackbeard/Blackbeard.cs new file mode 100644 index 000000000..e887ff156 --- /dev/null +++ b/src/Melodee.Common/Plugins/MetaData/Directory/Blackbeard/Blackbeard.cs @@ -0,0 +1,580 @@ +using System.Text.Json.Serialization; +using Melodee.Common.Configuration; +using Melodee.Common.Constants; +using Melodee.Common.Enums; +using Melodee.Common.Extensions; +using Melodee.Common.Models; +using Melodee.Common.Models.Extensions; +using Melodee.Common.Plugins.Validation; +using Melodee.Common.Serialization; +using Melodee.Common.Services.Scanning; +using Melodee.Common.Utility; +using Serilog; +using Serilog.Events; +using SerilogTimings; +using MelodeeSong = Melodee.Common.Models.Song; + +namespace Melodee.Common.Plugins.MetaData.Directory.Blackbeard; + +/// +/// Processes Blackbeard provenance files and builds album metadata from the normalized release manifest. +/// +public sealed class Blackbeard( + ISerializer serializer, + IAlbumValidator albumValidator, + IMelodeeConfiguration configuration) + : AlbumMetaDataBase(configuration), IDirectoryPlugin +{ + public const string HandlesFileName = ".blackbeard.provenance.json"; + public const int SupportedSchemaVersion = 1; + + public override string Id => "8397443D-B13E-477E-B7C3-82F3459DB878"; + + public override string DisplayName => nameof(Blackbeard); + + public override bool IsEnabled { get; set; } = true; + + public override int SortOrder { get; } = 1; + + public async Task> ProcessDirectoryAsync(FileSystemDirectoryInfo fileSystemDirectoryInfo, + CancellationToken cancellationToken = default) + { + StopProcessing = false; + + var provenanceFiles = ProvenanceFiles(fileSystemDirectoryInfo); + if (provenanceFiles.Length == 0) + { + return new OperationResult("Skipping Blackbeard provenance. No provenance files found.") + { + Type = OperationResponseType.NotFound, + Data = -1 + }; + } + + var messages = new List(); + var processedFiles = 0; + foreach (var provenanceFile in provenanceFiles) + { + using (Operation.At(LogEventLevel.Debug) + .Time("[{Plugin}] Processing [{FileName}]", DisplayName, provenanceFile.Name)) + { + try + { + var album = await AlbumForProvenanceFileAsync( + provenanceFile, + fileSystemDirectoryInfo, + cancellationToken, + messages) + .ConfigureAwait(false); + if (album is null) + { + continue; + } + + var stagingAlbumDataName = Path.Combine( + fileSystemDirectoryInfo.FullName(), + album.ToMelodeeJsonName(MelodeeConfiguration)); + if (File.Exists(stagingAlbumDataName)) + { + var existingAlbum = await Album + .DeserializeAndInitializeAlbumAsync(serializer, stagingAlbumDataName, cancellationToken) + .ConfigureAwait(false); + if (existingAlbum is not null) + { + album = album.Merge(existingAlbum); + } + } + + var validationResult = albumValidator.ValidateAlbum(album); + album.ValidationMessages = validationResult.Data.Messages ?? []; + album.Status = validationResult.Data.AlbumStatus; + album.StatusReasons = validationResult.Data.AlbumStatusReasons; + + await File.WriteAllTextAsync( + stagingAlbumDataName, + serializer.Serialize(album), + cancellationToken) + .ConfigureAwait(false); + + if (SafeParser.ToBoolean(Configuration[SettingRegistry.ProcessingDoDeleteOriginal])) + { + provenanceFile.Delete(); + Log.Information("[{Plugin}] Deleted Blackbeard provenance file [{FileName}]", + DisplayName, + provenanceFile.Name); + } + + Log.Debug( + "[{Plugin}] created [{StagingAlbumDataName}] Status [{Status}] validation reason [{ValidationReason}]", + DisplayName, + album.ToMelodeeJsonName(MelodeeConfiguration), + album.Status.ToString(), + album.StatusReasons.ToString()); + + processedFiles++; + } + catch (Exception e) + { + Log.Error(e, "[{Plugin}] processing provenance file [{FileName}]", DisplayName, provenanceFile.Name); + StopProcessing = true; + return new OperationResult + { + Type = OperationResponseType.Error, + Errors = [e], + Data = processedFiles + }; + } + } + } + + return new OperationResult(messages) + { + Data = processedFiles + }; + } + + public override bool DoesHandleFile(FileSystemDirectoryInfo directoryInfo, FileSystemFileInfo fileSystemInfo) + { + return fileSystemInfo.Name.DoStringsMatch(HandlesFileName); + } + + public async Task AlbumForProvenanceFileAsync( + FileInfo provenanceFile, + FileSystemDirectoryInfo directoryInfo, + CancellationToken cancellationToken = default, + ICollection? messages = null) + { + var document = serializer.Deserialize( + await File.ReadAllBytesAsync(provenanceFile.FullName, cancellationToken).ConfigureAwait(false)); + if (document is null) + { + Log.Warning("[{Plugin}] unable to deserialize Blackbeard provenance [{FileName}]", + DisplayName, + provenanceFile.Name); + return null; + } + + if (document.SchemaVersion != SupportedSchemaVersion) + { + var message = + $"[{DisplayName}] unsupported Blackbeard schema version [{document.SchemaVersion}] in [{provenanceFile.Name}]. Supported version is [{SupportedSchemaVersion}]."; + Log.Warning("[{Plugin}] unsupported Blackbeard schema version [{SchemaVersion}] in [{FileName}]. Supported version is [{SupportedSchemaVersion}]", + DisplayName, + document.SchemaVersion, + provenanceFile.Name, + SupportedSchemaVersion); + messages?.Add(message); + return null; + } + + var songs = SongsForDocument(document, directoryInfo).ToArray(); + if (songs.Length == 0) + { + Log.Warning("[{Plugin}] no matching song files found for provenance [{FileName}]", + DisplayName, + provenanceFile.Name); + return null; + } + + var albumArtist = document.Release?.AlbumArtist.Nullify() ?? + songs.FirstOrDefault(x => x.AlbumArtist().Nullify() is not null)?.AlbumArtist() ?? + songs.FirstOrDefault(x => x.SongArtist().Nullify() is not null)?.SongArtist(); + var albumTitle = document.Release?.AlbumTitle.Nullify() ?? + songs.FirstOrDefault(x => x.AlbumTitle().Nullify() is not null)?.AlbumTitle(); + var releaseYear = document.Release?.ReleaseYear ?? + songs.FirstOrDefault(x => x.AlbumYear() is not null)?.AlbumYear(); + var tracks = document.Tracks ?? []; + var songTotal = tracks.Length == 0 ? songs.Length : tracks.Max(x => x.TrackTotal ?? x.TrackNumber) ?? songs.Length; + var discTotal = tracks.Length == 0 ? 1 : tracks.Max(x => x.DiscTotal ?? x.DiscNumber) ?? 1; + + var albumTags = AlbumTags(albumTitle, albumArtist, releaseYear, songTotal, discTotal, songs); + var dirInfo = new DirectoryInfo(directoryInfo.FullName()); + var parentDirectory = dirInfo.Parent?.ToDirectorySystemInfo(); + + return new Album + { + AlbumType = albumTitle.TryToDetectAlbumType(), + Artist = Artist.NewArtistFromName(albumArtist ?? throw new Exception("Invalid Blackbeard album artist.")), + Directory = directoryInfo, + Files = + [ + new AlbumFile + { + AlbumFileType = AlbumFileType.MetaData, + ProcessedByPlugin = DisplayName, + FileSystemFileInfo = provenanceFile.ToFileSystemInfo() + } + ], + OriginalDirectory = new FileSystemDirectoryInfo + { + ParentId = parentDirectory?.UniqueId ?? 0, + Path = directoryInfo.Path, + Name = directoryInfo.Name, + TotalItemsFound = songs.Length, + MusicFilesFound = songs.Length, + MusicMetaDataFilesFound = 1 + }, + Images = [], + Tags = albumTags, + Songs = songs.OrderBy(x => x.SortOrder).ToArray(), + ViaPlugins = [DisplayName] + }; + } + + private static FileInfo[] ProvenanceFiles(FileSystemDirectoryInfo fileSystemDirectoryInfo) + { + var dirInfo = new DirectoryInfo(fileSystemDirectoryInfo.FullName()); + return !dirInfo.Exists + ? [] + : dirInfo.GetFiles(HandlesFileName, SearchOption.TopDirectoryOnly).OrderBy(x => x.Name).ToArray(); + } + + private static IEnumerable SongsForDocument( + BlackbeardProvenanceDocument document, + FileSystemDirectoryInfo directoryInfo) + { + foreach (var track in document.Tracks ?? []) + { + if (track.FinalValidation?.Passed == false) + { + continue; + } + + var trackFile = FileForTrack(directoryInfo, track); + if (trackFile is null) + { + continue; + } + + var tagsAfter = track.Normalization?.TagsAfter; + var title = tagsAfter?.Title.Nullify() ?? track.Title.Nullify(); + var artist = tagsAfter?.Artist.Nullify() ?? track.Artist.Nullify(); + var albumArtist = tagsAfter?.AlbumArtist.Nullify() ?? track.AlbumArtist.Nullify(); + var albumTitle = tagsAfter?.AlbumTitle.Nullify() ?? track.AlbumTitle.Nullify(); + var releaseYear = tagsAfter?.ReleaseYear ?? track.ReleaseYear; + var trackNumber = tagsAfter?.TrackNumber ?? track.TrackNumber; + var trackTotal = tagsAfter?.TrackTotal ?? track.TrackTotal; + var discNumber = tagsAfter?.DiscNumber ?? track.DiscNumber ?? 1; + var discTotal = tagsAfter?.DiscTotal ?? track.DiscTotal ?? 1; + + yield return new MelodeeSong + { + CrcHash = Crc32.Calculate(trackFile), + File = trackFile.ToFileSystemInfo(), + Tags = SongTags( + title, + artist, + albumArtist, + albumTitle, + releaseYear, + trackNumber, + trackTotal, + discNumber, + discTotal, + tagsAfter?.Genres), + MediaAudios = MediaAudios(tagsAfter), + SortOrder = (trackNumber ?? 0) + (discNumber * MediaEditService.SortOrderMediaMultiplier) - + MediaEditService.SortOrderMediaMultiplier + }; + } + } + + private static FileInfo? FileForTrack(FileSystemDirectoryInfo directoryInfo, BlackbeardTrack track) + { + foreach (var relativePath in new[] { track.Output?.Path, track.StagedTrackPath }) + { + var fileInfo = FileUnderDirectory(directoryInfo, relativePath); + if (fileInfo?.Exists == true) + { + return fileInfo; + } + } + + return null; + } + + private static FileInfo? FileUnderDirectory(FileSystemDirectoryInfo directoryInfo, string? relativePath) + { + var path = relativePath.Nullify(); + if (path is null || Path.IsPathFullyQualified(path)) + { + return null; + } + + var root = Path.GetFullPath(directoryInfo.FullName()); + var rootWithSeparator = root.EndsWith(Path.DirectorySeparatorChar) + ? root + : $"{root}{Path.DirectorySeparatorChar}"; + var candidate = Path.GetFullPath(Path.Combine(root, path)); + if (!candidate.StartsWith(rootWithSeparator, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + return new FileInfo(candidate); + } + + private static MetaTag[] AlbumTags( + string? albumTitle, + string? albumArtist, + int? releaseYear, + int songTotal, + int discTotal, + MelodeeSong[] songs) + { + var tags = new List>(); + AddTag(tags, MetaTagIdentifier.Album, albumTitle, 1); + AddTag(tags, MetaTagIdentifier.AlbumArtist, albumArtist, 2); + AddTag(tags, MetaTagIdentifier.DiscTotal, discTotal, 4); + + var sortOrder = 5; + foreach (var genre in songs.SelectMany(x => x.Tags ?? []) + .Where(x => x.Identifier == MetaTagIdentifier.Genre) + .Select(x => x.Value?.ToString()) + .Where(x => x.Nullify() is not null) + .Distinct(StringComparer.OrdinalIgnoreCase)) + { + AddTag(tags, MetaTagIdentifier.Genre, genre, sortOrder++); + } + + AddTag(tags, MetaTagIdentifier.RecordingYear, releaseYear, 100); + AddTag(tags, MetaTagIdentifier.SongTotal, songTotal, 101); + + return tags.ToArray(); + } + + private static MetaTag[] SongTags( + string? title, + string? artist, + string? albumArtist, + string? albumTitle, + int? releaseYear, + int? trackNumber, + int? trackTotal, + int? discNumber, + int? discTotal, + string[]? genres) + { + var tags = new List>(); + AddTag(tags, MetaTagIdentifier.Title, title, 1); + AddTag(tags, MetaTagIdentifier.Artist, artist, 2); + AddTag(tags, MetaTagIdentifier.AlbumArtist, albumArtist, 3); + AddTag(tags, MetaTagIdentifier.Album, albumTitle, 4); + AddTag(tags, MetaTagIdentifier.DiscTotal, discTotal, 5); + AddTag(tags, MetaTagIdentifier.DiscNumber, discNumber, 6); + AddTag(tags, MetaTagIdentifier.RecordingYear, releaseYear, 100); + AddTag(tags, MetaTagIdentifier.TrackNumber, trackNumber, 101); + AddTag(tags, MetaTagIdentifier.SongTotal, trackTotal, 102); + + var sortOrder = 200; + foreach (var genre in genres ?? []) + { + AddTag(tags, MetaTagIdentifier.Genre, genre, sortOrder++); + } + + return tags.ToArray(); + } + + private static MediaAudio[] MediaAudios(BlackbeardTrackTags? tags) + { + if (tags is null) + { + return []; + } + + var mediaAudios = new List>(); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.DurationMs, tags.DurationMs, 1); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.BitRate, tags.Bitrate, 2); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.SampleRate, tags.SampleRate, 3); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.Channels, tags.Channels, 4); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.BitDepth, tags.BitDepth, 5); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.CodecLongName, tags.Codec, 6); + AddMediaAudio(mediaAudios, MediaAudioIdentifier.FormatName, tags.Container, 7); + return mediaAudios.ToArray(); + } + + private static void AddTag(List> tags, MetaTagIdentifier identifier, object? value, int sortOrder) + { + if (value is string s && s.Nullify() is null) + { + return; + } + + if (value is null) + { + return; + } + + tags.Add(new MetaTag + { + Identifier = identifier, + Value = value, + SortOrder = sortOrder + }); + } + + private static void AddMediaAudio( + List> mediaAudios, + MediaAudioIdentifier identifier, + object? value, + int sortOrder) + { + if (value is string s && s.Nullify() is null) + { + return; + } + + if (value is null) + { + return; + } + + mediaAudios.Add(new MediaAudio + { + Identifier = identifier, + Value = value, + SortOrder = sortOrder + }); + } +} + +public sealed record BlackbeardProvenanceDocument +{ + [JsonPropertyName("schema_version")] + public int SchemaVersion { get; init; } + + [JsonPropertyName("blackbeard_version")] + public string? BlackbeardVersion { get; init; } + + [JsonPropertyName("release")] + public BlackbeardRelease? Release { get; init; } + + [JsonPropertyName("tracks")] + public BlackbeardTrack[]? Tracks { get; init; } +} + +public sealed record BlackbeardRelease +{ + [JsonPropertyName("album_artist")] + public string? AlbumArtist { get; init; } + + [JsonPropertyName("album_title")] + public string? AlbumTitle { get; init; } + + [JsonPropertyName("release_year")] + public int? ReleaseYear { get; init; } +} + +public sealed record BlackbeardTrack +{ + [JsonPropertyName("staged_track_path")] + public string? StagedTrackPath { get; init; } + + [JsonPropertyName("track_number")] + public int? TrackNumber { get; init; } + + [JsonPropertyName("track_total")] + public int? TrackTotal { get; init; } + + [JsonPropertyName("disc_number")] + public int? DiscNumber { get; init; } + + [JsonPropertyName("disc_total")] + public int? DiscTotal { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("artist")] + public string? Artist { get; init; } + + [JsonPropertyName("album_artist")] + public string? AlbumArtist { get; init; } + + [JsonPropertyName("album_title")] + public string? AlbumTitle { get; init; } + + [JsonPropertyName("release_year")] + public int? ReleaseYear { get; init; } + + [JsonPropertyName("output")] + public BlackbeardTrackOutput? Output { get; init; } + + [JsonPropertyName("normalization")] + public BlackbeardTrackNormalization? Normalization { get; init; } + + [JsonPropertyName("final_validation")] + public BlackbeardFinalValidation? FinalValidation { get; init; } +} + +public sealed record BlackbeardTrackOutput +{ + [JsonPropertyName("path")] + public string? Path { get; init; } +} + +public sealed record BlackbeardTrackNormalization +{ + [JsonPropertyName("tags_after")] + public BlackbeardTrackTags? TagsAfter { get; init; } +} + +public sealed record BlackbeardTrackTags +{ + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("artist")] + public string? Artist { get; init; } + + [JsonPropertyName("album_artist")] + public string? AlbumArtist { get; init; } + + [JsonPropertyName("album_title")] + public string? AlbumTitle { get; init; } + + [JsonPropertyName("release_year")] + public int? ReleaseYear { get; init; } + + [JsonPropertyName("track_number")] + public int? TrackNumber { get; init; } + + [JsonPropertyName("track_total")] + public int? TrackTotal { get; init; } + + [JsonPropertyName("disc_number")] + public int? DiscNumber { get; init; } + + [JsonPropertyName("disc_total")] + public int? DiscTotal { get; init; } + + [JsonPropertyName("genres")] + public string[]? Genres { get; init; } + + [JsonPropertyName("codec")] + public string? Codec { get; init; } + + [JsonPropertyName("container")] + public string? Container { get; init; } + + [JsonPropertyName("bitrate")] + public int? Bitrate { get; init; } + + [JsonPropertyName("sample_rate")] + public int? SampleRate { get; init; } + + [JsonPropertyName("channels")] + public int? Channels { get; init; } + + [JsonPropertyName("duration_ms")] + public double? DurationMs { get; init; } + + [JsonPropertyName("bit_depth")] + public int? BitDepth { get; init; } +} + +public sealed record BlackbeardFinalValidation +{ + [JsonPropertyName("passed")] + public bool? Passed { get; init; } +} diff --git a/src/Melodee.Common/Plugins/MetaData/Directory/Nfo/Nfo.cs b/src/Melodee.Common/Plugins/MetaData/Directory/Nfo/Nfo.cs index accca39c5..4a749ab8a 100644 --- a/src/Melodee.Common/Plugins/MetaData/Directory/Nfo/Nfo.cs +++ b/src/Melodee.Common/Plugins/MetaData/Directory/Nfo/Nfo.cs @@ -350,16 +350,20 @@ public static bool IsLineForSong(string line) if (IsLineForSong(line)) { - var l = line.OnlyAlphaNumeric(); - var songNumber = SafeParser.ToNumber(l?.Substring(0, 2) ?? string.Empty); - var songDuration = l?.Substring(l.Length - 7).Trim() ?? string.Empty; - var songTitle = ReplaceMultiplePeriodsRegex() - .Replace(l?.Substring(3, l.Length - songDuration.Length - 4) ?? string.Empty, string.Empty) - .Trim(); + if (!TryParseSongLine(line, out var songNumber, out var songTitle, out var songDuration)) + { + continue; + } + + var songTitleNormalized = songTitle.ToNormalizedString().Nullify(); + if (songTitleNormalized is null) + { + missingSongFiles.Add(songTitle); + continue; + } var fileForSong = mediaFilesForDirectory?.FirstOrDefault(x => - x.Name.ToNormalizedString()!.Contains(songTitle.ToNormalizedString()!, - StringComparison.OrdinalIgnoreCase)); + (x.Name.ToNormalizedString()?.Contains(songTitleNormalized, StringComparison.OrdinalIgnoreCase) ?? false)); if (fileForSong == null) { missingSongFiles.Add(songTitle); @@ -443,6 +447,13 @@ public static bool IsLineForSong(string line) ?.ToString(); var albumName = albumTags.FirstOrDefault(x => x.Identifier is MetaTagIdentifier.Album)?.Value?.ToString(); + var artistNameValue = artistName.Nullify(); + if (artistNameValue is null) + { + Log.Warning("[{Plugin}] NFO parse missing required artist in [{FileName}]", DisplayName, fileInfo.FullName); + return null; + } + if (missingSongFiles.Count > 0) { Log.Warning("[{Plugin}] NFO parse missing [{MissingCount}] tracks in [{Dir}] (examples: {Samples})", @@ -456,8 +467,8 @@ public static bool IsLineForSong(string line) { AlbumType = albumName.TryToDetectAlbumType(), Artist = new Artist( - artistName ?? throw new Exception($"Invalid artist on {nameof(Nfo)}"), - artistName.ToNormalizedString() ?? artistName, + artistNameValue, + artistNameValue.ToNormalizedString() ?? artistNameValue, null), Directory = parentDirectoryInfo ?? fileInfo.Directory?.ToDirectorySystemInfo() ?? new FileSystemDirectoryInfo @@ -501,8 +512,39 @@ public static bool IsLineForSong(string line) return null; } + private static bool TryParseSongLine( + string line, + out int songNumber, + out string songTitle, + out string songDuration) + { + songNumber = 0; + songTitle = string.Empty; + songDuration = string.Empty; + + var match = TrackLineRegex().Match(line); + if (!match.Success) + { + return false; + } + + songNumber = SafeParser.ToNumber(match.Groups["track"].Value); + songDuration = match.Groups["duration"].Value.Trim(); + songTitle = ReplaceMultiplePeriodsRegex() + .Replace(match.Groups["title"].Value.OnlyAlphaNumeric() ?? string.Empty, string.Empty) + .Trim(); + + if (songNumber < 1 || songTitle.Nullify() is null || songDuration.Nullify() is null) + { + Log.Warning("[{Plugin}] NFO parse ignored malformed track line [{Line}]", nameof(Nfo), line); + return false; + } + + return true; + } + //[GeneratedRegex(@"[0-9]+[a-z]+[0-9]{3,4}")] - [GeneratedRegex(@"\d{1,3}\s*[>\.\-)]?\s*[^\r\n]*\d{1,2}:\d{2}", RegexOptions.IgnoreCase)] + [GeneratedRegex(@"(?\d{1,3})\s*[>\.\-)]?\s*(?[^\r\n]*?)\s*(?<duration>\d{1,2}:\d{2})", RegexOptions.IgnoreCase)] private static partial Regex TrackLineRegex(); [GeneratedRegex(@"\.{2,}")] diff --git a/src/Melodee.Common/Plugins/MetaData/Song/IdSharpMetaTag.cs b/src/Melodee.Common/Plugins/MetaData/Song/IdSharpMetaTag.cs deleted file mode 100644 index 82a1dd581..000000000 --- a/src/Melodee.Common/Plugins/MetaData/Song/IdSharpMetaTag.cs +++ /dev/null @@ -1,308 +0,0 @@ -using System.Diagnostics; -using ATL; -using IdSharp.Tagging.ID3v1; -using IdSharp.Tagging.ID3v2; -using Melodee.Common.Configuration; -using Melodee.Common.Enums; -using Melodee.Common.Extensions; -using Melodee.Common.Models; -using Melodee.Common.Models.Extensions; -using Melodee.Common.Plugins.Processor; -using Melodee.Common.Services.Scanning; -using Melodee.Common.Utility; -using Serilog; -using Serilog.Events; -using SerilogTimings; - -namespace Melodee.Common.Plugins.MetaData.Song; - -public sealed class IdSharpMetaTag( - IMetaTagsProcessorPlugin metaTagsProcessorPlugin, - IMelodeeConfiguration configuration) : MetaDataBase(configuration), ISongPlugin -{ - private readonly IMetaTagsProcessorPlugin _metaTagsProcessorPlugin = metaTagsProcessorPlugin; - - public override string Id => "0AE16462-6924-496B-AC5E-C9CD70EA078D"; - - public override string DisplayName => nameof(IdSharpMetaTag); - - public override bool IsEnabled { get; set; } = true; - - public override int SortOrder { get; } = 1; - - public override bool DoesHandleFile(FileSystemDirectoryInfo directoryInfo, FileSystemFileInfo fileSystemInfo) - { - if (!IsEnabled || !fileSystemInfo.Exists(directoryInfo)) - { - return false; - } - - return FileHelper.IsFileMediaType(fileSystemInfo.Extension(directoryInfo)); - } - - public async Task<OperationResult<Models.Song>> ProcessFileAsync(FileSystemDirectoryInfo directoryInfo, - FileSystemFileInfo fileSystemInfo, CancellationToken cancellationToken = default) - { - using (Operation.At(LogEventLevel.Debug) - .Time("[{PluginName}] Processing [{fileSystemInfo}]", DisplayName, fileSystemInfo.Name)) - { - var tags = new List<MetaTag<object?>>(); - var mediaAudios = new List<MediaAudio<object?>>(); - var images = new List<ImageInfo>(); - - try - { - if (fileSystemInfo.Exists(directoryInfo)) - { - var fullname = fileSystemInfo.FullName(directoryInfo); - if (!await OptimizedFileOperations.WaitForFileStabilityAsync(fullname, cancellationToken: cancellationToken) - .ConfigureAwait(false)) - { - Log.Warning("[{Plugin}] File [{File}] not stable for read, skipping.", DisplayName, fullname); - return new OperationResult<Models.Song>(["File not stable for read"]) - { - Data = new Models.Song - { - CrcHash = string.Empty, - File = fileSystemInfo - } - }; - } - - if (ID3v1Tag.DoesTagExist(fullname)) - { - var id3V1 = new ID3v1Tag(fullname); - if (id3V1.Album.Nullify() != null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Album, - Value = id3V1.Album - }); - } - - if (id3V1.Artist.Nullify() != null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Artist, - Value = id3V1.Artist - }); - } - - if (id3V1.Title.Nullify() != null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Title, - Value = id3V1.Title.ToTitleCase(false) - }); - } - - if (id3V1.TrackNumber > 0) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.TrackNumber, - Value = SafeParser.ToNumber<short?>(id3V1.TrackNumber) - }); - } - - var date = SafeParser.ToDateTime(id3V1.Year); - if (date != null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.OrigAlbumYear, - Value = date?.Year - }); - } - } - - if (ID3v2Tag.DoesTagExist(fullname)) - { - IID3v2Tag id3V2 = new ID3v2Tag(fullname); - if (tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.Album) == null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Album, - Value = id3V2.Album - }); - } - - if (tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.Artist) == null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Artist, - Value = id3V2.AlbumArtist ?? id3V2.Artist - }); - } - - if (tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.Title) == null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.Title, - Value = id3V2.Title.ToTitleCase(false) - }); - } - - if (tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.TrackNumber) == null) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.TrackNumber, - Value = SafeParser.ToNumber<short?>(id3V2.TrackNumber) - }); - } - - if (tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.OrigAlbumYear) == null) - { - var date = SafeParser.ToDateTime(id3V2.Year); - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.OrigAlbumYear, - Value = date?.Year - }); - } - - if (mediaAudios.FirstOrDefault(x => x.Identifier == MediaAudioIdentifier.DurationMs) == null) - { - var duration = SafeParser.ToNumber<double?>(id3V2.LengthMilliseconds); - if ((duration ?? 0) < 1) - { - try - { - var atlTag = new Track(fullname); - duration = atlTag.DurationMs; - } - catch - { - // Dont do anything at this point with exception. - } - } - - mediaAudios.Add(new MediaAudio<object?> - { - Identifier = MediaAudioIdentifier.DurationMs, - Value = duration - }); - } - } - } - } - catch (Exception e) - { - Log.Error(e, "fileSystemInfo [{fileSystemInfo}]", fileSystemInfo); - } - - // Ensure that OrigAlbumYear exists and if not add with invalid date (will get set later by MetaTagProcessor.) - if (tags.All(x => x.Identifier != MetaTagIdentifier.OrigAlbumYear)) - { - tags.Add(new MetaTag<object?> - { - Identifier = MetaTagIdentifier.OrigAlbumYear, - Value = 0 - }); - } - - var metaTagsProcessorResult = - await _metaTagsProcessorPlugin.ProcessMetaTagAsync(directoryInfo, fileSystemInfo, tags, - cancellationToken); - if (!metaTagsProcessorResult.IsSuccess) - { - return new OperationResult<Models.Song>(metaTagsProcessorResult.Messages) - { - Errors = metaTagsProcessorResult.Errors, - Data = new Models.Song - { - CrcHash = string.Empty, - File = fileSystemInfo - } - }; - } - - var song = new Models.Song - { - CrcHash = Crc32.Calculate(new FileInfo(fileSystemInfo.FullName(directoryInfo))), - File = fileSystemInfo, - Images = images, - Tags = metaTagsProcessorResult.Data, - MediaAudios = mediaAudios, - SortOrder = tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.TrackNumber)?.Value as int? ?? 0 - }; - if (!song.IsValid(Configuration)) - { - Trace.WriteLine("Song is invalid"); - } - - return new OperationResult<Models.Song> - { - Data = song - }; - } - } - - public Task<OperationResult<bool>> UpdateSongAsync(FileSystemDirectoryInfo directoryInfo, Models.Song song, - CancellationToken cancellationToken = default) - { - // Use ATL to write common tags back to the media file - var fullPath = song.File.FullName(directoryInfo); - if (!OptimizedFileOperations.WaitForFileStabilityAsync(fullPath, cancellationToken: cancellationToken) - .GetAwaiter() - .GetResult()) - { - Log.Warning("[{Plugin}] File [{File}] not stable for write, skipping.", DisplayName, fullPath); - return Task.FromResult(new OperationResult<bool>(["File not stable for write"]) - { - Data = false - }); - } - - var track = new Track(fullPath); - - try - { - // Map core tags from song meta - track.Title = song.Title(); - track.Album = song.AlbumTitle(); - track.Artist = song.SongArtist(); - track.AlbumArtist = song.AlbumArtist(); - track.TrackNumber = song.SongNumber(); - - var year = song.AlbumYear(); - if (year.HasValue) - { - track.Date = new DateTime(year.Value, 1, 1); - } - - var comment = song.Comment(); - if (!string.IsNullOrWhiteSpace(comment)) - { - track.Comment = comment; - } - - // Genre can be a single or multiple values; join as a single string - var genres = song.Tags?.Where(t => t.Identifier == MetaTagIdentifier.Genre) - .Select(t => t.Value?.ToString()) - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Distinct() - .ToArray() ?? []; - if (genres.Length > 0) - { - track.Genre = string.Join(';', genres); - } - - track.Save(); - - return Task.FromResult(new OperationResult<bool> { Data = true }); - } - catch (Exception ex) - { - Log.Error(ex, "Failed to update tags for file [{File}]", fullPath); - return Task.FromResult(new OperationResult<bool>([$"Failed to update tags: {ex.Message}"]) { Data = false }); - } - } -} diff --git a/src/Melodee.Common/Plugins/MetaData/Song/NativeId3MetaTag.cs b/src/Melodee.Common/Plugins/MetaData/Song/NativeId3MetaTag.cs new file mode 100644 index 000000000..ba6b7b995 --- /dev/null +++ b/src/Melodee.Common/Plugins/MetaData/Song/NativeId3MetaTag.cs @@ -0,0 +1,232 @@ +using System.Diagnostics; +using ATL; +using Melodee.Common.Configuration; +using Melodee.Common.Enums; +using Melodee.Common.Extensions; +using Melodee.Common.Metadata.AudioTags; +using Melodee.Common.Metadata.AudioTags.Models; +using Melodee.Common.Models; +using Melodee.Common.Models.Extensions; +using Melodee.Common.Plugins.Processor; +using Melodee.Common.Services.Scanning; +using Melodee.Common.Utility; +using Serilog; +using Serilog.Events; +using SerilogTimings; + +namespace Melodee.Common.Plugins.MetaData.Song; + +public sealed class NativeId3MetaTag( + IMetaTagsProcessorPlugin metaTagsProcessorPlugin, + IMelodeeConfiguration configuration) : MetaDataBase(configuration), ISongPlugin +{ + private readonly IMetaTagsProcessorPlugin _metaTagsProcessorPlugin = metaTagsProcessorPlugin; + + public override string Id => "0AE16462-6924-496B-AC5E-C9CD70EA078D"; + + public override string DisplayName => nameof(NativeId3MetaTag); + + public override bool IsEnabled { get; set; } = true; + + public override int SortOrder { get; } = 1; + + public override bool DoesHandleFile(FileSystemDirectoryInfo directoryInfo, FileSystemFileInfo fileSystemInfo) + { + if (!IsEnabled || !fileSystemInfo.Exists(directoryInfo)) + { + return false; + } + + return string.Equals(fileSystemInfo.Extension(directoryInfo), ".mp3", StringComparison.OrdinalIgnoreCase); + } + + public async Task<OperationResult<Models.Song>> ProcessFileAsync(FileSystemDirectoryInfo directoryInfo, + FileSystemFileInfo fileSystemInfo, CancellationToken cancellationToken = default) + { + using (Operation.At(LogEventLevel.Debug) + .Time("[{PluginName}] Processing [{fileSystemInfo}]", DisplayName, fileSystemInfo.Name)) + { + var tags = new List<MetaTag<object?>>(); + var mediaAudios = new List<MediaAudio<object?>>(); + var images = new List<ImageInfo>(); + var fullName = fileSystemInfo.FullName(directoryInfo); + + try + { + if (fileSystemInfo.Exists(directoryInfo)) + { + if (!await OptimizedFileOperations.WaitForFileStabilityAsync(fullName, cancellationToken: cancellationToken) + .ConfigureAwait(false)) + { + Log.Warning("[{Plugin}] File [{File}] not stable for read, skipping.", DisplayName, fullName); + return new OperationResult<Models.Song>(["File not stable for read"]) + { + Data = new Models.Song + { + CrcHash = string.Empty, + File = fileSystemInfo + } + }; + } + + var tagData = await ReadTagDataAsync(fullName, cancellationToken).ConfigureAwait(false); + tags.AddRange(tagData.Select(ToMetaTag)); + + var duration = ReadDurationMs(fullName); + if (duration is > 0) + { + mediaAudios.Add(new MediaAudio<object?> + { + Identifier = MediaAudioIdentifier.DurationMs, + Value = duration + }); + } + } + } + catch (Exception e) + { + Log.Error(e, "fileSystemInfo [{fileSystemInfo}]", fileSystemInfo); + } + + if (tags.All(x => x.Identifier != MetaTagIdentifier.RecordingYear)) + { + tags.Add(new MetaTag<object?> + { + Identifier = MetaTagIdentifier.RecordingYear, + Value = 0 + }); + } + + var metaTagsProcessorResult = + await _metaTagsProcessorPlugin.ProcessMetaTagAsync(directoryInfo, fileSystemInfo, tags, + cancellationToken); + if (!metaTagsProcessorResult.IsSuccess) + { + return new OperationResult<Models.Song>(metaTagsProcessorResult.Messages) + { + Errors = metaTagsProcessorResult.Errors, + Data = new Models.Song + { + CrcHash = string.Empty, + File = fileSystemInfo + } + }; + } + + var song = new Models.Song + { + CrcHash = Crc32.Calculate(new FileInfo(fullName)), + File = fileSystemInfo, + Images = images, + Tags = metaTagsProcessorResult.Data, + MediaAudios = mediaAudios, + SortOrder = SafeParser.ToNumber<int>(tags.FirstOrDefault(x => x.Identifier == MetaTagIdentifier.TrackNumber)?.Value) + }; + if (!song.IsValid(Configuration)) + { + Trace.WriteLine("Song is invalid"); + } + + return new OperationResult<Models.Song> + { + Data = song + }; + } + } + + public Task<OperationResult<bool>> UpdateSongAsync(FileSystemDirectoryInfo directoryInfo, Models.Song song, + CancellationToken cancellationToken = default) + { + var fullPath = song.File.FullName(directoryInfo); + if (!OptimizedFileOperations.WaitForFileStabilityAsync(fullPath, cancellationToken: cancellationToken) + .GetAwaiter() + .GetResult()) + { + Log.Warning("[{Plugin}] File [{File}] not stable for write, skipping.", DisplayName, fullPath); + return Task.FromResult(new OperationResult<bool>(["File not stable for write"]) + { + Data = false + }); + } + + var track = new Track(fullPath); + + try + { + track.Title = song.Title(); + track.Album = song.AlbumTitle(); + track.Artist = song.SongArtist(); + track.AlbumArtist = song.AlbumArtist(); + track.TrackNumber = song.SongNumber(); + + var year = song.AlbumYear(); + if (year.HasValue) + { + track.Date = new DateTime(year.Value, 1, 1); + } + + var comment = song.Comment(); + if (!string.IsNullOrWhiteSpace(comment)) + { + track.Comment = comment; + } + + var genres = song.Tags?.Where(t => t.Identifier == MetaTagIdentifier.Genre) + .Select(t => t.Value?.ToString()) + .Where(v => !string.IsNullOrWhiteSpace(v)) + .Distinct() + .ToArray() ?? []; + if (genres.Length > 0) + { + track.Genre = string.Join(';', genres); + } + + track.Save(); + + return Task.FromResult(new OperationResult<bool> { Data = true }); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to update tags for file [{File}]", fullPath); + return Task.FromResult(new OperationResult<bool>([$"Failed to update tags: {ex.Message}"]) { Data = false }); + } + } + + private static async Task<IEnumerable<KeyValuePair<MetaTagIdentifier, object>>> ReadTagDataAsync(string fullName, CancellationToken cancellationToken) + { + try + { + var tagData = await AudioTagManager.ReadAllTagsAsync(fullName, cancellationToken).ConfigureAwait(false); + return tagData.Tags + .Where(x => x.Value.ToString().Nullify() is not null) + .Where(x => x.Value is not byte[] && x.Value is not IEnumerable<AudioImage>); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Debug(ex, "Native ID3 tag reader could not read [{File}]", fullName); + return []; + } + } + + private static MetaTag<object?> ToMetaTag(KeyValuePair<MetaTagIdentifier, object> tag) + { + return new MetaTag<object?> + { + Identifier = tag.Key, + Value = tag.Value + }; + } + + private static double? ReadDurationMs(string fullName) + { + try + { + var atlTag = new Track(fullName); + return atlTag.DurationMs > 0 ? atlTag.DurationMs : null; + } + catch + { + return null; + } + } +} diff --git a/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesImageSearchResultModels.cs b/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesImageSearchResultModels.cs index 75dedc59b..1501bd0fa 100644 --- a/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesImageSearchResultModels.cs +++ b/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesImageSearchResultModels.cs @@ -8,9 +8,9 @@ public record ITunesSearchResult public record Result { - public int? AmgArtistId { get; init; } + public long? AmgArtistId { get; init; } - public int? ArtistId { get; init; } + public long? ArtistId { get; init; } public string? ArtistLinkUrl { get; init; } @@ -28,7 +28,7 @@ public record Result public string? CollectionExplicitness { get; init; } - public int? CollectionId { get; init; } + public long? CollectionId { get; init; } public string? CollectionName { get; init; } @@ -46,7 +46,7 @@ public record Result public string? PrimaryGenreName { get; init; } - public int? PrimaryGenreId { get; init; } + public long? PrimaryGenreId { get; init; } public DateTime? ReleaseDate { get; init; } diff --git a/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesSearchEngine.cs b/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesSearchEngine.cs index d2d4dae97..9c667a5e0 100644 --- a/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesSearchEngine.cs +++ b/src/Melodee.Common/Plugins/SearchEngine/ITunes/ITunesSearchEngine.cs @@ -97,7 +97,7 @@ public class ITunesSearchEngine( ReleaseDate = sr.ReleaseDate, ThumbnailUrl = sr.ArtworkUrl100!, Title = sr.CollectionName, - UniqueId = sr.CollectionId ?? 0 + sr.ArtistId ?? 0, + UniqueId = sr.CollectionId ?? sr.ArtistId ?? 0, Width = 600 }); } @@ -141,7 +141,7 @@ public class ITunesSearchEngine( var httpClient = httpClientFactory.CreateClient(); var requestUri = - $"https://itunes.apple.com/search?term={Uri.EscapeDataString(query.Name.Trim())}&entity=allArtist&country={query.Country}&media=all&limit={{maxResults}}"; + $"https://itunes.apple.com/search?term={Uri.EscapeDataString(query.Name.Trim())}&entity=allArtist&country={query.Country}&media=all&limit={maxResults}"; try { @@ -175,7 +175,7 @@ public class ITunesSearchEngine( Rank = 1, ThumbnailUrl = sr.ArtworkUrl100!, Title = sr.CollectionName, - UniqueId = sr.CollectionId ?? 0 + sr.ArtistId ?? 0, + UniqueId = sr.CollectionId ?? sr.ArtistId ?? 0, Width = 600 }); } diff --git a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzImportSummary.cs b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzImportSummary.cs new file mode 100644 index 000000000..eecc7a308 --- /dev/null +++ b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzImportSummary.cs @@ -0,0 +1,19 @@ +namespace Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; + +/// <summary> +/// Row-count summary produced by the DecentDB MusicBrainz streaming importer. +/// </summary> +public sealed record DecentDBMusicBrainzImportSummary( + int Artists, + int ArtistAliases, + int Links, + int ArtistArtistLinks, + int ArtistRelations, + int PrimaryArtistCredits, + int ReleaseGroups, + int ReleaseGroupMetaRows, + int Releases, + int Albums) +{ + public bool HasMaterializedData => Artists > 0 && Albums > 0; +} diff --git a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs index 202f84702..80261ec54 100644 --- a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs +++ b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBMusicBrainzRepository.cs @@ -78,7 +78,10 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( var startTicks = Stopwatch.GetTimestamp(); var maxSearchResults = 10; - var cacheKey = $"{query.NameNormalized}:{query.MusicBrainzIdValue}:{maxResults}"; + var albumKey = query.AlbumKeyValues is { Length: > 0 } + ? string.Join("|", query.AlbumKeyValues.Select(x => $"{x.Key}:{x.Value.ToNormalizedString()}")) + : string.Empty; + var cacheKey = $"{query.NameNormalized}:{query.MusicBrainzIdValue}:{maxResults}:{albumKey}"; if (SearchCache.TryGetValue(cacheKey, out var cached) && cached.CachedAt > DateTime.UtcNow.AddMinutes(-CacheExpirationMinutes)) { @@ -88,6 +91,12 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( var data = new List<ArtistSearchResult>(); var totalCount = 0; + var nameLookupMs = 0.0; + var idLookupMs = 0.0; + var albumLoadMs = 0.0; + var aliasLoadMs = 0.0; + var rankingMs = 0.0; + var releaseLoadMode = "none"; try { @@ -101,7 +110,9 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( if (!string.IsNullOrEmpty(query.NameNormalized)) { + var phaseTicks = Stopwatch.GetTimestamp(); foundArtists = await SearchByNameAsync(context, query, maxSearchResults, cancellationToken); + nameLookupMs += Stopwatch.GetElapsedTime(phaseTicks).TotalMilliseconds; if (foundArtists.Length > 0 && !string.IsNullOrEmpty(mbIdRaw)) { @@ -115,11 +126,14 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( if (foundArtists.Length == 0 && !string.IsNullOrEmpty(mbIdRaw)) { + var phaseTicks = Stopwatch.GetTimestamp(); foundArtists = await context.Artists .AsNoTracking() .Where(a => a.MusicBrainzIdRaw == mbIdRaw) + .OrderBy(a => a.Id) .Take(maxSearchResults) .ToArrayAsync(cancellationToken); + idLookupMs += Stopwatch.GetElapsedTime(phaseTicks).TotalMilliseconds; } logger.Debug("[{RepoName}] Search found [{Count}] artists for [{NameNormalized}]", @@ -128,10 +142,17 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( if (foundArtists.Length > 0) { var artistIds = foundArtists.Select(a => a.MusicBrainzArtistId).ToArray(); - var allAlbums = await context.Albums - .AsNoTracking() - .Where(a => artistIds.Contains(a.MusicBrainzArtistId) && a.ReleaseDate > DateTime.MinValue) - .ToArrayAsync(cancellationToken); + var shouldLoadFullReleaseList = ShouldLoadFullReleaseList(maxResults); + releaseLoadMode = shouldLoadFullReleaseList ? "full" : "matching"; + var phaseTicks = Stopwatch.GetTimestamp(); + var allAlbums = await LoadAlbumsForArtistsAsync( + context, + artistIds, + query, + shouldLoadFullReleaseList, + maxSearchResults, + cancellationToken); + albumLoadMs += Stopwatch.GetElapsedTime(phaseTicks).TotalMilliseconds; var albumsByArtist = allAlbums .GroupBy(a => a.MusicBrainzArtistId) @@ -139,8 +160,14 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( .GroupBy(x => x.ReleaseGroupMusicBrainzIdRaw) .Select(rg => rg.OrderBy(x => x.ReleaseDate).First()) .ToArray()); - var aliasValuesByArtist = await LoadAliasValuesByArtistAsync(context, artistIds, cancellationToken); - + var shouldLoadAliases = ShouldLoadAliasValues(query, foundArtists, maxResults); + phaseTicks = Stopwatch.GetTimestamp(); + var aliasValuesByArtist = shouldLoadAliases + ? await LoadAliasValuesByArtistAsync(context, artistIds, cancellationToken) + : new Dictionary<long, string[]>(); + aliasLoadMs += Stopwatch.GetElapsedTime(phaseTicks).TotalMilliseconds; + + phaseTicks = Stopwatch.GetTimestamp(); foreach (var artist in foundArtists) { var alternateNamesValues = artist.AlternateNamesValues @@ -206,6 +233,7 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( } totalCount = foundArtists.Length; + rankingMs += Stopwatch.GetElapsedTime(phaseTicks).TotalMilliseconds; } } } @@ -246,6 +274,17 @@ public async Task<PagedResult<ArtistSearchResult>> SearchArtist( nameof(DecentDBMusicBrainzRepository), LogSanitizer.Sanitize(query.NameNormalized), elapsedMs); } + logger.Debug( + "[{RepoName}] SearchArtist timings for [{Query}]: nameLookup={NameLookupMs:F1}ms, idLookup={IdLookupMs:F1}ms, albumLoad={AlbumLoadMs:F1}ms, aliasLoad={AliasLoadMs:F1}ms, ranking={RankingMs:F1}ms, releaseLoadMode={ReleaseLoadMode}", + nameof(DecentDBMusicBrainzRepository), + LogSanitizer.Sanitize(query.NameNormalized), + nameLookupMs, + idLookupMs, + albumLoadMs, + aliasLoadMs, + rankingMs, + releaseLoadMode); + return result; } @@ -286,24 +325,28 @@ public async Task<OperationResult<bool>> ImportData( { var importer = new DecentDBStreamingMusicBrainzImporter(logger); - await importer.ImportAsync( + var importSummary = await importer.ImportAsync( ct => CreateImportContextAsync(request.TargetDatabasePath, ct), storagePath, progressCallback, cancellationToken); - await using var context = await CreateImportContextAsync(request.TargetDatabasePath, cancellationToken) - .ConfigureAwait(false); - var artistCount = await context.Artists.CountAsync(cancellationToken); - var albumCount = await context.Albums.CountAsync(cancellationToken); - logger.Information( - "DecentDBMusicBrainzRepository: Streaming import complete. Artists: {ArtistCount:N0}, Albums: {AlbumCount:N0}", - artistCount, albumCount); + "DecentDBMusicBrainzRepository: Streaming import complete. Artists: {ArtistCount:N0}, Aliases: {AliasCount:N0}, Artist relations: {RelationCount:N0}, Albums: {AlbumCount:N0}", + importSummary.Artists, + importSummary.ArtistAliases, + importSummary.ArtistRelations, + importSummary.Albums); + + if (request.VerifyFinalCounts) + { + await VerifyImportCountsAsync(request.TargetDatabasePath, importSummary, cancellationToken) + .ConfigureAwait(false); + } return new OperationResult<bool> { - Data = artistCount > 0 && albumCount > 0 + Data = importSummary.HasMaterializedData }; } catch (OperationCanceledException) @@ -340,6 +383,30 @@ private static Exception CreateImportFailureException(Exception exception) return new InvalidOperationException($"MusicBrainz import failed: {exception.Message}", exception); } + private async Task VerifyImportCountsAsync( + string? targetDatabasePath, + DecentDBMusicBrainzImportSummary expectedSummary, + CancellationToken cancellationToken) + { + await using var context = await CreateImportContextAsync(targetDatabasePath, cancellationToken) + .ConfigureAwait(false); + var artistCount = await context.Artists.CountAsync(cancellationToken); + var aliasCount = await context.ArtistAliases.CountAsync(cancellationToken); + var relationCount = await context.ArtistRelations.CountAsync(cancellationToken); + var albumCount = await context.Albums.CountAsync(cancellationToken); + + logger.Information( + "DecentDBMusicBrainzRepository: Verified import counts. Artists: {ArtistCount:N0}/{ExpectedArtistCount:N0}, Aliases: {AliasCount:N0}/{ExpectedAliasCount:N0}, Artist relations: {RelationCount:N0}/{ExpectedRelationCount:N0}, Albums: {AlbumCount:N0}/{ExpectedAlbumCount:N0}", + artistCount, + expectedSummary.Artists, + aliasCount, + expectedSummary.ArtistAliases, + relationCount, + expectedSummary.ArtistRelations, + albumCount, + expectedSummary.Albums); + } + private async Task<MusicBrainzDbContext> CreateImportContextAsync( string? targetDatabasePath, CancellationToken cancellationToken) @@ -432,6 +499,53 @@ private static async Task<Artist[]> SearchByNameAsync( .ToArrayAsync(cancellationToken); } + private static bool ShouldLoadFullReleaseList(int maxResults) + { + return maxResults > 1; + } + + private static bool ShouldLoadAliasValues(ArtistQuery query, Artist[] foundArtists, int maxResults) + { + return maxResults > 1 || + foundArtists.Any(artist => artist.NameNormalized != query.NameNormalized); + } + + private static async Task<Album[]> LoadAlbumsForArtistsAsync( + MusicBrainzDbContext context, + long[] artistIds, + ArtistQuery query, + bool loadFullReleaseList, + int maxSearchResults, + CancellationToken cancellationToken) + { + var albumsQuery = context.Albums + .AsNoTracking() + .Where(a => artistIds.Contains(a.MusicBrainzArtistId) && a.ReleaseDate > DateTime.MinValue); + + if (loadFullReleaseList) + { + return await albumsQuery.ToArrayAsync(cancellationToken); + } + + var normalizedAlbumNames = query.AlbumKeyValues? + .Select(x => x.Value.ToNormalizedString()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Distinct(StringComparer.Ordinal) + .ToArray() ?? []; + + if (normalizedAlbumNames.Length == 0) + { + return []; + } + + return await albumsQuery + .Where(a => normalizedAlbumNames.Contains(a.NameNormalized)) + .OrderBy(a => a.ReleaseDate) + .ThenBy(a => a.SortName) + .Take(Math.Max(maxSearchResults, normalizedAlbumNames.Length * maxSearchResults)) + .ToArrayAsync(cancellationToken); + } + private static async Task<Dictionary<long, string[]>> LoadAliasValuesByArtistAsync( MusicBrainzDbContext context, long[] artistIds, diff --git a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs index 05b678ac6..c6343e1ff 100644 --- a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs +++ b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/DecentDBStreamingMusicBrainzImporter.cs @@ -21,13 +21,13 @@ public sealed class DecentDBStreamingMusicBrainzImporter(ILogger logger) private const int MaxIndexSize = 255; private const int TotalImportSteps = 18; - public async Task ImportAsync( + public async Task<DecentDBMusicBrainzImportSummary> ImportAsync( MusicBrainzDbContext context, string storagePath, ImportProgressCallback? progressCallback = null, CancellationToken cancellationToken = default) { - await ImportAsync( + return await ImportAsync( _ => Task.FromResult(context), storagePath, progressCallback, @@ -36,13 +36,13 @@ await ImportAsync( .ConfigureAwait(false); } - public async Task ImportAsync( + public async Task<DecentDBMusicBrainzImportSummary> ImportAsync( Func<CancellationToken, Task<MusicBrainzDbContext>> contextFactory, string storagePath, ImportProgressCallback? progressCallback = null, CancellationToken cancellationToken = default) { - await ImportAsync( + return await ImportAsync( contextFactory, storagePath, progressCallback, @@ -51,7 +51,7 @@ await ImportAsync( .ConfigureAwait(false); } - private async Task ImportAsync( + private async Task<DecentDBMusicBrainzImportSummary> ImportAsync( Func<CancellationToken, Task<MusicBrainzDbContext>> contextFactory, string storagePath, ImportProgressCallback? progressCallback, @@ -67,28 +67,55 @@ private async Task ImportAsync( await context.Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); } - await ImportArtistStagingDataAsync(context, mbDumpPath, progressCallback, cancellationToken) + var artistStagingSummary = await ImportArtistStagingDataAsync( + context, + mbDumpPath, + progressCallback, + cancellationToken) .ConfigureAwait(false); ResetContextState(context, cancellationToken); - await MaterializeArtistsAsync(context, progressCallback, cancellationToken).ConfigureAwait(false); + await MaterializeArtistsAsync( + artistStagingSummary.Artists, + artistStagingSummary.ArtistAliases, + progressCallback, + cancellationToken) + .ConfigureAwait(false); ResetContextState(context, cancellationToken); - await MaterializeArtistRelationsAsync(context, progressCallback, cancellationToken).ConfigureAwait(false); + var artistRelationCount = await MaterializeArtistRelationsAsync(context, progressCallback, cancellationToken) + .ConfigureAwait(false); ResetContextState(context, cancellationToken); await DropArtistStagingTablesAsync(context, progressCallback, cancellationToken).ConfigureAwait(false); ResetContextState(context, cancellationToken); - var releaseCount = await ImportAlbumStagingDataAsync(context, mbDumpPath, progressCallback, cancellationToken) + var albumStagingSummary = await ImportAlbumStagingDataAsync(context, mbDumpPath, progressCallback, cancellationToken) .ConfigureAwait(false); ResetContextState(context, cancellationToken); - await MaterializeAlbumsAsync(context, releaseCount, progressCallback, cancellationToken).ConfigureAwait(false); + var albumCount = await MaterializeAlbumsAsync( + context, + albumStagingSummary.Releases, + progressCallback, + cancellationToken) + .ConfigureAwait(false); ResetContextState(context, cancellationToken); await DropAlbumStagingTablesAsync(context, progressCallback, cancellationToken).ConfigureAwait(false); ResetContextState(context, cancellationToken); + + return new DecentDBMusicBrainzImportSummary( + artistStagingSummary.Artists, + artistStagingSummary.ArtistAliases, + artistStagingSummary.Links, + artistStagingSummary.ArtistArtistLinks, + artistRelationCount, + albumStagingSummary.PrimaryArtistCredits, + albumStagingSummary.ReleaseGroups, + albumStagingSummary.ReleaseGroupMetaRows, + albumStagingSummary.Releases, + albumCount); } finally { @@ -101,7 +128,7 @@ await ImportArtistStagingDataAsync(context, mbDumpPath, progressCallback, cancel #region Phase 1: Artist Staging Data - private async Task ImportArtistStagingDataAsync( + private async Task<ArtistStagingImportSummary> ImportArtistStagingDataAsync( MusicBrainzDbContext context, string mbDumpPath, ImportProgressCallback? progressCallback, @@ -233,6 +260,12 @@ await context.Database.ExecuteSqlRawAsync( await RebuildMaterializedArtistLookupIndexesAsync(context, cancellationToken).ConfigureAwait(false); progressCallback?.Invoke("Loading Artists", 4, 4, WithImportStep(5, "Staging indices created")); progressCallback?.Invoke("Loading Artists", 4, 4, WithImportStep(5, "Artist staging data loaded")); + + return new ArtistStagingImportSummary( + artistCount, + aliasCount, + linkCount, + artistLinkCount); } } @@ -367,14 +400,14 @@ await context.Database.ExecuteSqlRawAsync( #region Phase 2: Materialize Artists private async Task MaterializeArtistsAsync( - MusicBrainzDbContext context, + int totalArtists, + int totalAliasRows, ImportProgressCallback? progressCallback, CancellationToken cancellationToken) { using (Operation.At(LogEventLevel.Debug).Time("DecentDbStreamingImporter: Materialize artists")) { - var totalArtists = await context.Artists.CountAsync(cancellationToken); - var totalAliasRows = await context.ArtistAliases.CountAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); var totalArtistsForProgress = Math.Max(totalArtists, 1); progressCallback?.Invoke( "Materializing Artists", @@ -400,7 +433,7 @@ private async Task MaterializeArtistsAsync( #region Phase 3: Materialize Artist Relations - private async Task MaterializeArtistRelationsAsync( + private async Task<int> MaterializeArtistRelationsAsync( MusicBrainzDbContext context, ImportProgressCallback? progressCallback, CancellationToken cancellationToken) @@ -427,6 +460,7 @@ private async Task MaterializeArtistRelationsAsync( logger.Debug("DecentDbStreamingImporter: Materialized {Count} artist relations", rowsAffected); progressCallback?.Invoke("Materializing Relations", 1, 1, WithImportStep(7, $"Materialized {rowsAffected:N0} artist relations")); + return rowsAffected; } } @@ -453,7 +487,7 @@ private async Task DropArtistStagingTablesAsync( #region Phase 5: Album Staging Data - private async Task<int> ImportAlbumStagingDataAsync( + private async Task<AlbumStagingImportSummary> ImportAlbumStagingDataAsync( MusicBrainzDbContext context, string mbDumpPath, ImportProgressCallback? progressCallback, @@ -576,7 +610,11 @@ await context.Database.ExecuteSqlRawAsync( progressCallback?.Invoke("Loading Albums", 6, 6, WithImportStep(16, "Creating resolved helper tables...")); progressCallback?.Invoke("Loading Albums", 6, 6, WithImportStep(16, "Album staging data loaded")); - return releaseCount; + return new AlbumStagingImportSummary( + creditNameCount, + groupCount, + metaCount, + releaseCount); } } @@ -584,7 +622,7 @@ await context.Database.ExecuteSqlRawAsync( #region Phase 6: Materialize Albums - private async Task MaterializeAlbumsAsync( + private async Task<int> MaterializeAlbumsAsync( MusicBrainzDbContext context, int expectedAlbumCount, ImportProgressCallback? progressCallback, @@ -646,6 +684,7 @@ AND rgm.""DateMonth"" > 0 logger.Debug("DecentDbStreamingImporter: Materialized {Count} albums", rowsAffected); progressCallback?.Invoke("Materializing Albums", 1, 1, WithImportStep(17, $"Materialized {rowsAffected:N0} albums")); + return rowsAffected; } } @@ -699,6 +738,18 @@ private readonly record struct AlbumInsertRow( int ReleaseType, DateTime ReleaseDate); + private readonly record struct ArtistStagingImportSummary( + int Artists, + int ArtistAliases, + int Links, + int ArtistArtistLinks); + + private readonly record struct AlbumStagingImportSummary( + int PrimaryArtistCredits, + int ReleaseGroups, + int ReleaseGroupMetaRows, + int Releases); + private async Task<int> StreamFileToStagingRawAsync( MusicBrainzDbContext context, string filePath, @@ -716,7 +767,7 @@ private async Task<int> StreamFileToStagingRawAsync( return 0; } - var totalCount = 0; + var insertedCount = 0; var pendingRows = new List<object?[]>(StagingRowsPerTransaction); var connection = context.Database.GetDbConnection(); @@ -743,11 +794,10 @@ private async Task<int> StreamFileToStagingRawAsync( } pendingRows.Add(values); - totalCount++; if (pendingRows.Count >= StagingRowsPerTransaction) { - await FlushPendingRowsAsync( + insertedCount += await FlushPendingRowsAsync( connection, tableName, columns, @@ -766,7 +816,7 @@ await FlushPendingRowsAsync( if (pendingRows.Count > 0) { - await FlushPendingRowsAsync( + insertedCount += await FlushPendingRowsAsync( connection, tableName, columns, @@ -776,10 +826,10 @@ await FlushPendingRowsAsync( .ConfigureAwait(false); } - return totalCount; + return insertedCount; } - private static async Task FlushPendingRowsAsync( + private static async Task<int> FlushPendingRowsAsync( DbConnection connection, string tableName, string[] columns, @@ -789,7 +839,7 @@ private static async Task FlushPendingRowsAsync( { if (pendingRows.Count == 0) { - return; + return 0; } if (connection.State != System.Data.ConnectionState.Open) @@ -802,6 +852,7 @@ private static async Task FlushPendingRowsAsync( { transaction = connection.BeginTransaction(); + var affectedRows = 0; for (var offset = 0; offset < pendingRows.Count; offset += StagingRowsPerInsertStatement) { var rowCount = Math.Min(StagingRowsPerInsertStatement, pendingRows.Count - offset); @@ -813,13 +864,15 @@ private static async Task FlushPendingRowsAsync( pendingRows, offset, rowCount, - onConflictDoNothing); + onConflictDoNothing); - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + var commandAffectedRows = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + affectedRows += commandAffectedRows >= 0 ? commandAffectedRows : rowCount; } transaction.Commit(); pendingRows.Clear(); + return affectedRows; } catch { diff --git a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/IMusicBrainzRepository.cs b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/IMusicBrainzRepository.cs index 5aa8a9ae1..a1be311f9 100644 --- a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/IMusicBrainzRepository.cs +++ b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/IMusicBrainzRepository.cs @@ -13,7 +13,10 @@ namespace Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; /// <param name="message">Optional message with additional details</param> public delegate void ImportProgressCallback(string phase, int currentItem, int totalItems, string? message = null); -public sealed record MusicBrainzImportRequest(string? StoragePath = null, string? TargetDatabasePath = null); +public sealed record MusicBrainzImportRequest( + string? StoragePath = null, + string? TargetDatabasePath = null, + bool VerifyFinalCounts = false); public interface IMusicBrainzRepository { diff --git a/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/MusicBrainzDecentDbWarmupService.cs b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/MusicBrainzDecentDbWarmupService.cs new file mode 100644 index 000000000..e351a1692 --- /dev/null +++ b/src/Melodee.Common/Plugins/SearchEngine/MusicBrainz/Data/MusicBrainzDecentDbWarmupService.cs @@ -0,0 +1,284 @@ +using System.Data.Common; +using System.Diagnostics; +using DecentDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; +using Serilog; + +namespace Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; + +/// <summary> +/// Warms the indexed DecentDB query shapes used by the MusicBrainz search provider. +/// </summary> +public sealed class MusicBrainzDecentDbWarmupService( + ILogger logger, + IDbContextFactory<MusicBrainzDbContext> dbContextFactory) +{ + /// <summary> + /// Warms the configured MusicBrainz DecentDB database. + /// </summary> + public Task<MusicBrainzDecentDbWarmupResult> WarmHotIndexesAsync( + CancellationToken cancellationToken = default) + { + return WarmHotIndexesAsync(null, cancellationToken); + } + + /// <summary> + /// Warms the MusicBrainz DecentDB database at the supplied path, or the configured database when omitted. + /// </summary> + public async Task<MusicBrainzDecentDbWarmupResult> WarmHotIndexesAsync( + string? databasePath, + CancellationToken cancellationToken = default) + { + var started = Stopwatch.GetTimestamp(); + var measurements = new List<MusicBrainzDecentDbWarmupMeasurement>(); + + try + { + await using var context = await CreateContextAsync(databasePath, cancellationToken).ConfigureAwait(false); + var canConnect = await context.Database.CanConnectAsync(cancellationToken).ConfigureAwait(false); + if (!canConnect) + { + return MusicBrainzDecentDbWarmupResult.CreateSkipped( + measurements, + Stopwatch.GetElapsedTime(started), + "MusicBrainz DecentDB database is not reachable."); + } + + var sample = await ResolveSampleAsync(context, measurements, cancellationToken).ConfigureAwait(false); + if (sample is null) + { + return MusicBrainzDecentDbWarmupResult.CreateSkipped( + measurements, + Stopwatch.GetElapsedTime(started), + "MusicBrainz DecentDB database has no materialized artists."); + } + + await MeasureAsync( + measurements, + "exact-normalized-name", + async () => (await context.Artists + .AsNoTracking() + .Where(artist => artist.NameNormalized == sample.NameNormalized) + .OrderBy(artist => artist.SortName) + .Select(artist => artist.Id) + .Take(10) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false)).Length) + .ConfigureAwait(false); + + await MeasureAsync( + measurements, + "exact-musicbrainz-id-raw", + async () => (await context.Artists + .AsNoTracking() + .Where(artist => artist.MusicBrainzIdRaw == sample.MusicBrainzIdRaw) + .OrderBy(artist => artist.Id) + .Select(artist => artist.Id) + .Take(1) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false)).Length) + .ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(sample.AliasNormalized)) + { + await MeasureAsync( + measurements, + "exact-normalized-alias", + async () => (await context.ArtistAliases + .AsNoTracking() + .Where(alias => alias.NameNormalized == sample.AliasNormalized) + .OrderBy(alias => alias.MusicBrainzArtistId) + .Select(alias => alias.MusicBrainzArtistId) + .Take(10) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false)).Length) + .ConfigureAwait(false); + } + + await MeasureAsync( + measurements, + "albums-by-artist-id", + async () => (await context.Albums + .AsNoTracking() + .Where(album => album.MusicBrainzArtistId == sample.MusicBrainzArtistId) + .OrderBy(album => album.ReleaseDate) + .ThenBy(album => album.SortName) + .Select(album => album.Id) + .Take(25) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false)).Length) + .ConfigureAwait(false); + + var result = MusicBrainzDecentDbWarmupResult.CreateCompleted( + measurements, + Stopwatch.GetElapsedTime(started)); + + logger.Information( + "MusicBrainz DecentDB warm-up completed in {ElapsedMilliseconds:F1} ms. Warmed {QueryCount} query shapes.", + result.Elapsed.TotalMilliseconds, + result.WarmedQueryCount); + + return result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + var result = MusicBrainzDecentDbWarmupResult.CreateFailed( + measurements, + Stopwatch.GetElapsedTime(started), + ex.Message); + + logger.Warning( + ex, + "MusicBrainz DecentDB warm-up failed after {ElapsedMilliseconds:F1} ms. Search remains available; indexes will warm on demand.", + result.Elapsed.TotalMilliseconds); + + return result; + } + } + + private async Task<MusicBrainzDbContext> CreateContextAsync( + string? databasePath, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(databasePath)) + { + return await dbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + } + + await using var baseContext = await dbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var connectionString = baseContext.Database.GetConnectionString() + ?? throw new InvalidOperationException("MusicBrainzDbContext has no connection string configured."); + var builder = new DbConnectionStringBuilder + { + ConnectionString = connectionString + }; + builder["Data Source"] = databasePath; + + var options = new DbContextOptionsBuilder<MusicBrainzDbContext>() + .UseDecentDB(builder.ConnectionString, optionsBuilder => optionsBuilder.UseNodaTime()) + .Options; + + return new MusicBrainzDbContext(options); + } + + private static async Task<WarmupArtistSample?> ResolveSampleAsync( + MusicBrainzDbContext context, + List<MusicBrainzDecentDbWarmupMeasurement> measurements, + CancellationToken cancellationToken) + { + var sampleStarted = Stopwatch.GetTimestamp(); + var artist = await context.Artists + .AsNoTracking() + .OrderBy(artist => artist.Id) + .Select(artist => new + { + artist.Id, + artist.MusicBrainzArtistId, + artist.NameNormalized, + artist.MusicBrainzIdRaw + }) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + var sampleElapsed = Stopwatch.GetElapsedTime(sampleStarted); + measurements.Add(new MusicBrainzDecentDbWarmupMeasurement( + "sample-artist-row", + artist is null ? 0 : 1, + sampleElapsed)); + + if (artist is null) + { + return null; + } + + var aliasStarted = Stopwatch.GetTimestamp(); + var alias = await context.ArtistAliases + .AsNoTracking() + .Where(alias => alias.MusicBrainzArtistId == artist.MusicBrainzArtistId) + .OrderBy(alias => alias.NameNormalized) + .Select(alias => alias.NameNormalized) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + var aliasElapsed = Stopwatch.GetElapsedTime(aliasStarted); + measurements.Add(new MusicBrainzDecentDbWarmupMeasurement( + "sample-alias-by-artist-id", + string.IsNullOrWhiteSpace(alias) ? 0 : 1, + aliasElapsed)); + + return new WarmupArtistSample( + artist.MusicBrainzArtistId, + artist.NameNormalized, + artist.MusicBrainzIdRaw, + alias); + } + + private static async Task MeasureAsync( + List<MusicBrainzDecentDbWarmupMeasurement> measurements, + string name, + Func<Task<int>> query) + { + var started = Stopwatch.GetTimestamp(); + var rowCount = await query().ConfigureAwait(false); + measurements.Add(new MusicBrainzDecentDbWarmupMeasurement( + name, + rowCount, + Stopwatch.GetElapsedTime(started))); + } + + private sealed record WarmupArtistSample( + long MusicBrainzArtistId, + string NameNormalized, + string MusicBrainzIdRaw, + string? AliasNormalized); +} + +/// <summary> +/// Result of a MusicBrainz DecentDB warm-up run. +/// </summary> +public sealed record MusicBrainzDecentDbWarmupResult( + bool Succeeded, + bool Skipped, + string? Message, + TimeSpan Elapsed, + IReadOnlyList<MusicBrainzDecentDbWarmupMeasurement> Measurements) +{ + /// <summary> + /// Number of indexed query shapes executed during warm-up. + /// </summary> + public int WarmedQueryCount => Measurements.Count(measurement => + !measurement.Name.StartsWith("sample-", StringComparison.Ordinal)); + + internal static MusicBrainzDecentDbWarmupResult CreateCompleted( + IReadOnlyList<MusicBrainzDecentDbWarmupMeasurement> measurements, + TimeSpan elapsed) + { + return new MusicBrainzDecentDbWarmupResult(true, false, null, elapsed, measurements); + } + + internal static MusicBrainzDecentDbWarmupResult CreateSkipped( + IReadOnlyList<MusicBrainzDecentDbWarmupMeasurement> measurements, + TimeSpan elapsed, + string message) + { + return new MusicBrainzDecentDbWarmupResult(false, true, message, elapsed, measurements); + } + + internal static MusicBrainzDecentDbWarmupResult CreateFailed( + IReadOnlyList<MusicBrainzDecentDbWarmupMeasurement> measurements, + TimeSpan elapsed, + string message) + { + return new MusicBrainzDecentDbWarmupResult(false, false, message, elapsed, measurements); + } +} + +/// <summary> +/// Timing for a single MusicBrainz DecentDB warm-up query. +/// </summary> +public sealed record MusicBrainzDecentDbWarmupMeasurement( + string Name, + int RowCount, + TimeSpan Elapsed); diff --git a/src/Melodee.Common/Plugins/Validation/AlbumValidator.cs b/src/Melodee.Common/Plugins/Validation/AlbumValidator.cs index c199d61b2..142bcaa7a 100644 --- a/src/Melodee.Common/Plugins/Validation/AlbumValidator.cs +++ b/src/Melodee.Common/Plugins/Validation/AlbumValidator.cs @@ -207,22 +207,24 @@ private void AlbumIsStudioTypeAlbum(Album album) private void ArtistHasSearchEngineResult(Artist albumArtist) { - if (albumArtist.SearchEngineResultUniqueId == null) + if (albumArtist.SearchEngineResultUniqueId != null || albumArtist.IsValid()) { - _validationMessages.Add(new ValidationResultMessage - { - Message = "Album Artist is unknown, will need manual validation.", - Severity = ValidationResultMessageSeverity.Critical - }); - if (albumArtist.Name.Nullify() == null && - _albumNeedsAttentionReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists) == false) - { - _albumNeedsAttentionReasons |= AlbumNeedsAttentionReasons.HasUnknownArtist; - } - else if (!_albumNeedsAttentionReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists)) - { - _albumNeedsAttentionReasons |= AlbumNeedsAttentionReasons.HasInvalidArtists; - } + return; + } + + _validationMessages.Add(new ValidationResultMessage + { + Message = "Album Artist is unknown, will need manual validation.", + Severity = ValidationResultMessageSeverity.Critical + }); + if (albumArtist.Name.Nullify() == null && + _albumNeedsAttentionReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists) == false) + { + _albumNeedsAttentionReasons |= AlbumNeedsAttentionReasons.HasUnknownArtist; + } + else if (!_albumNeedsAttentionReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists)) + { + _albumNeedsAttentionReasons |= AlbumNeedsAttentionReasons.HasInvalidArtists; } } diff --git a/src/Melodee.Common/Services/ArtistService.cs b/src/Melodee.Common/Services/ArtistService.cs index 227b09c0e..d69e96435 100644 --- a/src/Melodee.Common/Services/ArtistService.cs +++ b/src/Melodee.Common/Services/ArtistService.cs @@ -356,6 +356,28 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals Guid? byMusicBrainzId, string? bySpotifyId, CancellationToken cancellationToken = default) + { + return await FindArtistAsync( + byId, + byApiKey, + byName, + byMusicBrainzId, + bySpotifyId, + null, + cancellationToken).ConfigureAwait(false); + } + + /// <summary> + /// Find the Artist using various given Ids. + /// </summary> + public async Task<MelodeeModels.OperationResult<Artist?>> FindArtistAsync( + int? byId, + Guid byApiKey, + string? byName, + Guid? byMusicBrainzId, + string? bySpotifyId, + string? byItunesId, + CancellationToken cancellationToken = default) { int? id = null; @@ -385,13 +407,17 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals .ConfigureAwait(false); } - // Try to find by MusicBrainz ID or Spotify ID - if (id == null && (byMusicBrainzId != null || bySpotifyId != null)) + var spotifyId = bySpotifyId.Nullify(); + var itunesId = byItunesId.Nullify(); + + // Try to find by trusted external IDs + if (id == null && (byMusicBrainzId != null || spotifyId != null || itunesId != null)) { id = await scopedContext.Artists .AsNoTracking() .Where(a => (byMusicBrainzId != null && a.MusicBrainzId == byMusicBrainzId) || - (bySpotifyId != null && a.SpotifyId == bySpotifyId)) + (spotifyId != null && a.SpotifyId == spotifyId) || + (itunesId != null && a.ItunesId == itunesId)) .Select(a => (int?)a.Id) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); @@ -411,13 +437,14 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals catch (Exception e) { Logger.Error(e, - "[{ServiceName}] attempting to Find Artist id [{Id}], apiKey [{ApiKey}], name [{Name}] musicbrainzId [{MbId}] spotifyId [{SpotifyId}]", + "[{ServiceName}] attempting to Find Artist id [{Id}], apiKey [{ApiKey}], name [{Name}] musicbrainzId [{MbId}] spotifyId [{SpotifyId}] itunesId [{ItunesId}]", nameof(ArtistService), byId, byApiKey, byName, byMusicBrainzId, - bySpotifyId); + bySpotifyId, + byItunesId); } if (id == null) diff --git a/src/Melodee.Common/Services/LibraryService.cs b/src/Melodee.Common/Services/LibraryService.cs index 3c2a0cd6b..cfa55617e 100644 --- a/src/Melodee.Common/Services/LibraryService.cs +++ b/src/Melodee.Common/Services/LibraryService.cs @@ -3,7 +3,6 @@ using System.Linq.Dynamic.Core; using System.Linq.Expressions; using Ardalis.GuardClauses; -using IdSharp.Common.Utils; using Melodee.Common.Configuration; using Melodee.Common.Constants; using Melodee.Common.Data; @@ -595,7 +594,7 @@ await ProcessExistingDirectoryMoveMergeAsync(configuration, if (image.FileInfo != null) { if (existingArtistImagesCrc32S.Contains( - CRC32.Calculate(image.FileInfo.ToFileInfo(libraryArtistDirectoryInfo)))) + Crc32.Calculate(image.FileInfo.ToFileInfo(libraryArtistDirectoryInfo)))) { var fileToDeleteFullName = Path.Combine(libraryArtistDirectoryInfo.FullName(), image.FileInfo.Name); @@ -655,6 +654,18 @@ await ProcessExistingDirectoryMoveMergeAsync(configuration, }; } + private static void TrackSkippedAlbumReason( + IDictionary<string, int> skippedByReason, + MelodeeModels.Album album) + { + var reason = album.StatusReasons == AlbumNeedsAttentionReasons.NotSet + ? album.Status.ToString() + : album.StatusReasons.ToString(); + + skippedByReason.TryGetValue(reason, out var count); + skippedByReason[reason] = count + 1; + } + public async Task<MelodeeModels.OperationResult<LibraryScanHistory?>> CreateLibraryScanHistory(Library library, LibraryScanHistory libraryScanHistory, CancellationToken cancellationToken = default) @@ -1078,6 +1089,7 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals var deserializationFailures = 0; var mergedExistingCount = 0; var movedAlbumCount = 0; + var skippedByReason = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < albumsForFromLibrary.Length; i += batchSize) { @@ -1145,6 +1157,7 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals else { skippedByStatus++; + TrackSkippedAlbumReason(skippedByReason, rr.album); } } } @@ -1209,7 +1222,8 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals AlbumsMergedWithExisting: mergedExistingCount, AlbumsSkippedByStatus: skippedByStatus, AlbumsSkippedAsDuplicateDirectory: skippedByDuplicatePrefix, - AlbumsFailedToLoad: deserializationFailures) + AlbumsFailedToLoad: deserializationFailures, + AlbumsSkippedByReason: skippedByReason) )); return new MelodeeModels.OperationResult<bool> { @@ -1271,6 +1285,7 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals var deserializationFailures = 0; var mergedExistingCount = 0; var movedAlbumCount = 0; + var skippedByReason = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < albumsForFromPath.Length; i += batchSize) { @@ -1336,6 +1351,7 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals else { skippedByStatus++; + TrackSkippedAlbumReason(skippedByReason, rr.album); } } } @@ -1404,7 +1420,8 @@ await ContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(fals AlbumsMergedWithExisting: mergedExistingCount, AlbumsSkippedByStatus: skippedByStatus, AlbumsSkippedAsDuplicateDirectory: skippedByDuplicatePrefix, - AlbumsFailedToLoad: deserializationFailures) + AlbumsFailedToLoad: deserializationFailures, + AlbumsSkippedByReason: skippedByReason) )); return new MelodeeModels.OperationResult<bool> { diff --git a/src/Melodee.Common/Services/Models/ProcessingEvent.cs b/src/Melodee.Common/Services/Models/ProcessingEvent.cs index 77dd811b6..49b3f684f 100644 --- a/src/Melodee.Common/Services/Models/ProcessingEvent.cs +++ b/src/Melodee.Common/Services/Models/ProcessingEvent.cs @@ -22,4 +22,5 @@ public record ProcessingEventStatistics( int AlbumsMergedWithExisting, int AlbumsSkippedByStatus, int AlbumsSkippedAsDuplicateDirectory, - int AlbumsFailedToLoad); + int AlbumsFailedToLoad, + IReadOnlyDictionary<string, int>? AlbumsSkippedByReason = null); diff --git a/src/Melodee.Common/Services/Scanning/DirectoryProcessorToStagingService.cs b/src/Melodee.Common/Services/Scanning/DirectoryProcessorToStagingService.cs index 2ef71caa6..a38bdb0ab 100644 --- a/src/Melodee.Common/Services/Scanning/DirectoryProcessorToStagingService.cs +++ b/src/Melodee.Common/Services/Scanning/DirectoryProcessorToStagingService.cs @@ -14,6 +14,7 @@ using Melodee.Common.Plugins.Conversion.Image; using Melodee.Common.Plugins.Conversion.Media; using Melodee.Common.Plugins.MetaData.Directory; +using Melodee.Common.Plugins.MetaData.Directory.Blackbeard; using Melodee.Common.Plugins.MetaData.Directory.Nfo; using Melodee.Common.Plugins.MetaData.Song; using Melodee.Common.Plugins.Processor; @@ -56,7 +57,21 @@ public sealed class DirectoryProcessorToStagingService( IImageProcessor imageProcessor) : ServiceBase(logger, cacheManager, contextFactory), IDisposable { + private const string CueSheetExtension = "CUE"; + private static readonly HashSet<string> SourceSidecarMetadataExtensions = new(StringComparer.OrdinalIgnoreCase) + { + CueSheetExtension, + M3UPlaylist.HandlesExtension, + Nfo.HandlesExtension, + SimpleFileVerification.HandlesExtension + }; + private static readonly HashSet<string> SourceResidueTextExtensions = new(StringComparer.OrdinalIgnoreCase) + { + "txt" + }; + private readonly SemaphoreSlim _processingThrottle = new(Environment.ProcessorCount); + private readonly SemaphoreSlim _conversionThrottle = new(CalculateMaxConcurrentConversions(Environment.ProcessorCount)); private bool _disposed; private IAlbumNamesInDirectoryPlugin _albumNamesInDirectoryPlugin = null!; private IAlbumValidator _albumValidator = new AlbumValidator(new MelodeeConfiguration([])); @@ -96,6 +111,20 @@ public void Dispose() { _disposed = true; _processingThrottle.Dispose(); + _conversionThrottle.Dispose(); + } + + /// <summary> + /// Calculates the conversion concurrency cap used to avoid saturating CPU and disk with ffmpeg processes. + /// </summary> + public static int CalculateMaxConcurrentConversions(int processorCount) + { + if (processorCount < 1) + { + return 1; + } + + return Math.Max(1, Math.Min(2, processorCount / 2)); } public async Task InitializeAsync(IMelodeeConfiguration? configuration = null, CancellationToken token = default) @@ -127,7 +156,7 @@ public async Task InitializeAsync(IMelodeeConfiguration? configuration, string? [ new AtlMetaTag(new MetaTagsProcessor(_configuration, serializer), imageProcessor, _imageConvertor, _imageValidator, _configuration), - new IdSharpMetaTag(new MetaTagsProcessor(_configuration, serializer), _configuration) + new NativeId3MetaTag(new MetaTagsProcessor(_configuration, serializer), _configuration) ]; _albumNamesInDirectoryPlugin = new AtlMetaTag(new MetaTagsProcessor(_configuration, serializer), imageProcessor, _imageConvertor, _imageValidator, _configuration); @@ -144,6 +173,10 @@ public async Task InitializeAsync(IMelodeeConfiguration? configuration, string? { IsEnabled = _configuration.GetValue<bool>(SettingRegistry.PluginEnabledCueSheet) }, + new Blackbeard(serializer, _albumValidator, _configuration) + { + IsEnabled = _configuration.GetValue<bool?>(SettingRegistry.PluginEnabledBlackbeard) ?? true + }, new SimpleFileVerification(serializer, _songPlugins, _albumValidator, _configuration) { IsEnabled = _configuration.GetValue<bool>(SettingRegistry.PluginEnabledSimpleFileVerification) @@ -194,13 +227,36 @@ public async Task<OperationResult<DirectoryProcessorResult>> ProcessDirectoryAsy FileSystemDirectoryInfo fileSystemDirectoryInfo, Instant? lastProcessDate, int? maxAlbumsToProcess, CancellationToken cancellationToken = default) { - return await ProcessDirectoryAsync(fileSystemDirectoryInfo, lastProcessDate, maxAlbumsToProcess, null, cancellationToken); + return await ProcessDirectoryAsync(fileSystemDirectoryInfo, lastProcessDate, maxAlbumsToProcess, (int?)null, cancellationToken); + } + + public async Task<OperationResult<DirectoryProcessorResult>> ProcessDirectoryAsync( + FileSystemDirectoryInfo fileSystemDirectoryInfo, + Instant? lastProcessDate, + int? maxAlbumsToProcess, + DirectoryRunContext? runContext, + CancellationToken cancellationToken = default) + { + return await ProcessDirectoryAsync(fileSystemDirectoryInfo, lastProcessDate, maxAlbumsToProcess, null, runContext, cancellationToken) + .ConfigureAwait(false); } public async Task<OperationResult<DirectoryProcessorResult>> ProcessDirectoryAsync( FileSystemDirectoryInfo fileSystemDirectoryInfo, Instant? lastProcessDate, int? maxAlbumsToProcess, int? libraryId, CancellationToken cancellationToken = default) + { + return await ProcessDirectoryAsync(fileSystemDirectoryInfo, lastProcessDate, maxAlbumsToProcess, libraryId, null, cancellationToken) + .ConfigureAwait(false); + } + + public async Task<OperationResult<DirectoryProcessorResult>> ProcessDirectoryAsync( + FileSystemDirectoryInfo fileSystemDirectoryInfo, + Instant? lastProcessDate, + int? maxAlbumsToProcess, + int? libraryId, + DirectoryRunContext? runContext, + CancellationToken cancellationToken = default) { CheckInitialized(); @@ -236,8 +292,9 @@ public async Task<OperationResult<DirectoryProcessorResult>> ProcessDirectoryAsy var startTicks = Stopwatch.GetTimestamp(); - // Create a run context for caching and observability - using var runContext = new DirectoryRunContext(); + // Standalone processing owns the run context; full library scans pass one shared context across stages. + using var localRunContext = runContext is null ? new DirectoryRunContext() : null; + var activeRunContext = runContext ?? localRunContext!; // Ensure directory to process exists Trace.WriteLine($"Ensuring processing path [{fileSystemDirectoryInfo.Path}] exists..."); @@ -355,12 +412,12 @@ await Parallel.ForEachAsync(directoriesToProcess, parallelOptions, async (direct artistsIdsSeen, albumsIdsSeen, songsIdsSeen, - runContext, + activeRunContext, libraryId, ct); numberOfAlbumsProcessed += processingResult.Item1; numberOfValidAlbumsProcessed += processingResult.Item2; - runContext.IncrementDirectoriesProcessed(); + activeRunContext.IncrementDirectoriesProcessed(); var currentCount = Interlocked.Increment(ref processedCount); if (currentCount >= nextProgressReport || currentCount == totalDirectories) @@ -423,6 +480,11 @@ await Parallel.ForEachAsync(directoriesToProcess, parallelOptions, async (direct } } + if (_configuration.GetValue<bool>(SettingRegistry.ProcessingDoDeleteOriginal)) + { + DeleteSourceResidueOnlyDirectoryFiles(fileSystemDirectoryInfo, Logger); + } + fileSystemDirectoryInfo.DeleteAllEmptyDirectories(); LogAndRaiseEvent(LogEventLevel.Debug, "Processing Complete!"); @@ -473,6 +535,12 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex Log.Write(logLevel, messageTemplate, args); } + OnProcessingEvent?.Invoke(this, FormatProcessingEventMessage(messageTemplate, exception, args)); + } + + public static string FormatProcessingEventMessage(string messageTemplate, Exception? exception = null, + params object[] args) + { var eventMessage = messageTemplate; if (args.Length > 0) { @@ -480,13 +548,15 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex { eventMessage = Smart.Format(eventMessage, args); } - catch (Exception e) + catch { - Trace.WriteLine(e); + eventMessage = $"{messageTemplate} [{string.Join(", ", args.Select(x => x?.ToString() ?? string.Empty))}]"; } } - OnProcessingEvent?.Invoke(this, exception?.ToString() ?? eventMessage); + return exception is null + ? eventMessage.ReplaceLineEndings(" ") + : $"Error: {eventMessage}: {exception.Message}".ReplaceLineEndings(" "); } private async Task<(int, int)> ProcessSingleDirectoryAsync( @@ -510,7 +580,19 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex Trace.WriteLine($"DirectoryInfoToProcess: [{directoryInfoToProcess}]"); try { - // Script evaluation hooks - process delete event first, then start event + var unstableSourceFile = await FindUnstableSourceFileAsync(directoryInfoToProcess, cancellationToken: cancellationToken) + .ConfigureAwait(false); + if (unstableSourceFile.Nullify() != null) + { + var message = $"Deferred directory [{directoryInfoToProcess.Path}] because source file [{unstableSourceFile}] is still changing."; + processingMessages.Add(message); + LogAndRaiseEvent(LogEventLevel.Warning, message); + + return (numberOfAlbumsProcessed, numberOfValidAlbumsProcessed); + } + + // Script evaluation hooks can skip a directory, but ingestion must not + // physically delete releases before they have a chance to reach staging. var scriptResult = await EvaluateDirectoryScriptsAsync( directoryInfoToProcess, cancellationToken); @@ -602,7 +684,7 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex } } - var convertedSongFiles = false; + var convertedSourceFilesByOriginalName = new Dictionary<string, FileSystemFileInfo>(StringComparer.OrdinalIgnoreCase); // Run Enabled Conversion scripts on each file in directory // e.g. Convert FLAC to MP3, Convert non JPEG files into JPEGs, etc. @@ -625,8 +707,30 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex if (plugin.DoesHandleFile(directoryInfoToProcess, fsi)) { - var pluginResult = await plugin - .ProcessFileAsync(directoryInfoToProcess, fsi, cancellationToken).ConfigureAwait(false); + await _conversionThrottle.WaitAsync(cancellationToken).ConfigureAwait(false); + var conversionStartTicks = Stopwatch.GetTimestamp(); + OperationResult<FileSystemFileInfo> pluginResult; + try + { + pluginResult = await plugin + .ProcessFileAsync(directoryInfoToProcess, fsi, cancellationToken).ConfigureAwait(false); + } + finally + { + runContext.AddConversionTime((long)Stopwatch.GetElapsedTime(conversionStartTicks).TotalMilliseconds); + if (!_disposed) + { + try + { + _conversionThrottle.Release(); + } + catch (ObjectDisposedException) + { + // Service shutdown disposed the semaphore while processing was being cancelled. + } + } + } + if (!pluginResult.IsSuccess) { // ConcurrentBag doesn't have AddRange, so add items individually @@ -648,7 +752,12 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex } else { - convertedSongFiles = pluginResult.Data.Extension(directoryInfoToProcess) != originalFileExtension; + if (!string.Equals(pluginResult.Data.Extension(directoryInfoToProcess), originalFileExtension, + StringComparison.OrdinalIgnoreCase) && + !string.Equals(pluginResult.Data.Name, fsi.Name, StringComparison.OrdinalIgnoreCase)) + { + convertedSourceFilesByOriginalName[fsi.Name] = pluginResult.Data; + } } } @@ -659,16 +768,10 @@ private void LogAndRaiseEvent(LogEventLevel logLevel, string messageTemplate, Ex } } - if (convertedSongFiles) + if (convertedSourceFilesByOriginalName.Count > 0) { - // Delete any melodee json files and let them get recreated as part of the conversion process - var melodeeFiles = directoryInfoToProcess.MelodeeJsonFiles().Select(f => f.FullName).ToList(); - if (melodeeFiles.Count > 0) - { - var deletedCount = await OptimizedFileOperations.DeleteFilesAsync(melodeeFiles, cancellationToken) - .ConfigureAwait(false); - LogAndRaiseEvent(LogEventLevel.Debug, "Deleted [{0}] existing Melodee files", null, deletedCount); - } + LogAndRaiseEvent(LogEventLevel.Debug, "Mapped [{0}] converted source files for staging", null, + convertedSourceFilesByOriginalName.Count); } // If no albums were created by previous plugins, create from media files @@ -730,6 +833,7 @@ await plugin.ProcessDirectoryAsync(directoryInfoToProcess, cancellationToken) albumsIdsSeen, songsIdsSeen, runContext, + convertedSourceFilesByOriginalName, cancellationToken); numberOfAlbumsProcessed += processingResult.Item1; numberOfValidAlbumsProcessed += processingResult.Item2; @@ -751,6 +855,7 @@ await plugin.ProcessDirectoryAsync(directoryInfoToProcess, cancellationToken) ConcurrentBag<long?> albumsIdsSeen, ConcurrentBag<Guid> songsIdsSeen, DirectoryRunContext runContext, + IReadOnlyDictionary<string, FileSystemFileInfo> convertedSourceFilesByOriginalName, CancellationToken cancellationToken) { var httpClient = httpClientFactory.CreateClient(); @@ -848,32 +953,52 @@ await plugin.ProcessDirectoryAsync(directoryInfoToProcess, cancellationToken) // Prepare song file operations if (album.Songs != null) { - foreach (var song in album.Songs.Where(x => x.File.OriginalName != null)) + var songs = album.Songs.ToArray(); + for (var i = 0; i < songs.Length; i++) { if (cancellationToken.IsCancellationRequested || _stopProcessingTriggered) { break; } - if (song.File.OriginalName != null) + var song = songs[i]; + if (!TryResolveSourceFileForStaging( + album.OriginalDirectory, + song.File, + convertedSourceFilesByOriginalName, + fileSystemService, + out var sourceSongFile, + out var oldSongFilename)) { - var oldSongFilename = fileSystemService.CombinePath(album.OriginalDirectory.FullName(), - song.File.OriginalName!); - if (!fileSystemService.FileExists(oldSongFilename)) - { - continue; - } + continue; + } - var newSongFileName = fileSystemService.CombinePath(albumDirectorySystemInfo.FullName(), - song.ToSongFileName(albumDirectorySystemInfo)); - if (!string.Equals(oldSongFilename, newSongFileName, - StringComparison.OrdinalIgnoreCase)) + if (!string.Equals(song.File.Name, sourceSongFile.Name, StringComparison.OrdinalIgnoreCase) || + song.File.Size != sourceSongFile.Size) + { + song = song with { - filesToCopy.Add((oldSongFilename, newSongFileName)); - song.File.Name = fileSystemService.GetFileName(newSongFileName); - } + File = new FileSystemFileInfo + { + Name = sourceSongFile.Name, + Size = sourceSongFile.Size, + OriginalName = sourceSongFile.Name + } + }; + songs[i] = song; + } + + var newSongFileName = fileSystemService.CombinePath(albumDirectorySystemInfo.FullName(), + song.ToSongFileName(albumDirectorySystemInfo)); + if (!string.Equals(oldSongFilename, newSongFileName, + StringComparison.OrdinalIgnoreCase)) + { + filesToCopy.Add((oldSongFilename, newSongFileName)); + song.File.Name = fileSystemService.GetFileName(newSongFileName); } } + + album.Songs = songs; } // Perform batch file operations with streaming and timing @@ -902,11 +1027,31 @@ await plugin.ProcessDirectoryAsync(directoryInfoToProcess, cancellationToken) album.Songs!.Any(x => (x.Tags ?? []).Any(y => y.WasModified))) { Trace.WriteLine("Running plugins on songs with modified tags..."); + var songsWithModifiedTags = album.Songs + .Where(x => x.Tags?.Any(t => t.WasModified) ?? false) + .ToArray(); + var songsWithExistingFiles = songsWithModifiedTags + .Where(x => File.Exists(x.File.FullName(albumDirectorySystemInfo))) + .ToArray(); + var missingSongFiles = songsWithModifiedTags + .Except(songsWithExistingFiles) + .Select(x => x.File.Name) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (missingSongFiles.Length > 0) + { + Logger.Warning( + "[{Name}] Skipping tag updates for [{Count}] missing staged files in album [{Album}] (examples: {Samples})", + nameof(DirectoryProcessorToStagingService), + missingSongFiles.Length, + album.AlbumTitle(), + string.Join(", ", missingSongFiles.Take(3))); + } foreach (var songPlugin in _songPlugins) { - foreach (var song in album.Songs.Where(x => - x.Tags?.Any(t => t.WasModified) ?? false)) + foreach (var song in songsWithExistingFiles) { if (cancellationToken.IsCancellationRequested || _stopProcessingTriggered) { @@ -1174,6 +1319,19 @@ await fileSystemService.WriteAllBytesAsync( { fileSystemService.DeleteFile(album.MelodeeDataFileName); } + + if (deleteOriginal) + { + var deletedSourceMetadataFiles = DeleteSourceSidecarMetadataFiles(album.OriginalDirectory, Logger); + if (deletedSourceMetadataFiles > 0) + { + LogAndRaiseEvent(LogEventLevel.Debug, + "Deleted [{0}] source metadata sidecar files for album [{1}]", + null, + deletedSourceMetadataFiles, + album.AlbumTitle() ?? string.Empty); + } + } } else { @@ -1194,8 +1352,265 @@ await fileSystemService.WriteAllBytesAsync( return new ValueTuple<int, int>(numberOfAlbumsProcessed, numberOfValidAlbumsProcessed); } + public static bool IsSourceSidecarMetadataFile(FileInfo fileInfo) + { + if (fileInfo.Name.DoStringsMatch(Blackbeard.HandlesFileName)) + { + return true; + } + + var extension = fileInfo.Extension.TrimStart('.'); + return SourceSidecarMetadataExtensions.Contains(extension); + } + + public static bool IsSourceResidueFile(FileInfo fileInfo) + { + if (IsSourceSidecarMetadataFile(fileInfo)) + { + return true; + } + + var extension = fileInfo.Extension.TrimStart('.'); + return FileHelper.IsFileImageType(extension) || + SourceResidueTextExtensions.Contains(extension); + } + + public static async Task<string?> FindUnstableSourceFileAsync( + FileSystemDirectoryInfo directoryInfo, + int delayMs = 250, + CancellationToken cancellationToken = default) + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (!dirInfo.Exists) + { + return null; + } + + var firstSnapshot = SnapshotSourceFiles(dirInfo); + if (firstSnapshot.Count == 0) + { + return null; + } + + await Task.Delay(delayMs, cancellationToken).ConfigureAwait(false); + + var secondSnapshot = SnapshotSourceFiles(dirInfo); + foreach (var (path, first) in firstSnapshot) + { + if (!secondSnapshot.TryGetValue(path, out var second) || + first.Length != second.Length || + first.LastWriteTimeUtc != second.LastWriteTimeUtc) + { + return path; + } + } + + return secondSnapshot.Keys.FirstOrDefault(path => !firstSnapshot.ContainsKey(path)); + } + + public static bool TryResolveSourceFileForStaging( + FileSystemDirectoryInfo sourceDirectory, + FileSystemFileInfo file, + IReadOnlyDictionary<string, FileSystemFileInfo> convertedSourceFilesByOriginalName, + IFileSystemService fileSystemService, + out FileSystemFileInfo sourceFile, + out string sourcePath) + { + sourceFile = file; + sourcePath = string.Empty; + + var sourceName = file.OriginalName.Nullify() ?? file.Name; + if (convertedSourceFilesByOriginalName.TryGetValue(sourceName, out var convertedSourceFile) && + TryResolveSourceFile(sourceDirectory, convertedSourceFile.Name, convertedSourceFile, fileSystemService, + out sourceFile, out sourcePath)) + { + return true; + } + + if (!string.Equals(sourceName, file.Name, StringComparison.OrdinalIgnoreCase) && + convertedSourceFilesByOriginalName.TryGetValue(file.Name, out convertedSourceFile) && + TryResolveSourceFile(sourceDirectory, convertedSourceFile.Name, convertedSourceFile, fileSystemService, + out sourceFile, out sourcePath)) + { + return true; + } + + if (TryResolveSourceFile(sourceDirectory, sourceName, file, fileSystemService, out sourceFile, out sourcePath)) + { + return true; + } + + return !string.Equals(sourceName, file.Name, StringComparison.OrdinalIgnoreCase) && + TryResolveSourceFile(sourceDirectory, file.Name, file, fileSystemService, out sourceFile, out sourcePath); + } + + private static bool TryResolveSourceFile( + FileSystemDirectoryInfo sourceDirectory, + string sourceName, + FileSystemFileInfo file, + IFileSystemService fileSystemService, + out FileSystemFileInfo sourceFile, + out string sourcePath) + { + sourceFile = file; + sourcePath = fileSystemService.CombinePath(sourceDirectory.FullName(), sourceName); + return fileSystemService.FileExists(sourcePath); + } + + public static bool IsSourceMetadataOnlyDirectory(FileSystemDirectoryInfo directoryInfo) + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (!dirInfo.Exists || dirInfo.EnumerateDirectories("*", SearchOption.TopDirectoryOnly).Any()) + { + return false; + } + + var files = dirInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly).ToArray(); + return files.Length > 0 && files.All(IsSourceSidecarMetadataFile); + } + + public static bool IsSourceResidueOnlyDirectory(FileSystemDirectoryInfo directoryInfo) + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (!dirInfo.Exists || dirInfo.EnumerateDirectories("*", SearchOption.TopDirectoryOnly).Any()) + { + return false; + } + + var files = dirInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly).ToArray(); + return files.Length > 0 && + !files.Any(file => FileHelper.IsFileMediaType(file.Extension)) && + files.All(IsSourceResidueFile); + } + + public static int DeleteSourceSidecarMetadataFiles(FileSystemDirectoryInfo directoryInfo, ILogger? logger = null) + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (!dirInfo.Exists) + { + return 0; + } + + var deletedCount = 0; + foreach (var fileInfo in dirInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly) + .Where(IsSourceSidecarMetadataFile)) + { + try + { + fileInfo.Delete(); + deletedCount++; + } + catch (Exception e) + { + logger?.Warning(e, "Unable to delete source metadata sidecar file [{FileName}]", fileInfo.FullName); + } + } + + return deletedCount; + } + + public static int DeleteSourceResidueFiles(FileSystemDirectoryInfo directoryInfo, ILogger? logger = null) + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (!dirInfo.Exists) + { + return 0; + } + + var deletedCount = 0; + foreach (var fileInfo in dirInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly) + .Where(IsSourceResidueFile)) + { + try + { + fileInfo.Delete(); + deletedCount++; + } + catch (Exception e) + { + logger?.Warning(e, "Unable to delete source residue file [{FileName}]", fileInfo.FullName); + } + } + + return deletedCount; + } + + public static int DeleteSourceResidueOnlyDirectoryFiles(FileSystemDirectoryInfo rootDirectory, ILogger logger) + { + var rootDirectoryInfo = rootDirectory.ToDirectoryInfo(); + if (!rootDirectoryInfo.Exists) + { + return 0; + } + + var deletedCount = 0; + foreach (var directoryInfo in rootDirectoryInfo.EnumerateDirectories("*", SearchOption.AllDirectories) + .OrderByDescending(x => x.FullName.Length) + .Select(x => x.ToDirectorySystemInfo())) + { + if (!IsSourceResidueOnlyDirectory(directoryInfo)) + { + continue; + } + + deletedCount += DeleteSourceResidueFiles(directoryInfo, logger); + TryDeleteDirectoryIfEmpty(directoryInfo, logger); + } + + if (deletedCount > 0) + { + logger.Information( + "[{ServiceName}] Deleted [{Count}] source residue files from media-free directories", + nameof(DirectoryProcessorToStagingService), + deletedCount); + } + + return deletedCount; + } + private record DirectoryScriptEvaluationResult(bool ShouldContinue, string? Message = null); + private static IReadOnlyDictionary<string, SourceFileSnapshot> SnapshotSourceFiles(DirectoryInfo dirInfo) + { + var snapshot = new Dictionary<string, SourceFileSnapshot>(StringComparer.OrdinalIgnoreCase); + foreach (var fileInfo in dirInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly)) + { + try + { + snapshot[fileInfo.FullName] = new SourceFileSnapshot(fileInfo.Length, fileInfo.LastWriteTimeUtc); + } + catch (IOException) + { + snapshot[fileInfo.FullName] = new SourceFileSnapshot(-1, DateTime.MinValue); + } + catch (UnauthorizedAccessException) + { + snapshot[fileInfo.FullName] = new SourceFileSnapshot(-1, DateTime.MinValue); + } + } + + return snapshot; + } + + private static void TryDeleteDirectoryIfEmpty(FileSystemDirectoryInfo directoryInfo, ILogger logger) + { + try + { + var dirInfo = directoryInfo.ToDirectoryInfo(); + if (dirInfo.Exists && + !dirInfo.EnumerateFileSystemInfos("*", SearchOption.TopDirectoryOnly).Any()) + { + dirInfo.Delete(); + } + } + catch (Exception e) + { + logger.Warning(e, "Unable to delete empty source residue directory [{Directory}]", directoryInfo.Path); + } + } + + private readonly record struct SourceFileSnapshot(long Length, DateTime LastWriteTimeUtc); + private async Task<DirectoryScriptEvaluationResult> EvaluateDirectoryScriptsAsync( FileSystemDirectoryInfo directory, CancellationToken cancellationToken) @@ -1207,37 +1622,6 @@ private async Task<DirectoryScriptEvaluationResult> EvaluateDirectoryScriptsAsyn Logger.Debug("Script context for [{Directory}]: TotalFilesCount={TotalFilesCount}, TotalDurationMinutes={TotalDurationMinutes}, HasTrackNumberGaps={HasTrackNumberGaps}, MediaFilesCount={MediaFilesCount}", directory.Path, context.TotalFilesCount, context.TotalDurationMinutes, context.HasTrackNumberGaps, context.MediaFilesCount); - // Evaluate DirectoryProcessingDelete script first - var deleteResult = await scriptOrchestrationService.EvaluateScriptForEventAsync( - ScriptEventNames.DirectoryProcessingDelete, - context, - cancellationToken); - - Logger.Debug("DirectoryProcessingDelete result: Result={Result}, IsDefault={IsDefault}, OnDeny={OnDeny}", - deleteResult.Result, deleteResult.IsDefault, deleteResult.OnDeny); - - if (deleteResult.Result && !deleteResult.IsDefault) - { - var onDeny = deleteResult.OnDeny?.ToLowerInvariant() ?? "delete"; - if (onDeny == "delete") - { - var handler = denyActionHandlerFactory.CreateHandler("delete"); - var deleteSuccess = await handler.ExecuteAsync(directory.Path, cancellationToken); - LogAndRaiseEvent( - deleteSuccess ? LogEventLevel.Information : LogEventLevel.Warning, - "DirectoryProcessingDelete script returned true; directory [{0}] {1}", - null, - directory.Path, - deleteSuccess ? "deleted" : "delete failed, continuing processing"); - - if (deleteSuccess) - { - return new DirectoryScriptEvaluationResult(false, "Directory deleted by script"); - } - } - } - - // Evaluate DirectoryProcessingStart script var startResult = await scriptOrchestrationService.EvaluateScriptForEventAsync( ScriptEventNames.DirectoryProcessingStart, context, @@ -1246,6 +1630,14 @@ private async Task<DirectoryScriptEvaluationResult> EvaluateDirectoryScriptsAsyn if (!startResult.Result && !startResult.IsDefault) { var onDeny = startResult.OnDeny?.ToLowerInvariant() ?? "skip"; + if (onDeny == "delete") + { + Logger.Warning( + "DirectoryProcessingStart requested delete for [{Directory}], using skip instead.", + directory.Path); + onDeny = "skip"; + } + var handler = denyActionHandlerFactory.CreateHandler(onDeny); await handler.ExecuteAsync(directory.Path, cancellationToken); diff --git a/src/Melodee.Common/Services/Scanning/DirectoryRunContext.cs b/src/Melodee.Common/Services/Scanning/DirectoryRunContext.cs index c5a1274a0..ef95cb1c1 100644 --- a/src/Melodee.Common/Services/Scanning/DirectoryRunContext.cs +++ b/src/Melodee.Common/Services/Scanning/DirectoryRunContext.cs @@ -5,6 +5,30 @@ namespace Melodee.Common.Services.Scanning; +/// <summary> +/// Aggregate performance metrics collected during one directory processing or library scan run. +/// </summary> +public sealed record DirectoryRunPerformanceSummary( + long RuntimeMs, + int DirectoriesProcessed, + long PluginTimeMs, + long AlbumProcessingTimeMs, + long EnrichmentTimeMs, + long CopyTimeMs, + long ConversionTimeMs, + int ConversionFilesProcessed, + int ArtistSearchPersistenceRetries, + int ArtistSearchPersistenceConflicts, + int ArtistSearchPersistenceCorruptions, + int ArtistSearchReadErrors, + int ArtistSearchReadCorruptions, + int AlbumsSkippedRevalidation, + int AlbumsDeferredRevalidation, + CacheStatistics ArtistSearchCache, + CacheStatistics ForcedArtistSearchCache, + CacheStatistics AlbumImageCache, + IReadOnlyDictionary<string, ThrottleStatistics> ApiThrottleStatistics); + /// <summary> /// Context for a single library processing run. /// Holds per-run caches to avoid duplicate API calls within a processing session. @@ -16,7 +40,16 @@ public sealed class DirectoryRunContext : IDisposable private long _albumProcessingTimeMs; private long _enrichmentTimeMs; private long _copyTimeMs; + private long _conversionTimeMs; + private int _conversionFilesProcessed; private int _directoriesProcessed; + private int _artistSearchPersistenceRetries; + private int _artistSearchPersistenceConflicts; + private int _artistSearchPersistenceCorruptions; + private int _artistSearchReadErrors; + private int _artistSearchReadCorruptions; + private int _albumsSkippedRevalidation; + private int _albumsDeferredRevalidation; /// <summary> /// Per-run cache for artist search results. @@ -24,6 +57,12 @@ public sealed class DirectoryRunContext : IDisposable /// </summary> public SingleFlightCache<ArtistQuery, ArtistSearchResult[]> ArtistSearchCache { get; } + /// <summary> + /// Per-run cache for forced artist revalidation searches. + /// Kept separate so an inbound negative lookup does not block a later forced revalidation lookup. + /// </summary> + public SingleFlightCache<ArtistQuery, ArtistSearchResult[]> ForcedArtistSearchCache { get; } + /// <summary> /// Per-run cache for album image search results. /// Keyed by normalized album identity (artist + album name + year). @@ -51,6 +90,13 @@ public DirectoryRunContext( negativeTtl: negTtl, cacheName: "ArtistRunCache"); + ForcedArtistSearchCache = new SingleFlightCache<ArtistQuery, ArtistSearchResult[]>( + NormalizeArtistKey, + maxSize: artistCacheSize, + positiveTtl: TimeSpan.FromHours(24), + negativeTtl: negTtl, + cacheName: "ForcedArtistRunCache"); + AlbumImageCache = new SingleFlightCache<AlbumQuery, ImageSearchResult[]>( NormalizeAlbumImageKey, maxSize: albumImageCacheSize, @@ -128,6 +174,15 @@ public void AddCopyTime(long milliseconds) Interlocked.Add(ref _copyTimeMs, milliseconds); } + /// <summary> + /// Records time spent converting one source file. + /// </summary> + public void AddConversionTime(long milliseconds) + { + Interlocked.Add(ref _conversionTimeMs, milliseconds); + Interlocked.Increment(ref _conversionFilesProcessed); + } + /// <summary> /// Increments the count of processed directories. /// </summary> @@ -136,43 +191,155 @@ public void IncrementDirectoriesProcessed() Interlocked.Increment(ref _directoriesProcessed); } + /// <summary> + /// Records that an artist search cache write was retried. + /// </summary> + public void RecordArtistSearchPersistenceRetry() + { + Interlocked.Increment(ref _artistSearchPersistenceRetries); + } + + /// <summary> + /// Records that DecentDB reported a transient artist search persistence conflict. + /// </summary> + public void RecordArtistSearchPersistenceConflict() + { + Interlocked.Increment(ref _artistSearchPersistenceConflicts); + } + + /// <summary> + /// Records that DecentDB reported a non-retryable error during artist search persistence. + /// </summary> + public void RecordArtistSearchPersistenceCorruption() + { + Interlocked.Increment(ref _artistSearchPersistenceCorruptions); + } + + /// <summary> + /// Records an artist search database read error. + /// </summary> + public void RecordArtistSearchReadError() + { + Interlocked.Increment(ref _artistSearchReadErrors); + } + + /// <summary> + /// Records an artist search database open/read failure that should not be retried. + /// </summary> + public void RecordArtistSearchReadCorruption() + { + Interlocked.Increment(ref _artistSearchReadCorruptions); + } + + /// <summary> + /// Records a staging album skipped because it could not usefully be revalidated. + /// </summary> + public void RecordAlbumSkippedRevalidation() + { + Interlocked.Increment(ref _albumsSkippedRevalidation); + } + + /// <summary> + /// Records a staging album deferred by the persistent revalidation backoff policy. + /// </summary> + public void RecordAlbumDeferredRevalidation() + { + Interlocked.Increment(ref _albumsDeferredRevalidation); + } + + /// <summary> + /// Returns a snapshot of the run counters. + /// </summary> + public DirectoryRunPerformanceSummary GetPerformanceSummary() + { + return new DirectoryRunPerformanceSummary( + RuntimeMs: _runStopwatch.ElapsedMilliseconds, + DirectoriesProcessed: _directoriesProcessed, + PluginTimeMs: _pluginTimeMs, + AlbumProcessingTimeMs: _albumProcessingTimeMs, + EnrichmentTimeMs: _enrichmentTimeMs, + CopyTimeMs: _copyTimeMs, + ConversionTimeMs: _conversionTimeMs, + ConversionFilesProcessed: _conversionFilesProcessed, + ArtistSearchPersistenceRetries: _artistSearchPersistenceRetries, + ArtistSearchPersistenceConflicts: _artistSearchPersistenceConflicts, + ArtistSearchPersistenceCorruptions: _artistSearchPersistenceCorruptions, + ArtistSearchReadErrors: _artistSearchReadErrors, + ArtistSearchReadCorruptions: _artistSearchReadCorruptions, + AlbumsSkippedRevalidation: _albumsSkippedRevalidation, + AlbumsDeferredRevalidation: _albumsDeferredRevalidation, + ArtistSearchCache: ArtistSearchCache.GetStatistics(), + ForcedArtistSearchCache: ForcedArtistSearchCache.GetStatistics(), + AlbumImageCache: AlbumImageCache.GetStatistics(), + ApiThrottleStatistics: ApiThrottler.GetStatistics()); + } + /// <summary> /// Logs summary statistics for this run. /// </summary> public void LogSummary() { - var artistStats = ArtistSearchCache.GetStatistics(); - var albumImageStats = AlbumImageCache.GetStatistics(); - var throttleStats = ApiThrottler.GetStatistics(); + var summary = GetPerformanceSummary(); Log.Information( "[DirectoryRunContext] Run completed in {TotalMs}ms | " + - "Directories: {DirCount} | " + - "Plugin: {PluginMs}ms | Album: {AlbumMs}ms | Enrichment: {EnrichMs}ms | Copy: {CopyMs}ms", - _runStopwatch.ElapsedMilliseconds, - _directoriesProcessed, - _pluginTimeMs, - _albumProcessingTimeMs, - _enrichmentTimeMs, - _copyTimeMs); + "Directories: {DirCount} | Plugin: {PluginMs}ms | Album: {AlbumMs}ms | " + + "Enrichment: {EnrichMs}ms | Conversion: {ConversionMs}ms ({ConversionFiles} files) | Copy: {CopyMs}ms", + summary.RuntimeMs, + summary.DirectoriesProcessed, + summary.PluginTimeMs, + summary.AlbumProcessingTimeMs, + summary.EnrichmentTimeMs, + summary.ConversionTimeMs, + summary.ConversionFilesProcessed, + summary.CopyTimeMs); Log.Information( "[DirectoryRunContext] Artist cache: {Entries} entries, {Hits} hits, {Misses} misses, {Coalesced} coalesced ({HitRate:P1} hit rate)", - artistStats.TotalEntries, - artistStats.Hits, - artistStats.Misses, - artistStats.CoalescedRequests, - artistStats.HitRate); + summary.ArtistSearchCache.TotalEntries, + summary.ArtistSearchCache.Hits, + summary.ArtistSearchCache.Misses, + summary.ArtistSearchCache.CoalescedRequests, + summary.ArtistSearchCache.HitRate); + + Log.Information( + "[DirectoryRunContext] Forced artist cache: {Entries} entries, {Hits} hits, {Misses} misses, {Coalesced} coalesced ({HitRate:P1} hit rate)", + summary.ForcedArtistSearchCache.TotalEntries, + summary.ForcedArtistSearchCache.Hits, + summary.ForcedArtistSearchCache.Misses, + summary.ForcedArtistSearchCache.CoalescedRequests, + summary.ForcedArtistSearchCache.HitRate); Log.Information( "[DirectoryRunContext] Album image cache: {Entries} entries, {Hits} hits, {Misses} misses, {Coalesced} coalesced ({HitRate:P1} hit rate)", - albumImageStats.TotalEntries, - albumImageStats.Hits, - albumImageStats.Misses, - albumImageStats.CoalescedRequests, - albumImageStats.HitRate); + summary.AlbumImageCache.TotalEntries, + summary.AlbumImageCache.Hits, + summary.AlbumImageCache.Misses, + summary.AlbumImageCache.CoalescedRequests, + summary.AlbumImageCache.HitRate); + + if (summary.ArtistSearchPersistenceConflicts > 0 || + summary.ArtistSearchPersistenceRetries > 0 || + summary.ArtistSearchPersistenceCorruptions > 0 || + summary.ArtistSearchReadErrors > 0 || + summary.ArtistSearchReadCorruptions > 0 || + summary.AlbumsSkippedRevalidation > 0 || + summary.AlbumsDeferredRevalidation > 0) + { + Log.Information( + "[DirectoryRunContext] Artist search: {ReadErrors} read errors, {ReadFailures} open/read failures | " + + "Artist persistence: {Conflicts} conflicts, {Retries} retries, {NonRetryableErrors} non-retryable errors | " + + "Revalidation skipped: {Skipped}, deferred: {Deferred}", + summary.ArtistSearchReadErrors, + summary.ArtistSearchReadCorruptions, + summary.ArtistSearchPersistenceConflicts, + summary.ArtistSearchPersistenceRetries, + summary.ArtistSearchPersistenceCorruptions, + summary.AlbumsSkippedRevalidation, + summary.AlbumsDeferredRevalidation); + } - foreach (var (provider, stats) in throttleStats) + foreach (var (provider, stats) in summary.ApiThrottleStatistics) { Log.Information( "[DirectoryRunContext] {Provider} throttle: {Total} total requests, {Throttled} rate-limited", @@ -186,6 +353,7 @@ public void Dispose() { LogSummary(); ArtistSearchCache.Dispose(); + ForcedArtistSearchCache.Dispose(); AlbumImageCache.Dispose(); ApiThrottler.Dispose(); } diff --git a/src/Melodee.Common/Services/Scanning/IStagingAlbumRevalidationStateStore.cs b/src/Melodee.Common/Services/Scanning/IStagingAlbumRevalidationStateStore.cs new file mode 100644 index 000000000..2acec7f4c --- /dev/null +++ b/src/Melodee.Common/Services/Scanning/IStagingAlbumRevalidationStateStore.cs @@ -0,0 +1,28 @@ +using Melodee.Common.Models; + +namespace Melodee.Common.Services.Scanning; + +public interface IStagingAlbumRevalidationStateStore +{ + Task<IStagingAlbumRevalidationStateSession> OpenAsync( + string stagingPath, + IReadOnlyCollection<Album> currentAlbums, + CancellationToken cancellationToken); +} + +public interface IStagingAlbumRevalidationStateSession : IAsyncDisposable +{ + StagingAlbumRevalidationDecision GetDecision(Album album, DateTimeOffset now, bool force); + + void RecordAttempt(Album album, DateTimeOffset now, string outcome); + + void RecordSuccess(Album album); + + Task SaveChangesAsync(CancellationToken cancellationToken); +} + +public sealed record StagingAlbumRevalidationDecision( + bool IsDue, + int AttemptCount = 0, + DateTimeOffset? NextAttemptAt = null, + string? Reason = null); diff --git a/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationState.cs b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationState.cs new file mode 100644 index 000000000..76d126fd2 --- /dev/null +++ b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationState.cs @@ -0,0 +1,23 @@ +using System.ComponentModel.DataAnnotations; + +namespace Melodee.Common.Services.Scanning; + +public sealed class StagingAlbumRevalidationState +{ + [Key] + public string AlbumKey { get; set; } = string.Empty; + + public string Fingerprint { get; set; } = string.Empty; + + public string AlbumDirectory { get; set; } = string.Empty; + + public int AttemptCount { get; set; } + + public DateTimeOffset? LastAttemptedAt { get; set; } + + public DateTimeOffset? NextAttemptAt { get; set; } + + public string? LastOutcome { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } +} diff --git a/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateDbContext.cs b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateDbContext.cs new file mode 100644 index 000000000..545c18c7b --- /dev/null +++ b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateDbContext.cs @@ -0,0 +1,32 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace Melodee.Common.Services.Scanning; + +public sealed class StagingAlbumRevalidationStateDbContext( + DbContextOptions<StagingAlbumRevalidationStateDbContext> options) : DbContext(options) +{ + public DbSet<StagingAlbumRevalidationState> AlbumRevalidationStates { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity<StagingAlbumRevalidationState>(entity => + { + entity.HasKey(x => x.AlbumKey); + entity.HasIndex(x => x.NextAttemptAt); + }); + + foreach (var entityType in modelBuilder.Model.GetEntityTypes()) + { + var properties = entityType.ClrType.GetProperties().Where(p => p.PropertyType == typeof(DateTimeOffset) || + p.PropertyType == typeof(DateTimeOffset?)); + foreach (var property in properties) + { + modelBuilder + .Entity(entityType.Name) + .Property(property.Name) + .HasConversion(new DateTimeOffsetToBinaryConverter()); + } + } + } +} diff --git a/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateStore.cs b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateStore.cs new file mode 100644 index 000000000..c17d5b3fd --- /dev/null +++ b/src/Melodee.Common/Services/Scanning/StagingAlbumRevalidationStateStore.cs @@ -0,0 +1,324 @@ +using System.Security.Cryptography; +using System.Text; +using Melodee.Common.Extensions; +using Melodee.Common.Models; +using Melodee.Common.Models.Extensions; +using Microsoft.EntityFrameworkCore; +using Serilog; + +namespace Melodee.Common.Services.Scanning; + +public sealed class StagingAlbumRevalidationStateStore(ILogger logger) : IStagingAlbumRevalidationStateStore +{ + public const string DatabaseFileName = ".melodee-revalidation.ddb"; + + public async Task<IStagingAlbumRevalidationStateSession> OpenAsync( + string stagingPath, + IReadOnlyCollection<Album> currentAlbums, + CancellationToken cancellationToken) + { + System.IO.Directory.CreateDirectory(stagingPath); + + var databasePath = GetDatabasePath(stagingPath); + try + { + return await OpenSessionAsync(databasePath, currentAlbums, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.Warning( + ex, + "[{Name}] Unable to open staging revalidation state database [{DatabasePath}]. Recreating it.", + nameof(StagingAlbumRevalidationStateStore), + databasePath); + + try + { + await RecreateDatabaseAsync(databasePath, cancellationToken).ConfigureAwait(false); + return await OpenSessionAsync(databasePath, currentAlbums, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception retryEx) when (retryEx is not OperationCanceledException) + { + logger.Warning( + retryEx, + "[{Name}] Unable to recreate staging revalidation state database [{DatabasePath}]. Continuing without persistent revalidation backoff.", + nameof(StagingAlbumRevalidationStateStore), + databasePath); + return new PassthroughStagingAlbumRevalidationStateSession(); + } + } + } + + public static string GetDatabasePath(string stagingPath) + { + return Path.Combine(stagingPath, DatabaseFileName); + } + + public static DateTimeOffset CalculateNextAttemptAt(int attemptCount, DateTimeOffset now) + { + var delay = attemptCount switch + { + <= 1 => TimeSpan.FromHours(6), + 2 => TimeSpan.FromHours(12), + 3 => TimeSpan.FromDays(1), + 4 => TimeSpan.FromDays(3), + _ => TimeSpan.FromDays(7) + }; + + return now.ToUniversalTime().Add(delay); + } + + public static string CreateAlbumKey(Album album) + { + var identity = $"{album.Id:N}|{Normalize(album.Directory.FullName())}"; + return Hash(identity); + } + + public static string CreateFingerprint(Album album) + { + var artist = album.Artist.NameNormalized.Nullify() ?? + album.Artist.Name.Nullify() ?? + string.Empty; + var title = album.AlbumTitle().ToNormalizedString() ?? + album.AlbumTitle() ?? + string.Empty; + var year = album.AlbumYear()?.ToString() ?? string.Empty; + var identity = string.Join( + '|', + Normalize(artist), + Normalize(title), + year, + album.Artist.AmgId ?? string.Empty, + album.Artist.ArtistDbId?.ToString() ?? string.Empty, + album.Artist.DiscogsId ?? string.Empty, + album.Artist.ItunesId ?? string.Empty, + album.Artist.LastFmId ?? string.Empty, + album.Artist.MusicBrainzId?.ToString() ?? string.Empty, + album.Artist.SearchEngineResultUniqueId?.ToString() ?? string.Empty, + album.Artist.SpotifyId ?? string.Empty, + album.Artist.WikiDataId ?? string.Empty, + album.StatusReasons.ToString()); + + return Hash(identity); + } + + private async Task<StagingAlbumRevalidationStateSession> OpenSessionAsync( + string databasePath, + IReadOnlyCollection<Album> currentAlbums, + CancellationToken cancellationToken) + { + var databaseExists = File.Exists(databasePath); + var options = new DbContextOptionsBuilder<StagingAlbumRevalidationStateDbContext>() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + var dbContext = new StagingAlbumRevalidationStateDbContext(options); + try + { + await dbContext.Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + + var states = await dbContext.AlbumRevalidationStates + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var currentKeys = currentAlbums + .Select(CreateAlbumKey) + .ToHashSet(StringComparer.Ordinal); + + var staleStates = states + .Where(x => !currentKeys.Contains(x.AlbumKey)) + .ToArray(); + + if (staleStates.Length > 0) + { + dbContext.AlbumRevalidationStates.RemoveRange(staleStates); + await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + logger.Debug( + "[{Name}] Removed [{Count}] stale staging revalidation state rows from [{DatabasePath}]", + nameof(StagingAlbumRevalidationStateStore), + staleStates.Length, + databasePath); + } + + logger.Information( + "[{Name}] Opened staging revalidation state database [{DatabasePath}]. ExistsBeforeOpen={ExistsBeforeOpen}; currentAlbums={CurrentAlbumCount}; stateRows={StateRowCount}; staleRowsRemoved={StaleRowCount}", + nameof(StagingAlbumRevalidationStateStore), + databasePath, + databaseExists, + currentAlbums.Count, + states.Count - staleStates.Length, + staleStates.Length); + + return new StagingAlbumRevalidationStateSession( + dbContext, + states.Except(staleStates).ToDictionary(x => x.AlbumKey, StringComparer.Ordinal)); + } + catch + { + await dbContext.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + private static async Task RecreateDatabaseAsync(string databasePath, CancellationToken cancellationToken) + { + DeleteDatabaseFiles(databasePath); + + var options = new DbContextOptionsBuilder<StagingAlbumRevalidationStateDbContext>() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + await using var dbContext = new StagingAlbumRevalidationStateDbContext(options); + await dbContext.Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + } + + private static void DeleteDatabaseFiles(string databasePath) + { + foreach (var path in DatabasePaths(databasePath)) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } + + private static IEnumerable<string> DatabasePaths(string databasePath) + { + yield return databasePath; + yield return $"{databasePath}-wal"; + yield return $"{databasePath}-shm"; + yield return $"{databasePath}.wal"; + yield return $"{databasePath}.shm"; + } + + private static string Normalize(string value) + { + return value.Trim().ToUpperInvariant(); + } + + private static string Hash(string value) + { + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))).ToLowerInvariant(); + } + + private sealed class StagingAlbumRevalidationStateSession( + StagingAlbumRevalidationStateDbContext dbContext, + Dictionary<string, StagingAlbumRevalidationState> states) : IStagingAlbumRevalidationStateSession + { + public StagingAlbumRevalidationDecision GetDecision(Album album, DateTimeOffset now, bool force) + { + if (force) + { + return new StagingAlbumRevalidationDecision(true, Reason: "Force"); + } + + var albumKey = CreateAlbumKey(album); + if (!states.TryGetValue(albumKey, out var state)) + { + return new StagingAlbumRevalidationDecision(true, Reason: "NoState"); + } + + var fingerprint = CreateFingerprint(album); + if (!string.Equals(state.Fingerprint, fingerprint, StringComparison.Ordinal)) + { + return new StagingAlbumRevalidationDecision(true, Reason: "AlbumChanged"); + } + + var nextAttemptAt = state.NextAttemptAt?.ToUniversalTime(); + if (nextAttemptAt is null || nextAttemptAt <= now.ToUniversalTime()) + { + return new StagingAlbumRevalidationDecision( + true, + state.AttemptCount, + nextAttemptAt, + "Due"); + } + + return new StagingAlbumRevalidationDecision( + false, + state.AttemptCount, + nextAttemptAt, + "Backoff"); + } + + public void RecordAttempt(Album album, DateTimeOffset now, string outcome) + { + var albumKey = CreateAlbumKey(album); + var fingerprint = CreateFingerprint(album); + var nowUtc = now.ToUniversalTime(); + + if (!states.TryGetValue(albumKey, out var state)) + { + state = new StagingAlbumRevalidationState + { + AlbumKey = albumKey, + Fingerprint = fingerprint + }; + states[albumKey] = state; + dbContext.AlbumRevalidationStates.Add(state); + } + else if (!string.Equals(state.Fingerprint, fingerprint, StringComparison.Ordinal)) + { + state.Fingerprint = fingerprint; + state.AttemptCount = 0; + } + + state.AttemptCount++; + state.AlbumDirectory = album.Directory.FullName(); + state.LastAttemptedAt = nowUtc; + state.NextAttemptAt = CalculateNextAttemptAt(state.AttemptCount, nowUtc); + state.LastOutcome = outcome; + state.UpdatedAt = nowUtc; + } + + public void RecordSuccess(Album album) + { + var albumKey = CreateAlbumKey(album); + if (!states.Remove(albumKey, out var state)) + { + return; + } + + dbContext.AlbumRevalidationStates.Remove(state); + } + + public Task SaveChangesAsync(CancellationToken cancellationToken) + { + return dbContext.SaveChangesAsync(cancellationToken); + } + + public async ValueTask DisposeAsync() + { + await dbContext.DisposeAsync().ConfigureAwait(false); + } + } + + private sealed class PassthroughStagingAlbumRevalidationStateSession : IStagingAlbumRevalidationStateSession + { + public StagingAlbumRevalidationDecision GetDecision(Album album, DateTimeOffset now, bool force) + { + return new StagingAlbumRevalidationDecision(true, Reason: "Passthrough"); + } + + public void RecordAttempt(Album album, DateTimeOffset now, string outcome) + { + } + + public void RecordSuccess(Album album) + { + } + + public Task SaveChangesAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } +} diff --git a/src/Melodee.Common/Services/SearchEngines/AlbumImageSearchEngineService.cs b/src/Melodee.Common/Services/SearchEngines/AlbumImageSearchEngineService.cs index 5c3632a0c..9c48e5d84 100644 --- a/src/Melodee.Common/Services/SearchEngines/AlbumImageSearchEngineService.cs +++ b/src/Melodee.Common/Services/SearchEngines/AlbumImageSearchEngineService.cs @@ -37,6 +37,49 @@ public class AlbumImageSearchEngineService( IHttpClientFactory httpClientFactory) : ServiceBase(logger, cacheManager, contextFactory) { + private static readonly TimeSpan SearchEngineTimeout = TimeSpan.FromSeconds(10); + + protected virtual IReadOnlyCollection<IAlbumImageSearchEnginePlugin> CreateSearchEngines(IMelodeeConfiguration configuration) + { + return + [ + new MusicBrainzCoverArtArchiveSearchEngine(configuration, musicBrainzRepository) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMusicBrainzEnabled) + }, + new DeezerSearchEngine(Logger, serializer, httpClientFactory) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineDeezerEnabled) + }, + new ITunesSearchEngine(Logger, serializer, httpClientFactory, CacheManager) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineITunesEnabled) + }, + new Spotify(Logger, configuration, CacheManager, spotifyClientBuilder, settingService, ContextFactory) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineSpotifyEnabled) + }, + new LastFm(Logger, configuration, serializer, httpClientFactory, CacheManager) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineLastFmEnabled) + }, + new MetalApiAlbumImageSearchEngine( + new MetalApiClient( + httpClientFactory.CreateClient(), + Logger, + new MetalApiOptions { Enabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) }), + Logger, + new MetalApiOptions { Enabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) }) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) + }, + new BraveAlbumImageSearchEnginePlugin(Logger, httpClientFactory, configuration) + { + IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineBraveEnabled) + } + ]; + } + /// <summary> /// Performs album image search with directory-run caching and request coalescing. /// When a runContext is provided, uses the run-scoped cache to avoid duplicate API calls. @@ -87,97 +130,85 @@ public async Task<OperationResult<ImageSearchResult[]>> DoSearchAsync(AlbumQuery var configuration = await configurationFactory.GetConfigurationAsync(token); var maxResultsValue = maxResults ?? configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize); - - var searchEngines = new List<IAlbumImageSearchEnginePlugin> + if (maxResultsValue <= 0) { - new MusicBrainzCoverArtArchiveSearchEngine(configuration, musicBrainzRepository) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMusicBrainzEnabled) - }, - new DeezerSearchEngine(Logger, serializer, httpClientFactory) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineDeezerEnabled) - }, - new ITunesSearchEngine(Logger, serializer, httpClientFactory, CacheManager) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineITunesEnabled) - }, - new Spotify(Logger, configuration, CacheManager, spotifyClientBuilder, settingService, ContextFactory) + return new OperationResult<ImageSearchResult[]> { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineSpotifyEnabled) - }, - new LastFm(Logger, configuration, serializer, httpClientFactory, CacheManager) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineLastFmEnabled) - }, - new MetalApiAlbumImageSearchEngine( - new MetalApiClient( - httpClientFactory.CreateClient(), - Logger, - new MetalApiOptions { Enabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) }), - Logger, - new MetalApiOptions { Enabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) }) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineMetalApiEnabled) - }, - new BraveAlbumImageSearchEnginePlugin(Logger, httpClientFactory, configuration) - { - IsEnabled = configuration.GetValue<bool>(SettingRegistry.SearchEngineBraveEnabled) - } - }; - var result = new List<ImageSearchResult>(); - var enabledEngines = searchEngines.Where(x => x.IsEnabled).OrderBy(x => x.SortOrder).ToArray(); + Data = [] + }; + } + + token.ThrowIfCancellationRequested(); + + var enabledEngines = CreateSearchEngines(configuration).Where(x => x.IsEnabled).OrderBy(x => x.SortOrder).ToArray(); // Sanitize query for logging to prevent log forging - var sanitizedQuery = LogSanitizer.Sanitize(query.ToString()); + var sanitizedQuery = LogSanitizer.Sanitize(query.ToString()) ?? string.Empty; Logger.Debug("Starting album image search for query [{Query}] with [{Count}] enabled search engines: [{Engines}]", sanitizedQuery, enabledEngines.Length, string.Join(", ", enabledEngines.Select(x => x.DisplayName))); - foreach (var searchEngine in enabledEngines) + var searchResults = await Task.WhenAll(enabledEngines.Select(x => SearchAsync(x, query, maxResultsValue, sanitizedQuery, token))).ConfigureAwait(false); + var result = searchResults + .SelectMany(x => x) + .OrderByDescending(x => x.Rank) + .Take(maxResultsValue) + .ToArray(); + + Logger.Debug("Album image search completed for query [{Query}] with [{Count}] total result(s)", sanitizedQuery, result.Length); + + return new OperationResult<ImageSearchResult[]> { - try + Data = result + }; + } + + private async Task<ImageSearchResult[]> SearchAsync( + IAlbumImageSearchEnginePlugin searchEngine, + AlbumQuery query, + int maxResults, + string sanitizedQuery, + CancellationToken token) + { + using var timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); + timeoutTokenSource.CancelAfter(SearchEngineTimeout); + + try + { + Logger.Debug("[{Plugin}] searching for album images with query [{Query}]", searchEngine.DisplayName, sanitizedQuery); + var searchResult = await searchEngine.DoAlbumImageSearch(query, maxResults, timeoutTokenSource.Token).ConfigureAwait(false); + if (searchResult.IsSuccess) { - Logger.Debug("[{Plugin}] searching for album images with query [{Query}]", searchEngine.DisplayName, sanitizedQuery); - var searchResult = await searchEngine.DoAlbumImageSearch(query, maxResultsValue, token); - if (searchResult.IsSuccess) - { - var foundCount = searchResult.Data?.Length ?? 0; - if (foundCount > 0) - { - Logger.Debug("[{Plugin}] found [{Count}] image(s) for query [{Query}]", - searchEngine.DisplayName, foundCount, sanitizedQuery); - result.AddRange(searchResult.Data ?? []); - } - else - { - Logger.Debug("[{Plugin}] found no images for query [{Query}]", searchEngine.DisplayName, sanitizedQuery); - } - } - else + var foundCount = searchResult.Data?.Length ?? 0; + if (foundCount > 0) { - Logger.Warning("[{Plugin}] search failed for query [{Query}]: [{Errors}]", - searchEngine.DisplayName, sanitizedQuery, string.Join(", ", searchResult.Errors ?? [])); + Logger.Debug("[{Plugin}] found [{Count}] image(s) for query [{Query}]", + searchEngine.DisplayName, foundCount, sanitizedQuery); + return searchResult.Data ?? []; } - if (searchEngine.StopProcessing || result.Count >= maxResultsValue) - { - Logger.Debug("[{Plugin}] stopping search - StopProcessing: {Stop}, ResultCount: {Count}/{Max}", - searchEngine.DisplayName, searchEngine.StopProcessing, result.Count, maxResultsValue); - break; - } + Logger.Debug("[{Plugin}] found no images for query [{Query}]", searchEngine.DisplayName, sanitizedQuery); } - catch (Exception e) + else { - Logger.Error(e, "[{Plugin}] threw error with query [{Query}]", searchEngine.DisplayName, sanitizedQuery); + Logger.Warning("[{Plugin}] search failed for query [{Query}]: [{Errors}]", + searchEngine.DisplayName, sanitizedQuery, string.Join(", ", searchResult.Errors ?? [])); } } - - Logger.Debug("Album image search completed for query [{Query}] with [{Count}] total result(s)", sanitizedQuery, result.Count); - - return new OperationResult<ImageSearchResult[]> + catch (OperationCanceledException) when (token.IsCancellationRequested) { - Data = result.OrderByDescending(x => x.Rank).ToArray() - }; + throw; + } + catch (OperationCanceledException) when (timeoutTokenSource.IsCancellationRequested) + { + Logger.Warning("[{Plugin}] timed out after {TimeoutSeconds}s for query [{Query}]", + searchEngine.DisplayName, SearchEngineTimeout.TotalSeconds, sanitizedQuery); + } + catch (Exception e) + { + Logger.Error(e, "[{Plugin}] threw error with query [{Query}]", searchEngine.DisplayName, sanitizedQuery); + } + + return []; } } diff --git a/src/Melodee.Common/Services/SearchEngines/ArtistSearchCache.cs b/src/Melodee.Common/Services/SearchEngines/ArtistSearchCache.cs index 9d0a1294d..5aedcb0a5 100644 --- a/src/Melodee.Common/Services/SearchEngines/ArtistSearchCache.cs +++ b/src/Melodee.Common/Services/SearchEngines/ArtistSearchCache.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Melodee.Common.Models.SearchEngines; using Melodee.Common.Services.Caching; +using Melodee.Common.Utility; namespace Melodee.Common.Services.SearchEngines; @@ -98,17 +99,13 @@ public void Clear() private string GenerateKey(ArtistQuery query) { - // Create a unique key based on normalized artist name - return $"{query.NameNormalized}|{query.MusicBrainzId}|{query.SpotifyId}".ToUpperInvariant(); + // Create a stable key from normalized artist identity so equivalent names share cache entries. + var normalizedName = UnicodeNormalizer.NormalizeForSearch(query.NameNormalized ?? query.Name); + return $"{normalizedName}|{query.MusicBrainzId}|{query.SpotifyId}".ToUpperInvariant(); } private void CleanExpiredEntries() { - if (_cache.Count <= _maxCacheSize) - { - return; - } - var now = DateTime.UtcNow; var expiredKeys = _cache .Where(kvp => kvp.Value.Expiry <= now) diff --git a/src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs b/src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs index 918e64753..1a3b8aa52 100644 --- a/src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs +++ b/src/Melodee.Common/Services/SearchEngines/ArtistSearchEngineService.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Globalization; +using System.Text.RegularExpressions; using Ardalis.GuardClauses; using Melodee.Common.Configuration; using Melodee.Common.Constants; @@ -29,7 +30,6 @@ using SerilogTimings; using Album = Melodee.Common.Models.SearchEngines.ArtistSearchEngineServiceData.Album; using Artist = Melodee.Common.Models.SearchEngines.ArtistSearchEngineServiceData.Artist; -using StringExtensions = Melodee.Common.Extensions.StringExtensions; namespace Melodee.Common.Services.SearchEngines; @@ -50,7 +50,16 @@ public class ArtistSearchEngineService( private IArtistTopSongsSearchEnginePlugin[] _artistTopSongsSearchEnginePlugins = []; private IMelodeeConfiguration _configuration = new MelodeeConfiguration([]); private bool _initialized; + private int _configurationChangeSubscribed; private readonly ArtistSearchCache _searchCache = new(); + private static readonly SemaphoreSlim ArtistSearchPersistenceSemaphore = new(1, 1); + private static readonly Regex CompoundArtistSeparatorRegex = new( + @"\s*(?:;|\s+(?:&|\+|x|vs\.?|with|feat\.?|featuring)\s+)\s*", + RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + TimeSpan.FromMilliseconds(100)); + private const int ArtistSearchPersistenceMaxAttempts = 3; + private const int LocalArtistCandidateLimit = 25; + private const int LocalArtistAliasBackfillBatchSize = 500; /// <summary> /// MusicBrainz rate limit: 1 request per second. @@ -69,8 +78,10 @@ public async Task InitializeAsync(IMelodeeConfiguration? configuration = null, { _configuration = configuration ?? await configurationFactory.GetConfigurationAsync(cancellationToken).ConfigureAwait(false); - // Subscribe to configuration changes to reload plugins - configurationFactory.ConfigurationChanged += OnConfigurationChanged; + if (Interlocked.Exchange(ref _configurationChangeSubscribed, 1) == 0) + { + configurationFactory.ConfigurationChanged += OnConfigurationChanged; + } await InitializePluginsAsync(cancellationToken).ConfigureAwait(false); @@ -78,6 +89,7 @@ public async Task InitializeAsync(IMelodeeConfiguration? configuration = null, { await scopedContext.Database.EnsureCreatedAsync(cancellationToken); await EnsureHousekeepingIndexesAsync(scopedContext, cancellationToken).ConfigureAwait(false); + await EnsureLocalArtistAliasLookupAsync(scopedContext, cancellationToken).ConfigureAwait(false); } _initialized = true; @@ -100,6 +112,100 @@ CREATE INDEX IF NOT EXISTS "IX_Artists_IsLocked_LastRefreshed" cancellationToken); } + private static async Task EnsureLocalArtistAliasLookupAsync( + ArtistSearchEngineServiceDbContext context, + CancellationToken cancellationToken) + { + if (!context.Database.IsRelational()) + { + return; + } + + await context.Database.ExecuteSqlRawAsync( + """ + CREATE TABLE IF NOT EXISTS "ArtistAliases" ( + "Id" INTEGER NOT NULL PRIMARY KEY, + "ArtistId" INTEGER NOT NULL, + "NameNormalized" TEXT NOT NULL, + FOREIGN KEY ("ArtistId") REFERENCES "Artists" ("Id") ON DELETE CASCADE + ) + """, + cancellationToken); + await context.Database.ExecuteSqlRawAsync( + """ + CREATE INDEX IF NOT EXISTS "IX_ArtistAliases_NameNormalized" + ON "ArtistAliases" ("NameNormalized") + """, + cancellationToken); + await context.Database.ExecuteSqlRawAsync( + """ + CREATE UNIQUE INDEX IF NOT EXISTS "IX_ArtistAliases_ArtistId_NameNormalized" + ON "ArtistAliases" ("ArtistId", "NameNormalized") + """, + cancellationToken); + + await BackfillLocalArtistAliasLookupAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static async Task BackfillLocalArtistAliasLookupAsync( + ArtistSearchEngineServiceDbContext context, + CancellationToken cancellationToken) + { + var firstAliasId = await context.ArtistAliases + .AsNoTracking() + .OrderBy(x => x.Id) + .Select(x => (int?)x.Id) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + if (firstAliasId.HasValue) + { + return; + } + + var pendingAliases = new List<ArtistAlias>(LocalArtistAliasBackfillBatchSize); + var artistsWithAliases = context.Artists + .AsNoTracking() + .Where(x => x.AlternateNames != null && x.AlternateNames != string.Empty) + .OrderBy(x => x.Id) + .Select(x => new + { + x.Id, + x.NameNormalized, + x.AlternateNames + }); + + await foreach (var artist in artistsWithAliases.AsAsyncEnumerable() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + foreach (var alias in GetNormalizedLocalArtistAliases(artist.AlternateNames, artist.NameNormalized)) + { + pendingAliases.Add(new ArtistAlias + { + ArtistId = artist.Id, + NameNormalized = alias + }); + } + + if (pendingAliases.Count < LocalArtistAliasBackfillBatchSize) + { + continue; + } + + context.ArtistAliases.AddRange(pendingAliases); + await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + pendingAliases.Clear(); + } + + if (pendingAliases.Count == 0) + { + return; + } + + context.ArtistAliases.AddRange(pendingAliases); + await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + private void OnConfigurationChanged(object? sender, EventArgs e) => _ = OnConfigurationChangedAsync(sender, e); private async Task OnConfigurationChangedAsync(object? sender, EventArgs e) @@ -174,11 +280,185 @@ private void CheckInitialized() } } + private static bool HasTrustedArtistIdentifier(ArtistSearchResult artist) + { + return artist.MusicBrainzId.HasValue || + artist.SpotifyId.Nullify() != null || + artist.ItunesId.Nullify() != null; + } + + private static bool HasMatchingTrustedArtistIdentifier(ArtistSearchResult current, ArtistSearchResult candidate) + { + return (current.MusicBrainzId.HasValue && current.MusicBrainzId == candidate.MusicBrainzId) || + (current.SpotifyId.Nullify() != null && current.SpotifyId == candidate.SpotifyId) || + (current.ItunesId.Nullify() != null && current.ItunesId == candidate.ItunesId); + } + + /// <summary> + /// Splits a likely compound release artist into fallback names for targeted provider lookup. + /// </summary> + public static string[] GetCompoundArtistFallbackNames(string artistName) + { + var normalizedName = artistName.Nullify(); + if (normalizedName == null) + { + return []; + } + + var parts = CompoundArtistSeparatorRegex + .Split(normalizedName) + .Select(x => x.Trim()) + .Where(x => x.Length > 1) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(4) + .ToArray(); + + return parts.Length > 1 ? parts : []; + } + + /// <summary> + /// Returns whether a compound artist fallback result is trusted enough to attach to the original album query. + /// </summary> + public static bool ShouldUseCompoundArtistFallbackResult(ArtistSearchResult candidate, ArtistQuery originalQuery) + { + if (!HasTrustedArtistIdentifier(candidate)) + { + return false; + } + + if (originalQuery.AlbumKeyValues is null || originalQuery.AlbumKeyValues.Length == 0) + { + return true; + } + + return (candidate.Releases ?? []).Any(release => originalQuery.AlbumKeyValues.Any(album => + { + if (!string.Equals(release.NameNormalized, album.Value, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return album.Key.Nullify() == null || + album.Key == "0" || + release.Year?.ToString(CultureInfo.InvariantCulture) == album.Key; + })); + } + + /// <summary> + /// Returns whether the search should run extra compound-artist fallback provider lookups. + /// </summary> + public static bool ShouldAttemptCompoundArtistFallback(ArtistQuery originalQuery, bool bypassNegativeCache) + { + if (bypassNegativeCache) + { + return false; + } + + if (originalQuery.AlbumKeyValues is null || originalQuery.AlbumKeyValues.Length == 0) + { + return false; + } + + return GetCompoundArtistFallbackNames(originalQuery.Name).Length > 0; + } + + /// <summary> + /// Returns whether an exception represents a transient DecentDB transaction conflict. + /// </summary> + public static bool IsDecentDbTransientTransactionConflict(Exception exception) + { + var message = exception.ToString(); + return message.Contains("transaction conflict", StringComparison.OrdinalIgnoreCase) || + message.Contains("DecentDB error 4", StringComparison.OrdinalIgnoreCase); + } + + /// <summary> + /// Returns whether an exception represents a non-retryable DecentDB open/read error. + /// </summary> + public static bool IsDecentDbCorruption(Exception exception) + { + var message = exception.ToString(); + return message.Contains("corrupt", StringComparison.OrdinalIgnoreCase) || + message.Contains("checksum", StringComparison.OrdinalIgnoreCase) || + message.Contains("database corruption", StringComparison.OrdinalIgnoreCase) || + message.Contains("DecentDB error 2", StringComparison.OrdinalIgnoreCase) || + message.Contains("DecentDB error 11", StringComparison.OrdinalIgnoreCase); + } + + private async Task SaveArtistSearchChangesAsync( + ArtistSearchEngineServiceDbContext scopedContext, + DirectoryRunContext? runContext, + string operationName, + CancellationToken cancellationToken) + { + for (var attempt = 1; attempt <= ArtistSearchPersistenceMaxAttempts; attempt++) + { + var shouldRetry = false; + await ArtistSearchPersistenceSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var artistsRequiringAliasSync = GetArtistsRequiringAliasSync(scopedContext); + await SyncLocalArtistAliasesAsync(scopedContext, artistsRequiringAliasSync, cancellationToken) + .ConfigureAwait(false); + await scopedContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await SyncLocalArtistAliasesAsync(scopedContext, artistsRequiringAliasSync, cancellationToken) + .ConfigureAwait(false); + if (scopedContext.ChangeTracker.HasChanges()) + { + await scopedContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + + return; + } + catch (Exception ex) when (ex is not OperationCanceledException && IsDecentDbCorruption(ex)) + { + runContext?.RecordArtistSearchPersistenceCorruption(); + Logger.Error(ex, + "[{Name}] Non-retryable DecentDB error while saving artist search data during [{Operation}]. Retry suppressed.", + nameof(ArtistSearchEngineService), + operationName); + throw; + } + catch (Exception ex) when (ex is not OperationCanceledException && IsDecentDbTransientTransactionConflict(ex)) + { + runContext?.RecordArtistSearchPersistenceConflict(); + if (attempt >= ArtistSearchPersistenceMaxAttempts) + { + Logger.Warning(ex, + "[{Name}] DecentDB transaction conflict while saving artist search data during [{Operation}] after [{Attempt}] attempts.", + nameof(ArtistSearchEngineService), + operationName, + attempt); + throw; + } + + runContext?.RecordArtistSearchPersistenceRetry(); + shouldRetry = true; + Logger.Warning(ex, + "[{Name}] DecentDB transaction conflict while saving artist search data during [{Operation}], retrying attempt [{Attempt}/{MaxAttempts}].", + nameof(ArtistSearchEngineService), + operationName, + attempt + 1, + ArtistSearchPersistenceMaxAttempts); + } + finally + { + ArtistSearchPersistenceSemaphore.Release(); + } + + if (shouldRetry) + { + var delay = TimeSpan.FromMilliseconds(50 * attempt + Random.Shared.Next(25, 100)); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + } + public async Task<PagedResult<Artist>> ListAsync( PagedRequest pagedRequest, CancellationToken cancellationToken = default) { - int totalCount; + var totalCount = 0; Artist[] artists = []; using (Operation.At(LogEventLevel.Debug) @@ -187,117 +467,16 @@ public async Task<PagedResult<Artist>> ListAsync( await using (var scopedContext = await artistSearchEngineServiceDbContextFactory .CreateDbContextAsync(cancellationToken).ConfigureAwait(false)) { - // Build the base query with filters - var query = scopedContext.Artists.AsNoTracking(); + var query = ApplyArtistListFilters(scopedContext.Artists.AsNoTracking(), pagedRequest); - // Apply filters from PagedRequest.FilterBy if any - if (pagedRequest.FilterBy?.Length > 0) + if (pagedRequest.IsTotalCountOnlyRequest) { - foreach (var filter in pagedRequest.FilterBy) - { - var filterValue = filter.Value?.ToString() ?? string.Empty; - var filterValueLower = filterValue.ToLowerInvariant(); - - // Apply filters based on property name and operator - query = filter.PropertyName.ToLower() switch - { - "name" => filter.OperatorValue.ToUpper() switch - { - "LIKE" => ApplyLikeFilter(query, x => x.Name, filter.Operator, filterValueLower), - "=" => query.Where(x => x.Name == filterValue), - "!=" => query.Where(x => x.Name != filterValue), - _ => query - }, - "namenormalized" => filter.OperatorValue.ToUpper() switch - { - "LIKE" => ApplyLikeFilter(query, x => x.NameNormalized, filter.Operator, - filterValueLower), - "=" => query.Where(x => x.NameNormalized == filterValue), - "!=" => query.Where(x => x.NameNormalized != filterValue), - _ => query - }, - "sortname" => filter.OperatorValue.ToUpper() switch - { - "LIKE" => ApplyLikeFilter(query, x => x.SortName, filter.Operator, filterValueLower), - "=" => query.Where(x => x.SortName == filterValue), - "!=" => query.Where(x => x.SortName != filterValue), - _ => query - }, - "musicbrainzid" => filter.OperatorValue.ToUpper() switch - { - "=" => query.Where(x => x.MusicBrainzId.ToString() == filterValue), - "!=" => query.Where(x => x.MusicBrainzId.ToString() != filterValue), - _ => query - }, - "spotifyid" => filter.OperatorValue.ToUpper() switch - { - "=" => query.Where(x => x.SpotifyId == filterValue), - "!=" => query.Where(x => x.SpotifyId != filterValue), - "LIKE" => ApplyLikeFilter(query, x => x.SpotifyId!, filter.Operator, filterValueLower), - _ => query - }, - _ => query - }; - } + totalCount = await query.CountAsync(cancellationToken).ConfigureAwait(false); } - - // Get total count - totalCount = await query.CountAsync(cancellationToken).ConfigureAwait(false); - - if (!pagedRequest.IsTotalCountOnlyRequest) + else { - // Apply ordering - if (pagedRequest.OrderBy?.Count > 0) - { - var firstOrderBy = pagedRequest.OrderBy.First(); - var isDescending = firstOrderBy.Value.ToUpper() == "DESC"; - - query = firstOrderBy.Key.ToLower() switch - { - "id" => isDescending ? query.OrderByDescending(x => x.Id) : query.OrderBy(x => x.Id), - "name" => isDescending ? query.OrderByDescending(x => x.Name) : query.OrderBy(x => x.Name), - "namenormalized" => isDescending - ? query.OrderByDescending(x => x.NameNormalized) - : query.OrderBy(x => x.NameNormalized), - "sortname" => isDescending - ? query.OrderByDescending(x => x.SortName) - : query.OrderBy(x => x.SortName), - _ => isDescending ? query.OrderByDescending(x => x.Id) : query.OrderBy(x => x.Id) - }; - - // Apply additional ordering if present - foreach (var orderBy in pagedRequest.OrderBy.Skip(1)) - { - var isDesc = orderBy.Value.ToUpper() == "DESC"; - var orderedQuery = (IOrderedQueryable<Artist>)query; - - query = orderBy.Key.ToLower() switch - { - "id" => isDesc - ? orderedQuery.ThenByDescending(x => x.Id) - : orderedQuery.ThenBy(x => x.Id), - "name" => isDesc - ? orderedQuery.ThenByDescending(x => x.Name) - : orderedQuery.ThenBy(x => x.Name), - "namenormalized" => isDesc - ? orderedQuery.ThenByDescending(x => x.NameNormalized) - : orderedQuery.ThenBy(x => x.NameNormalized), - "sortname" => isDesc - ? orderedQuery.ThenByDescending(x => x.SortName) - : orderedQuery.ThenBy(x => x.SortName), - _ => orderedQuery - }; - } - } - else - { - query = query.OrderBy(x => x.Id); - } - - // Keep DecentDB from evaluating a correlated album-count subquery per artist row. - // Also avoid EF-generated LIMIT/OFFSET parameters because DecentDB rejects skipped - // parameter numbering when this projection has no WHERE parameters. - artists = (await query + var pageProbeSize = pagedRequest.TakeValue + 1; + var page = await ApplyArtistListOrdering(query, pagedRequest) .Select(x => new Artist { Id = x.Id, @@ -315,11 +494,16 @@ public async Task<PagedResult<Artist>> ListAsync( IsLocked = x.IsLocked, LastRefreshed = x.LastRefreshed }) - .ToArrayAsync(cancellationToken) - .ConfigureAwait(false)) .Skip(pagedRequest.SkipValue) - .Take(pagedRequest.TakeValue) - .ToArray(); + .Take(pageProbeSize) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + + var hasMoreRows = page.Length > pagedRequest.TakeValue; + artists = page.Take(pagedRequest.TakeValue).ToArray(); + totalCount = artists.Length == 0 && pagedRequest.SkipValue == 0 + ? 0 + : pagedRequest.SkipValue + artists.Length + (hasMoreRows ? 1 : 0); var artistIds = artists.Select(x => x.Id).ToArray(); if (artistIds.Length > 0) @@ -352,7 +536,7 @@ public async Task<PagedResult<SongSearchResult>> DoArtistTopSongsSearchAsync(str { CheckInitialized(); - var maxResultsValue = maxResults ?? _configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize); + var maxResultsValue = GetBoundedMaxResults(maxResults); var totalCount = 0; long operationTime = 0; @@ -446,7 +630,7 @@ public async Task<PagedResult<SongSearchResult>> DoArtistTopSongsSearchAsync(str } var startTicks = Stopwatch.GetTimestamp(); - var pluginResult = await plugin.DoArtistSearchAsync(query, int.MaxValue, cancellationToken).ConfigureAwait(false); + var pluginResult = await plugin.DoArtistSearchAsync(query, maxResultsValue, cancellationToken).ConfigureAwait(false); if (pluginResult is { IsSuccess: true, Data: not null }) { foreach (var d in pluginResult.Data) @@ -491,8 +675,7 @@ public async Task<PagedResult<SongSearchResult>> DoArtistTopSongsSearchAsync(str WikiDataId = pluginResult.WikiDataId }; var seenArtist = artistsFromSearchResult.FirstOrDefault(x => - x.MusicBrainzId == artistFromSearchResult.MusicBrainzId - || x.SpotifyId == artistFromSearchResult.SpotifyId); + HasMatchingTrustedArtistIdentifier(x, artistFromSearchResult)); if (seenArtist == null) { artistFromSearchResult = pluginsResult.Aggregate(artistFromSearchResult, (current, r) => @@ -550,7 +733,7 @@ current with .Where(x => x.AlbumType is AlbumType.Album or AlbumType.EP).ToArray(); artistsFromSearchResult.Add(artistFromSearchResult); - if (artistFromSearchResult.SpotifyId == null && artistFromSearchResult.MusicBrainzId == null) + if (!HasTrustedArtistIdentifier(artistFromSearchResult)) { Logger.Warning("[{Name}]:[{MethodName}] unable to find artist for provided query.", nameof(ArtistSearchEngineService), @@ -604,6 +787,47 @@ current with return null; } + private async Task<ArtistSearchResult?> GetArtistFromCompoundFallbackAsync( + ArtistQuery originalQuery, + int maxResultsValue, + CancellationToken cancellationToken) + { + var fallbackNames = GetCompoundArtistFallbackNames(originalQuery.Name); + if (fallbackNames.Length == 0) + { + return null; + } + + foreach (var fallbackName in fallbackNames) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + var fallbackQuery = originalQuery with + { + Name = fallbackName + }; + var candidate = await GetArtistFromSearchProviders(fallbackQuery, maxResultsValue, cancellationToken) + .ConfigureAwait(false); + if (candidate == null || !ShouldUseCompoundArtistFallbackResult(candidate, originalQuery)) + { + continue; + } + + candidate.Rank = Math.Max(1, candidate.Rank - 1); + Logger.Information( + "[{Name}] Using compound artist fallback [{FallbackArtist}] for original artist [{OriginalArtist}]", + nameof(ArtistSearchEngineService), + fallbackName, + originalQuery.Name); + return candidate; + } + + return null; + } + /// <summary> /// Performs artist search with directory-run caching and request coalescing. /// When a runContext is provided, uses the run-scoped cache to avoid duplicate API calls. @@ -613,19 +837,35 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync( int? maxResults, DirectoryRunContext? runContext, CancellationToken cancellationToken = default) + { + return await DoSearchAsync(query, maxResults, runContext, bypassNegativeCache: false, cancellationToken) + .ConfigureAwait(false); + } + + /// <summary> + /// Performs artist search with run-scoped caching and optional negative-cache bypass. + /// </summary> + public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync( + ArtistQuery query, + int? maxResults, + DirectoryRunContext? runContext, + bool bypassNegativeCache, + CancellationToken cancellationToken = default) { if (runContext == null) { - return await DoSearchAsync(query, maxResults, cancellationToken).ConfigureAwait(false); + return await DoSearchAsync(query, maxResults, bypassNegativeCache, cancellationToken).ConfigureAwait(false); } var startTicks = Stopwatch.GetTimestamp(); + var cache = bypassNegativeCache ? runContext.ForcedArtistSearchCache : runContext.ArtistSearchCache; - var (results, wasHit, wasCoalesced) = await runContext.ArtistSearchCache.GetOrCreateAsync( + var (results, wasHit, wasCoalesced) = await cache.GetOrCreateAsync( query, async (q, ct) => { - var searchResult = await DoSearchAsync(q, maxResults, ct).ConfigureAwait(false); + var searchResult = await DoSearchInternalAsync(q, maxResults, bypassNegativeCache, runContext, ct) + .ConfigureAwait(false); return searchResult.Data.ToArray(); }, cancellationToken).ConfigureAwait(false); @@ -634,9 +874,10 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync( runContext.AddEnrichmentTime((long)elapsedMs); Logger.Debug( - "[{Name}] Artist search for [{Artist}]: cacheHit={Hit}, coalesced={Coalesced}, duration={Duration}ms", + "[{Name}] Artist search for [{Artist}]: forced={Forced}, cacheHit={Hit}, coalesced={Coalesced}, duration={Duration}ms", nameof(ArtistSearchEngineService), query.Name, + bypassNegativeCache, wasHit, wasCoalesced, elapsedMs); @@ -651,6 +892,26 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync( public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync(ArtistQuery query, int? maxResults, CancellationToken cancellationToken = default) + { + return await DoSearchAsync(query, maxResults, bypassNegativeCache: false, cancellationToken).ConfigureAwait(false); + } + + public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync( + ArtistQuery query, + int? maxResults, + bool bypassNegativeCache, + CancellationToken cancellationToken = default) + { + return await DoSearchInternalAsync(query, maxResults, bypassNegativeCache, null, cancellationToken) + .ConfigureAwait(false); + } + + private async Task<PagedResult<ArtistSearchResult>> DoSearchInternalAsync( + ArtistQuery query, + int? maxResults, + bool bypassNegativeCache, + DirectoryRunContext? runContext, + CancellationToken cancellationToken = default) { CheckInitialized(); @@ -665,14 +926,20 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync(ArtistQuery que { if (!wasFound) { - Logger.Debug("[{Name}] Artist [{Artist}] not found (cached negative result)", - nameof(ArtistSearchEngineService), normalizedQuery.Name); - return new PagedResult<ArtistSearchResult> + if (!bypassNegativeCache) { - Data = [], - TotalCount = 0, - TotalPages = 0 - }; + Logger.Debug("[{Name}] Artist [{Artist}] not found (cached negative result)", + nameof(ArtistSearchEngineService), normalizedQuery.Name); + return new PagedResult<ArtistSearchResult> + { + Data = [], + TotalCount = 0, + TotalPages = 0 + }; + } + + Logger.Debug("[{Name}] Artist [{Artist}] cached negative result ignored for forced revalidation", + nameof(ArtistSearchEngineService), normalizedQuery.Name); } if (cachedArtistId.HasValue) @@ -685,7 +952,7 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync(ArtistQuery que var result = new List<ArtistSearchResult>(); - var maxResultsValue = maxResults ?? _configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize); + var maxResultsValue = GetBoundedMaxResults(maxResults); long operationTime = 0; var totalCount = 0; @@ -699,44 +966,9 @@ public async Task<PagedResult<ArtistSearchResult>> DoSearchAsync(ArtistQuery que await using (var scopedContext = await artistSearchEngineServiceDbContextFactory .CreateDbContextAsync(cancellationToken).ConfigureAwait(false)) { - var firstTag = $"{normalizedQuery.NameNormalized}{StringExtensions.TagsSeparator}"; - var inTag = - $"{StringExtensions.TagsSeparator}{normalizedQuery.NameNormalized}{StringExtensions.TagsSeparator}"; - var outerTag = $"{StringExtensions.TagsSeparator}{normalizedQuery.NameNormalized}"; - var artists = await scopedContext - .Artists.Include(x => x.Albums) - .Where(x => x.NameNormalized == normalizedQuery.NameNormalized || - (x.MusicBrainzId != null && normalizedQuery.MusicBrainzId != null && - x.MusicBrainzId == normalizedQuery.MusicBrainzIdValue) || - (x.AlternateNames != null && (x.AlternateNames.Contains(firstTag) || - x.AlternateNames.Contains(inTag) || - x.AlternateNames.Contains(outerTag))) || - (x.SpotifyId != null && normalizedQuery.SpotifyId != null && x.SpotifyId == normalizedQuery.SpotifyId)) - .ToArrayAsync(cancellationToken) + var artist = await FindLocalArtistForQueryAsync(scopedContext, normalizedQuery, cancellationToken) .ConfigureAwait(false); - if (artists.Length > 0 && normalizedQuery.AlbumKeyValues?.Length > 0) - { - foreach (var ar in artists) - { - // If any album is given then rank artist if any album matches - foreach (var album in ar.Albums) - { - foreach (var albumKey in normalizedQuery.AlbumKeyValues) - { - var isAlbumMatch = album.Year.ToString() == albumKey.Key && - album.NameNormalized == albumKey.Value; - if (isAlbumMatch) - { - ar.Rank++; - } - } - } - } - } - - var artist = artists.OrderByDescending(x => x.Rank).FirstOrDefault(); - if (artist != null) { if (artist.Albums.Count == 0) @@ -785,7 +1017,12 @@ await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellatio artist.Albums = albumsToAdd.ToArray(); try { - await scopedContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await SaveArtistSearchChangesAsync( + scopedContext, + runContext, + "refresh existing artist albums", + cancellationToken) + .ConfigureAwait(false); Logger.Debug("[{Name}] Updated existing artist [{Artist}] added [{Count}] albums.", nameof(ArtistSearchEngineService), artist, @@ -822,24 +1059,14 @@ await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellatio { var newArtist = await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellationToken) .ConfigureAwait(false); + if (newArtist == null && ShouldAttemptCompoundArtistFallback(normalizedQuery, bypassNegativeCache)) + { + newArtist = await GetArtistFromCompoundFallbackAsync(normalizedQuery, maxResultsValue, cancellationToken) + .ConfigureAwait(false); + } if (newArtist != null) { - var nameNormalized = newArtist.Name.ToNormalizedString() ?? newArtist.Name; - - artist = await scopedContext - .Artists - .Where(x => x.NameNormalized == nameNormalized || - (x.AlternateNames != null && (x.AlternateNames.Contains(firstTag) || - x.AlternateNames.Contains(inTag) || - x.AlternateNames.Contains(outerTag))) || - (x.AmgId != null && x.AmgId == newArtist.AmgId) || - (x.DiscogsId != null && x.DiscogsId == newArtist.DiscogsId) || - (x.ItunesId != null && x.ItunesId == newArtist.ItunesId) || - (x.LastFmId != null && x.LastFmId == newArtist.LastFmId) || - (x.MusicBrainzId != null && x.MusicBrainzId == newArtist.MusicBrainzId) || - (x.SpotifyId != null && x.SpotifyId == newArtist.SpotifyId) || - (x.WikiDataId != null && x.WikiDataId == newArtist.WikiDataId)) - .FirstOrDefaultAsync(cancellationToken) + artist = await FindLocalArtistForProviderResultAsync(scopedContext, newArtist, cancellationToken) .ConfigureAwait(false); if (artist != null) @@ -906,7 +1133,12 @@ await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellatio scopedContext.Artists.Add(newDbArtist); try { - await scopedContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await SaveArtistSearchChangesAsync( + scopedContext, + runContext, + "add artist search result", + cancellationToken) + .ConfigureAwait(false); } catch (DbUpdateException ex) when (IsAlbumUniqueConstraint(ex)) { @@ -929,7 +1161,12 @@ await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellatio else { scopedContext.Artists.Add(newDbArtist); - await scopedContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await SaveArtistSearchChangesAsync( + scopedContext, + runContext, + "retry add artist search result after duplicate album", + cancellationToken) + .ConfigureAwait(false); } } newArtist = newArtist with @@ -950,6 +1187,12 @@ await GetArtistFromSearchProviders(normalizedQuery, maxResultsValue, cancellatio } catch (Exception e) { + runContext?.RecordArtistSearchReadError(); + if (IsDecentDbCorruption(e)) + { + runContext?.RecordArtistSearchReadCorruption(); + } + Logger.Error(e, "Attempting to Search [{Artist}]", normalizedQuery.Name); } @@ -1301,7 +1544,7 @@ public async Task<ArtistLookupResult> LookupAsync( var startTicks = Stopwatch.GetTimestamp(); var pluginResult = await plugin.DoArtistSearchAsync( normalizedQuery, - maxResults ?? _configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize), + GetBoundedMaxResults(maxResults), cancellationToken).ConfigureAwait(false); totalOperationTime += Stopwatch.GetElapsedTime(startTicks).Milliseconds; @@ -1337,7 +1580,7 @@ public async Task<ArtistLookupResult> LookupAsync( return new ArtistLookupResult { Candidates = uniqueCandidates.Values - .Take(maxResults ?? _configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize)) + .Take(GetBoundedMaxResults(maxResults)) .ToArray(), HasPartialFailures = failedProviders.Count > 0, FailedProviderIds = failedProviders.Select(x => x.ProviderId).ToArray(), @@ -1353,6 +1596,16 @@ private static string GetCandidateKey(ArtistSearchResult result) result.Name.ToNormalizedString() ?? string.Empty); } + private int GetBoundedMaxResults(int? maxResults) + { + var defaultPageSize = _configuration.GetValue<int>(SettingRegistry.SearchEngineDefaultPageSize); + var maximumAllowedPageSize = _configuration.GetValue<int>(SettingRegistry.SearchEngineMaximumAllowedPageSize); + var upperBound = Math.Max(1, maximumAllowedPageSize > 0 ? maximumAllowedPageSize : defaultPageSize); + var requestedPageSize = maxResults ?? defaultPageSize; + + return Math.Clamp(requestedPageSize, 1, upperBound); + } + private static bool IsAlbumUniqueConstraint(DbUpdateException ex) { var message = ex.InnerException?.Message ?? ex.Message; @@ -1360,6 +1613,464 @@ private static bool IsAlbumUniqueConstraint(DbUpdateException ex) message.Contains("Albums", StringComparison.OrdinalIgnoreCase); } + private static Artist[] GetArtistsRequiringAliasSync(ArtistSearchEngineServiceDbContext context) + { + return context.ChangeTracker + .Entries<Artist>() + .Where(x => x.State is EntityState.Added or EntityState.Modified) + .Select(x => x.Entity) + .Distinct() + .ToArray(); + } + + private static async Task SyncLocalArtistAliasesAsync( + ArtistSearchEngineServiceDbContext context, + Artist[] artists, + CancellationToken cancellationToken) + { + foreach (var artist in artists.Where(x => x.Id > 0).DistinctBy(x => x.Id)) + { + var normalizedAliases = GetNormalizedLocalArtistAliases(artist); + var existingAliases = await context.ArtistAliases + .Where(x => x.ArtistId == artist.Id) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + + foreach (var existingAlias in existingAliases + .Where(x => !normalizedAliases.Contains(x.NameNormalized, StringComparer.OrdinalIgnoreCase))) + { + context.ArtistAliases.Remove(existingAlias); + } + + var existingAliasValues = existingAliases + .Select(x => x.NameNormalized) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var alias in normalizedAliases.Where(x => !existingAliasValues.Contains(x))) + { + context.ArtistAliases.Add(new ArtistAlias + { + ArtistId = artist.Id, + NameNormalized = alias + }); + } + } + } + + private static string[] GetNormalizedLocalArtistAliases(Artist artist) + { + return GetNormalizedLocalArtistAliases(artist.AlternateNames, artist.NameNormalized); + } + + private static string[] GetNormalizedLocalArtistAliases(string? alternateNames, string? primaryNameNormalized) + { + return (alternateNames.ToTags() ?? []) + .Select(x => x.ToNormalizedString() ?? x) + .Where(x => x.Nullify() is not null && + !string.Equals(x, primaryNameNormalized, StringComparison.OrdinalIgnoreCase)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IQueryable<Artist> ApplyArtistListFilters(IQueryable<Artist> query, PagedRequest pagedRequest) + { + if (pagedRequest.FilterBy?.Length > 0) + { + foreach (var filter in pagedRequest.FilterBy) + { + var filterValue = filter.Value?.ToString() ?? string.Empty; + var filterValueLower = filterValue.ToLowerInvariant(); + + query = filter.PropertyName.ToLowerInvariant() switch + { + "name" => filter.OperatorValue.ToUpperInvariant() switch + { + "LIKE" => ApplyLikeFilter(query, x => x.Name, filter.Operator, filterValueLower), + "=" => query.Where(x => x.Name == filterValue), + "!=" => query.Where(x => x.Name != filterValue), + _ => query + }, + "namenormalized" => filter.OperatorValue.ToUpperInvariant() switch + { + "LIKE" => ApplyLikeFilter(query, x => x.NameNormalized, filter.Operator, filterValueLower), + "=" => query.Where(x => x.NameNormalized == filterValue), + "!=" => query.Where(x => x.NameNormalized != filterValue), + _ => query + }, + "sortname" => filter.OperatorValue.ToUpperInvariant() switch + { + "LIKE" => ApplyLikeFilter(query, x => x.SortName, filter.Operator, filterValueLower), + "=" => query.Where(x => x.SortName == filterValue), + "!=" => query.Where(x => x.SortName != filterValue), + _ => query + }, + "musicbrainzid" => filter.OperatorValue.ToUpperInvariant() switch + { + "=" => query.Where(x => x.MusicBrainzId.ToString() == filterValue), + "!=" => query.Where(x => x.MusicBrainzId.ToString() != filterValue), + _ => query + }, + "spotifyid" => filter.OperatorValue.ToUpperInvariant() switch + { + "=" => query.Where(x => x.SpotifyId == filterValue), + "!=" => query.Where(x => x.SpotifyId != filterValue), + "LIKE" => ApplyLikeFilter(query, x => x.SpotifyId!, filter.Operator, filterValueLower), + _ => query + }, + _ => query + }; + } + } + + return query; + } + + private static IOrderedQueryable<Artist> ApplyArtistListOrdering(IQueryable<Artist> query, PagedRequest pagedRequest) + { + if (pagedRequest.OrderBy?.Count > 0) + { + var firstOrderBy = pagedRequest.OrderBy.First(); + var isDescending = firstOrderBy.Value.ToUpperInvariant() == PagedRequest.OrderDescDirection; + + var orderedQuery = firstOrderBy.Key.ToLowerInvariant() switch + { + "id" => isDescending ? query.OrderByDescending(x => x.Id) : query.OrderBy(x => x.Id), + "name" => isDescending ? query.OrderByDescending(x => x.Name) : query.OrderBy(x => x.Name), + "namenormalized" => isDescending + ? query.OrderByDescending(x => x.NameNormalized) + : query.OrderBy(x => x.NameNormalized), + "sortname" => isDescending + ? query.OrderByDescending(x => x.SortName) + : query.OrderBy(x => x.SortName), + _ => isDescending ? query.OrderByDescending(x => x.Id) : query.OrderBy(x => x.Id) + }; + + foreach (var orderBy in pagedRequest.OrderBy.Skip(1)) + { + var isDesc = orderBy.Value.ToUpperInvariant() == PagedRequest.OrderDescDirection; + + orderedQuery = orderBy.Key.ToLowerInvariant() switch + { + "id" => isDesc + ? orderedQuery.ThenByDescending(x => x.Id) + : orderedQuery.ThenBy(x => x.Id), + "name" => isDesc + ? orderedQuery.ThenByDescending(x => x.Name) + : orderedQuery.ThenBy(x => x.Name), + "namenormalized" => isDesc + ? orderedQuery.ThenByDescending(x => x.NameNormalized) + : orderedQuery.ThenBy(x => x.NameNormalized), + "sortname" => isDesc + ? orderedQuery.ThenByDescending(x => x.SortName) + : orderedQuery.ThenBy(x => x.SortName), + _ => orderedQuery + }; + } + + return orderedQuery; + } + + return query.OrderBy(x => x.Id); + } + + private static async Task<Artist?> FindLocalArtistForQueryAsync( + ArtistSearchEngineServiceDbContext context, + ArtistQuery query, + CancellationToken cancellationToken) + { + if (query.MusicBrainzIdValue is { } musicBrainzId) + { + var artist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.MusicBrainzId == musicBrainzId), + query, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (artist is not null) + { + return artist; + } + } + + var spotifyId = query.SpotifyId.Nullify(); + if (spotifyId is not null) + { + var artist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.SpotifyId == spotifyId), + query, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (artist is not null) + { + return artist; + } + } + + var nameNormalized = query.NameNormalized.Nullify(); + if (nameNormalized is null) + { + return null; + } + + var artistByName = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.NameNormalized == nameNormalized), + query, + LocalArtistCandidateLimit, + cancellationToken) + .ConfigureAwait(false); + if (artistByName is not null) + { + return artistByName; + } + + var aliasArtistIds = await context.ArtistAliases + .AsNoTracking() + .Where(alias => alias.NameNormalized == nameNormalized) + .OrderBy(alias => alias.ArtistId) + .Select(alias => alias.ArtistId) + .Take(LocalArtistCandidateLimit) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + if (aliasArtistIds.Length == 0) + { + return null; + } + + return await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => aliasArtistIds.Contains(x.Id)), + query, + LocalArtistCandidateLimit, + cancellationToken) + .ConfigureAwait(false); + } + + private static async Task<Artist?> FindLocalArtistForProviderResultAsync( + ArtistSearchEngineServiceDbContext context, + ArtistSearchResult artist, + CancellationToken cancellationToken) + { + if (artist.MusicBrainzId is { } musicBrainzId) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.MusicBrainzId == musicBrainzId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var spotifyId = artist.SpotifyId.Nullify(); + if (spotifyId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.SpotifyId == spotifyId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var itunesId = artist.ItunesId.Nullify(); + if (itunesId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.ItunesId == itunesId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var amgId = artist.AmgId.Nullify(); + if (amgId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.AmgId == amgId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var discogsId = artist.DiscogsId.Nullify(); + if (discogsId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.DiscogsId == discogsId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var lastFmId = artist.LastFmId.Nullify(); + if (lastFmId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.LastFmId == lastFmId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var wikiDataId = artist.WikiDataId.Nullify(); + if (wikiDataId is not null) + { + var localArtist = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.WikiDataId == wikiDataId), + albumRankingQuery: null, + limit: 1, + cancellationToken) + .ConfigureAwait(false); + if (localArtist is not null) + { + return localArtist; + } + } + + var nameNormalized = (artist.Name.ToNormalizedString() ?? artist.Name).Nullify(); + if (nameNormalized is null) + { + return null; + } + + var localArtistByName = await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => x.NameNormalized == nameNormalized), + albumRankingQuery: null, + LocalArtistCandidateLimit, + cancellationToken) + .ConfigureAwait(false); + if (localArtistByName is not null) + { + return localArtistByName; + } + + var aliasTerms = artist.AlternateNames is { Length: > 0 } + ? artist.AlternateNames + .Select(alias => alias.ToNormalizedString()?.Nullify()) + .Append(nameNormalized) + .OfType<string>() + .Distinct(StringComparer.Ordinal) + .Take(LocalArtistCandidateLimit) + .ToArray() + : [nameNormalized]; + + var aliasArtistIds = await context.ArtistAliases + .AsNoTracking() + .Where(alias => aliasTerms.Contains(alias.NameNormalized)) + .OrderBy(alias => alias.ArtistId) + .Select(alias => alias.ArtistId) + .Take(LocalArtistCandidateLimit) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + if (aliasArtistIds.Length == 0) + { + return null; + } + + return await FindBestLocalArtistAsync( + context, + context.Artists.Where(x => aliasArtistIds.Contains(x.Id)), + albumRankingQuery: null, + LocalArtistCandidateLimit, + cancellationToken) + .ConfigureAwait(false); + } + + private static async Task<Artist?> FindBestLocalArtistAsync( + ArtistSearchEngineServiceDbContext context, + IQueryable<Artist> candidateQuery, + ArtistQuery? albumRankingQuery, + int limit, + CancellationToken cancellationToken) + { + var candidateIds = await candidateQuery + .OrderBy(x => x.Id) + .Select(x => x.Id) + .Take(limit) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + if (candidateIds.Length == 0) + { + return null; + } + + var artists = await context.Artists + .Include(x => x.Albums) + .Where(x => candidateIds.Contains(x.Id)) + .ToArrayAsync(cancellationToken) + .ConfigureAwait(false); + + RankLocalArtistsByAlbumEvidence(artists, albumRankingQuery); + + return artists + .OrderByDescending(x => x.Rank) + .ThenBy(x => x.SortName) + .ThenBy(x => x.Id) + .FirstOrDefault(); + } + + private static void RankLocalArtistsByAlbumEvidence(Artist[] artists, ArtistQuery? query) + { + if (artists.Length == 0 || query?.AlbumKeyValues is not { Length: > 0 }) + { + return; + } + + foreach (var artist in artists) + { + foreach (var album in artist.Albums) + { + foreach (var albumKey in query.AlbumKeyValues) + { + var isAlbumMatch = album.Year.ToString(CultureInfo.InvariantCulture) == albumKey.Key && + album.NameNormalized == albumKey.Value; + if (isAlbumMatch) + { + artist.Rank++; + } + } + } + } + } + private static IQueryable<Artist> ApplyLikeFilter( IQueryable<Artist> query, System.Linq.Expressions.Expression<Func<Artist, string?>> propertySelector, diff --git a/src/Melodee.Mql/Melodee.Mql.csproj b/src/Melodee.Mql/Melodee.Mql.csproj index b9ae12a18..b2897177e 100644 --- a/src/Melodee.Mql/Melodee.Mql.csproj +++ b/src/Melodee.Mql/Melodee.Mql.csproj @@ -7,7 +7,7 @@ <GenerateAssemblyInfo>true</GenerateAssemblyInfo> <NoWarn>$(NoWarn);NU1507</NoWarn> - <VersionPrefix>2.1.0</VersionPrefix> + <VersionPrefix>2.1.3</VersionPrefix> <VersionSuffix>build$([System.DateTime]::UtcNow.ToString("yyyyMMddHHmmss"))</VersionSuffix> <AssemblyVersion>$(VersionPrefix).0</AssemblyVersion> <FileVersion>$(VersionPrefix).0</FileVersion> diff --git a/tests/Melodee.Tests.Blazor/Components/DashboardHealthWarningEvaluatorTests.cs b/tests/Melodee.Tests.Blazor/Components/DashboardHealthWarningEvaluatorTests.cs index f67721dae..badff8496 100644 --- a/tests/Melodee.Tests.Blazor/Components/DashboardHealthWarningEvaluatorTests.cs +++ b/tests/Melodee.Tests.Blazor/Components/DashboardHealthWarningEvaluatorTests.cs @@ -3,6 +3,8 @@ using Melodee.Blazor.Components.Pages; using Melodee.Common.Constants; using Moq; +using BlazorDoctorService = Melodee.Blazor.Services.IDoctorService; +using DoctorCheckResult = Melodee.Common.Services.Doctor.DoctorCheckResult; namespace Melodee.Tests.Blazor.Components; @@ -11,7 +13,7 @@ public class DashboardHealthWarningEvaluatorTests [Fact] public async Task ShouldShowAsync_WhenUserIsAdminAndDoctorNeedsAttention_ReturnsTrue() { - var doctorService = new Mock<IDoctorService>(); + var doctorService = new Mock<BlazorDoctorService>(); doctorService.Setup(service => service.NeedsAttentionAsync(It.IsAny<CancellationToken>())) .ReturnsAsync(true); @@ -24,7 +26,7 @@ public async Task ShouldShowAsync_WhenUserIsAdminAndDoctorNeedsAttention_Returns [Fact] public async Task ShouldShowAsync_WhenUserIsAdminAndDoctorIsHealthy_ReturnsFalse() { - var doctorService = new Mock<IDoctorService>(); + var doctorService = new Mock<BlazorDoctorService>(); doctorService.Setup(service => service.NeedsAttentionAsync(It.IsAny<CancellationToken>())) .ReturnsAsync(false); @@ -37,7 +39,7 @@ public async Task ShouldShowAsync_WhenUserIsAdminAndDoctorIsHealthy_ReturnsFalse [Fact] public async Task ShouldShowAsync_WhenUserIsNotAdmin_DoesNotCallDoctor() { - var doctorService = new Mock<IDoctorService>(MockBehavior.Strict); + var doctorService = new Mock<BlazorDoctorService>(MockBehavior.Strict); var result = await DashboardHealthWarningEvaluator.ShouldShowAsync(CreateUser(isAdmin: false), doctorService.Object); @@ -45,6 +47,64 @@ public async Task ShouldShowAsync_WhenUserIsNotAdmin_DoesNotCallDoctor() doctorService.Verify(service => service.NeedsAttentionAsync(It.IsAny<CancellationToken>()), Times.Never); } + [Fact] + public async Task GetIssuesAsync_WhenUserIsAdmin_ReturnsAttentionChecks() + { + var expected = new[] + { + new DoctorCheckResult("MusicBrainzDatabase", false, "unsupported DecentDB file format version 11", TimeSpan.Zero) + }; + var doctorService = new Mock<BlazorDoctorService>(); + doctorService.Setup(service => service.GetAttentionChecksAsync(It.IsAny<CancellationToken>())) + .ReturnsAsync(expected); + + var result = await DashboardHealthWarningEvaluator.GetIssuesAsync(CreateUser(isAdmin: true), doctorService.Object); + + result.Should().BeEquivalentTo(expected); + doctorService.Verify(service => service.GetAttentionChecksAsync(It.IsAny<CancellationToken>()), Times.Once); + } + + [Fact] + public async Task GetIssuesAsync_WhenUserIsNotAdmin_DoesNotCallDoctor() + { + var doctorService = new Mock<BlazorDoctorService>(MockBehavior.Strict); + + var result = await DashboardHealthWarningEvaluator.GetIssuesAsync(CreateUser(isAdmin: false), doctorService.Object); + + result.Should().BeEmpty(); + doctorService.Verify(service => service.GetAttentionChecksAsync(It.IsAny<CancellationToken>()), Times.Never); + } + + [Fact] + public void HasUnsupportedDecentDbIssue_WhenMusicBrainzUnsupportedFormat_ReturnsTrue() + { + var issues = new[] + { + new DoctorCheckResult( + "MusicBrainzDatabase", + false, + "MusicBrainz DecentDB database uses a file format that is not supported by the current DecentDB provider.", + TimeSpan.Zero) + }; + + var result = DashboardHealthWarningEvaluator.HasUnsupportedDecentDbIssue(issues); + + result.Should().BeTrue(); + } + + [Fact] + public void HasUnsupportedDecentDbIssue_WhenDifferentDoctorIssue_ReturnsFalse() + { + var issues = new[] + { + new DoctorCheckResult("PostgresDatabase", false, "Unable to connect to the primary database", TimeSpan.Zero) + }; + + var result = DashboardHealthWarningEvaluator.HasUnsupportedDecentDbIssue(issues); + + result.Should().BeFalse(); + } + private static ClaimsPrincipal CreateUser(bool isAdmin) { var claims = new List<Claim> { new(ClaimTypes.NameIdentifier, "1") }; diff --git a/tests/Melodee.Tests.Blazor/Services/DoctorServiceTests.cs b/tests/Melodee.Tests.Blazor/Services/DoctorServiceTests.cs index 34b0a580d..9ce653221 100644 --- a/tests/Melodee.Tests.Blazor/Services/DoctorServiceTests.cs +++ b/tests/Melodee.Tests.Blazor/Services/DoctorServiceTests.cs @@ -137,6 +137,92 @@ public async Task NeedsAttentionAsync_WhenArtistSearchFileMissing_ReturnsTrue() } } + [Fact] + public async Task GetAttentionChecksAsync_WhenMusicBrainzDecentDbFormatUnsupported_ReturnsVisibleIssue() + { + ConfigurePrimaryDatabaseCanConnect(); + + var musicBrainzPath = CreateNonEmptyFile("musicbrainz-unsupported-attention"); + var artistSearchPath = Path.Combine(Path.GetTempPath(), $"artist-search-compatible-{Guid.NewGuid():N}.ddb"); + + try + { + var artistSearchOptions = CreateArtistSearchOptions(artistSearchPath); + await using (var artistSearchContext = new ArtistSearchEngineServiceDbContext(artistSearchOptions)) + { + await artistSearchContext.Database.EnsureCreatedAsync(); + } + ConfigureArtistSearchDatabase(artistSearchOptions); + + _musicBrainzDbContextFactory + .Setup(x => x.CreateDbContextAsync(It.IsAny<CancellationToken>())) + .ThrowsAsync(new InvalidOperationException("unsupported DecentDB file format version 11")); + + var configuration = CreateConfiguration(new Dictionary<string, string?> + { + ["ConnectionStrings:DefaultConnection"] = "Host=localhost;Database=test", + ["ConnectionStrings:MusicBrainzConnection"] = $"Data Source={musicBrainzPath}", + ["ConnectionStrings:ArtistSearchEngineConnection"] = $"Data Source={artistSearchPath}" + }); + var service = CreateService(configuration); + + var results = await service.GetAttentionChecksAsync(); + + var musicBrainzCheck = Assert.Single(results, x => x.Name == "MusicBrainzDatabase"); + Assert.False(musicBrainzCheck.Success); + Assert.Contains("not supported by the current DecentDB provider", musicBrainzCheck.Details); + Assert.Contains("unsupported DecentDB file format version 11", musicBrainzCheck.Details); + } + finally + { + DeleteDatabaseArtifacts(musicBrainzPath); + DeleteDatabaseArtifacts(artistSearchPath); + } + } + + [Fact] + public async Task GetAttentionChecksAsync_WhenArtistSearchDecentDbFormatUnsupported_ReturnsVisibleIssue() + { + ConfigurePrimaryDatabaseCanConnect(); + + var musicBrainzPath = Path.Combine(Path.GetTempPath(), $"musicbrainz-compatible-{Guid.NewGuid():N}.ddb"); + var artistSearchPath = CreateNonEmptyFile("artist-search-unsupported-attention"); + + try + { + var musicBrainzOptions = CreateMusicBrainzOptions(musicBrainzPath); + await using (var musicBrainzContext = new MusicBrainzDbContext(musicBrainzOptions)) + { + await musicBrainzContext.Database.EnsureCreatedAsync(); + } + ConfigureMusicBrainzDatabase(musicBrainzOptions); + + _artistSearchEngineDbContextFactory + .Setup(x => x.CreateDbContextAsync(It.IsAny<CancellationToken>())) + .ThrowsAsync(new InvalidOperationException("unsupported DecentDB file format version 11")); + + var configuration = CreateConfiguration(new Dictionary<string, string?> + { + ["ConnectionStrings:DefaultConnection"] = "Host=localhost;Database=test", + ["ConnectionStrings:MusicBrainzConnection"] = $"Data Source={musicBrainzPath}", + ["ConnectionStrings:ArtistSearchEngineConnection"] = $"Data Source={artistSearchPath}" + }); + var service = CreateService(configuration); + + var results = await service.GetAttentionChecksAsync(); + + var artistSearchCheck = Assert.Single(results, x => x.Name == "ArtistSearchEngineDatabase"); + Assert.False(artistSearchCheck.Success); + Assert.Contains("not supported by the current DecentDB provider", artistSearchCheck.Details); + Assert.Contains("unsupported DecentDB file format version 11", artistSearchCheck.Details); + } + finally + { + DeleteDatabaseArtifacts(musicBrainzPath); + DeleteDatabaseArtifacts(artistSearchPath); + } + } + [Fact] public async Task IsMusicBrainzDatabaseEmptyAsync_NoConnectionString_ReturnsTrue() { diff --git a/tests/Melodee.Tests.Blazor/Services/MainLayoutProxyServiceTests.cs b/tests/Melodee.Tests.Blazor/Services/MainLayoutProxyServiceTests.cs new file mode 100644 index 000000000..8138afaaa --- /dev/null +++ b/tests/Melodee.Tests.Blazor/Services/MainLayoutProxyServiceTests.cs @@ -0,0 +1,47 @@ +using FluentAssertions; + +namespace Melodee.Tests.Blazor.Services; + +public class MainLayoutProxyServiceTests +{ + [Fact] + public void SetSpinnerVisible_WhenStateChanges_RaisesSpinnerVisibleChanged() + { + var service = new MainLayoutProxyService(); + var eventCount = 0; + service.SpinnerVisibleChanged += (_, _) => eventCount++; + + service.SetSpinnerVisible(true); + + service.ShowSpinner.Should().BeTrue(); + eventCount.Should().Be(1); + } + + [Fact] + public void SetSpinnerVisible_WhenStateDoesNotChange_DoesNotRaiseSpinnerVisibleChanged() + { + var service = new MainLayoutProxyService(); + var eventCount = 0; + service.SetSpinnerVisible(true); + service.SpinnerVisibleChanged += (_, _) => eventCount++; + + service.SetSpinnerVisible(true); + + service.ShowSpinner.Should().BeTrue(); + eventCount.Should().Be(0); + } + + [Fact] + public void ToggleSpinnerVisible_WithForceState_UsesSetStateSemantics() + { + var service = new MainLayoutProxyService(); + var eventCount = 0; + service.SpinnerVisibleChanged += (_, _) => eventCount++; + + service.ToggleSpinnerVisible(true); + service.ToggleSpinnerVisible(false); + + service.ShowSpinner.Should().BeFalse(); + eventCount.Should().Be(2); + } +} diff --git a/tests/Melodee.Tests.Cli/Commands/DoctorCommandDecentDbProbeTests.cs b/tests/Melodee.Tests.Cli/Commands/DoctorCommandDecentDbProbeTests.cs new file mode 100644 index 000000000..0aecc306e --- /dev/null +++ b/tests/Melodee.Tests.Cli/Commands/DoctorCommandDecentDbProbeTests.cs @@ -0,0 +1,148 @@ +using FluentAssertions; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using MusicBrainzAlbum = Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data.Models.Materialized.Album; +using MusicBrainzArtist = Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data.Models.Materialized.Artist; + +namespace Melodee.Tests.Cli.Commands; + +public class DoctorCommandDecentDbProbeTests +{ + [Fact] + public async Task ProbeDecentDbDatabaseAsync_WithMissingFile_ReturnsFailure() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"missing-musicbrainz-{Guid.NewGuid():N}.ddb"); + + var result = await CliDoctorService.ProbeDecentDbDatabaseAsync( + $"Data Source={databasePath}", + ["Artist"], + [], + requireRowsForReadQueries: true); + + result.Success.Should().BeFalse(); + result.Details.Should().Contain("does not exist"); + } + + [Fact] + public async Task ProbeDecentDbDatabaseAsync_WithEmptyFile_ReturnsFailure() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"empty-musicbrainz-{Guid.NewGuid():N}.ddb"); + + try + { + await File.WriteAllTextAsync(databasePath, string.Empty); + + var result = await CliDoctorService.ProbeDecentDbDatabaseAsync( + $"Data Source={databasePath}", + ["Artist"], + [], + requireRowsForReadQueries: true); + + result.Success.Should().BeFalse(); + result.Details.Should().Contain("empty"); + } + finally + { + DeleteDatabaseArtifacts(databasePath); + } + } + + [Fact] + public async Task ProbeDecentDbDatabaseAsync_WithMusicBrainzSchemaAndRows_ReturnsSuccess() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"musicbrainz-doctor-ok-{Guid.NewGuid():N}.ddb"); + + try + { + var options = new DbContextOptionsBuilder<MusicBrainzDbContext>() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + await using (var db = new MusicBrainzDbContext(options)) + { + await db.Database.EnsureCreatedAsync(); + await db.Artists.AddAsync(new MusicBrainzArtist + { + MusicBrainzArtistId = 14, + Name = "The Blackbelt Band", + SortName = "Blackbelt Band, The", + NameNormalized = "THEBLACKBELTBAND", + MusicBrainzIdRaw = Guid.NewGuid().ToString() + }); + await db.Albums.AddAsync(new MusicBrainzAlbum + { + MusicBrainzArtistId = 14, + Name = "Test Album", + SortName = "Test Album", + NameNormalized = "TESTALBUM", + ReleaseType = 1, + MusicBrainzIdRaw = Guid.NewGuid().ToString(), + ReleaseGroupMusicBrainzIdRaw = Guid.NewGuid().ToString(), + ReleaseDate = new DateTime(2026, 1, 1) + }); + await db.SaveChangesAsync(); + } + + var result = await CliDoctorService.ProbeDecentDbDatabaseAsync( + $"Data Source={databasePath}", + ["Artist", "Album", "ArtistAlias"], + [ + """SELECT "Id" FROM "Artist" LIMIT 1""", + """SELECT "Id" FROM "Album" LIMIT 1""" + ], + requireRowsForReadQueries: true); + + result.Success.Should().BeTrue(); + result.Details.Should().Contain("opened"); + result.Details.Should().Contain("readQueries=2"); + } + finally + { + DeleteDatabaseArtifacts(databasePath); + } + } + + [Fact] + public async Task ProbeDecentDbDatabaseAsync_WithMusicBrainzSchemaButNoRows_ReturnsFailure() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"musicbrainz-doctor-empty-{Guid.NewGuid():N}.ddb"); + + try + { + var options = new DbContextOptionsBuilder<MusicBrainzDbContext>() + .UseDecentDB($"Data Source={databasePath}") + .Options; + + await using (var db = new MusicBrainzDbContext(options)) + { + await db.Database.EnsureCreatedAsync(); + } + + var result = await CliDoctorService.ProbeDecentDbDatabaseAsync( + $"Data Source={databasePath}", + ["Artist", "Album", "ArtistAlias"], + [ + """SELECT "Id" FROM "Artist" LIMIT 1""", + """SELECT "Id" FROM "Album" LIMIT 1""" + ], + requireRowsForReadQueries: true); + + result.Success.Should().BeFalse(); + result.Details.Should().Contain("returned no rows"); + } + finally + { + DeleteDatabaseArtifacts(databasePath); + } + } + + private static void DeleteDatabaseArtifacts(string databasePath) + { + foreach (var path in new[] { databasePath, $"{databasePath}.wal", $"{databasePath}-wal", $"{databasePath}-shm" }) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} diff --git a/tests/Melodee.Tests.Common/Extensions/AlbumExtensionTests.cs b/tests/Melodee.Tests.Common/Extensions/AlbumExtensionTests.cs index 5090267ef..a5c6cd543 100644 --- a/tests/Melodee.Tests.Common/Extensions/AlbumExtensionTests.cs +++ b/tests/Melodee.Tests.Common/Extensions/AlbumExtensionTests.cs @@ -91,6 +91,57 @@ public static Album NewAlbum() }; } + [Fact] + public void RenumberImages_WhenCopiedImageHasDifferentOriginalName_KeepsStagedImage() + { + var albumDirectory = Path.Combine(Path.GetTempPath(), $"melodee-renumber-{Guid.NewGuid():N}"); + Directory.CreateDirectory(albumDirectory); + + try + { + var stagedImagePath = Path.Combine(albumDirectory, "i-01-Front.jpg"); + File.WriteAllText(stagedImagePath, "image"); + + var album = NewAlbum(); + album.Directory = new FileSystemDirectoryInfo + { + Path = albumDirectory, + Name = Path.GetFileName(albumDirectory) + }; + album.Images = + [ + new ImageInfo + { + CrcHash = "abc123", + FileInfo = new FileSystemFileInfo + { + Name = "i-01-Front.jpg", + OriginalName = "cover.jpg", + Size = new FileInfo(stagedImagePath).Length + }, + PictureIdentifier = PictureIdentifier.Front, + Width = 1200, + Height = 1200, + SortOrder = 1 + } + ]; + + var result = album.RenumberImages(); + + Assert.True(result); + var image = Assert.Single(album.Images!); + Assert.Equal("i-01-Front.jpg", image.FileInfo!.Name); + Assert.NotNull(album.CoverImage()); + } + finally + { + if (Directory.Exists(albumDirectory)) + { + Directory.Delete(albumDirectory, true); + } + } + } + [Fact] public void NonEnglishTags_DoNotThrow_WhenCreatingDirectoryAndJsonNames() { diff --git a/tests/Melodee.Tests.Common/Extensions/ArtistExtensionTests.cs b/tests/Melodee.Tests.Common/Extensions/ArtistExtensionTests.cs index c254d922b..2a8c2907a 100644 --- a/tests/Melodee.Tests.Common/Extensions/ArtistExtensionTests.cs +++ b/tests/Melodee.Tests.Common/Extensions/ArtistExtensionTests.cs @@ -1,5 +1,7 @@ +using Melodee.Common.Extensions; using Melodee.Common.Models; using Melodee.Common.Models.Extensions; +using Melodee.Common.Utility; namespace Melodee.Tests.Common.Extensions; @@ -31,4 +33,48 @@ public void ValidateIsCastRecordSongArtists(string input, bool shouldBe) { Assert.Equal(shouldBe, Artist.NewArtistFromName(input).IsCastRecording()); } + + [Fact] + public void IsValid_WithItunesId_ReturnsTrue() + { + var artistName = "iTunes Artist"; + var artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName) + { + ItunesId = "123456789" + }; + + Assert.True(artist.IsValid()); + } + + [Fact] + public void IsValid_WithNameButNoTrustedIdentifier_ReturnsFalse() + { + var artistName = "Unknown Artist"; + var artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName); + + Assert.False(artist.IsValid()); + } + + [Fact] + public void ToDirectoryName_WithItunesIdOnly_ReturnsDirectoryName() + { + var artistName = "iTunes Artist"; + var artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName) + { + ItunesId = "123456789" + }; + + var directoryName = artist.ToDirectoryName(255); + + Assert.Contains(SafeParser.Hash(artist.ItunesId).ToString(), directoryName); + } } diff --git a/tests/Melodee.Tests.Common/Jobs/MelodeeJobExecutionContextTests.cs b/tests/Melodee.Tests.Common/Jobs/MelodeeJobExecutionContextTests.cs index 4a6199c18..4868e9bca 100644 --- a/tests/Melodee.Tests.Common/Jobs/MelodeeJobExecutionContextTests.cs +++ b/tests/Melodee.Tests.Common/Jobs/MelodeeJobExecutionContextTests.cs @@ -47,6 +47,26 @@ public void Put_WithExistingKey_UpdatesValue() Assert.Equal(newValue, context.Get(key)); } + [Fact] + public void Constructor_MergedJobDataMap_IsNotNull() + { + var context = new MelodeeJobExecutionContext(CancellationToken.None); + + Assert.NotNull(context.MergedJobDataMap); + } + + [Fact] + public void Put_WithStringKey_MirrorsValueToMergedJobDataMap() + { + var context = new MelodeeJobExecutionContext(CancellationToken.None); + const string key = MelodeeJobExecutionContext.ChainOnComplete; + + context.Put(key, true); + + Assert.True(context.MergedJobDataMap.ContainsKey(key)); + Assert.True(context.MergedJobDataMap.GetBoolean(key)); + } + [Fact] public void Get_WithNonExistentKey_ReturnsNull() { diff --git a/tests/Melodee.Tests.Common/Jobs/MusicBrainzUpdateDatabaseJobTests.cs b/tests/Melodee.Tests.Common/Jobs/MusicBrainzUpdateDatabaseJobTests.cs index 99e5f73c7..7066274cb 100644 --- a/tests/Melodee.Tests.Common/Jobs/MusicBrainzUpdateDatabaseJobTests.cs +++ b/tests/Melodee.Tests.Common/Jobs/MusicBrainzUpdateDatabaseJobTests.cs @@ -1,4 +1,5 @@ using System.Net; +using DecentDB.AdoNet; using FluentAssertions; using Melodee.Common.Configuration; using Melodee.Common.Constants; @@ -29,6 +30,7 @@ public async Task Execute_WhenImportIsCancelled_RestoresDatabaseAndRemovesLockFi var configurationFactory = CreateConfigurationFactory(storagePath); var settingService = new SettingService(Logger, CacheManager, configurationFactory, MockFactory()); var musicBrainzDbContextFactory = CreateMusicBrainzDbContextFactory(databasePath); + var warmupService = new MusicBrainzDecentDbWarmupService(Logger, musicBrainzDbContextFactory); var repositoryMock = new Mock<IMusicBrainzRepository>(); MusicBrainzImportRequest? capturedRequest = null; repositoryMock.Setup(repository => repository.ImportData( @@ -47,7 +49,8 @@ public async Task Execute_WhenImportIsCancelled_RestoresDatabaseAndRemovesLockFi settingService, CreateHttpClientFactory(latestVersion), musicBrainzDbContextFactory, - repositoryMock.Object); + repositoryMock.Object, + warmupService); var context = new MelodeeJobExecutionContext(CancellationToken.None); File.Exists(databasePath).Should().BeTrue(); @@ -109,6 +112,7 @@ public async Task Execute_WhenRepositoryReturnsFailure_IncludesMeaningfulErrorMe var configurationFactory = CreateConfigurationFactory(storagePath); var settingService = new SettingService(Logger, CacheManager, configurationFactory, MockFactory()); var musicBrainzDbContextFactory = CreateMusicBrainzDbContextFactory(databasePath); + var warmupService = new MusicBrainzDecentDbWarmupService(Logger, musicBrainzDbContextFactory); var repositoryMock = new Mock<IMusicBrainzRepository>(); repositoryMock.Setup(repository => repository.ImportData( It.IsAny<MusicBrainzImportRequest>(), @@ -127,7 +131,8 @@ public async Task Execute_WhenRepositoryReturnsFailure_IncludesMeaningfulErrorMe settingService, CreateHttpClientFactory(latestVersion), musicBrainzDbContextFactory, - repositoryMock.Object); + repositoryMock.Object, + warmupService); var context = new MelodeeJobExecutionContext(CancellationToken.None); await job.Execute(context); @@ -170,22 +175,17 @@ public async Task Execute_WhenImportSucceeds_CheckpointsImportedDatabaseBeforePr { var storagePath = Path.Combine(Path.GetTempPath(), $"melodee-mb-job-{Guid.NewGuid():N}"); var databasePath = Path.Combine(Path.GetTempPath(), $"melodee-mb-job-db-{Guid.NewGuid():N}.ddb"); - var fakeCliDirectory = Path.Combine(Path.GetTempPath(), $"melodee-decentdb-cli-{Guid.NewGuid():N}"); Directory.CreateDirectory(storagePath); - Directory.CreateDirectory(fakeCliDirectory); - var previousCliPath = Environment.GetEnvironmentVariable("DECENTDB_CLI_PATH"); try { const string latestVersion = "20260418-002325"; await SeedPreparedStorageAsync(storagePath, latestVersion); - var checkpointArgsFile = Path.Combine(fakeCliDirectory, "checkpoint-args.txt"); - var fakeCliPath = await CreateFakeDecentDbCliAsync(fakeCliDirectory, checkpointArgsFile); - Environment.SetEnvironmentVariable("DECENTDB_CLI_PATH", fakeCliPath); var configurationFactory = CreateConfigurationFactory(storagePath); var settingService = new SettingService(Logger, CacheManager, configurationFactory, MockFactory()); var musicBrainzDbContextFactory = CreateMusicBrainzDbContextFactory(databasePath); + var warmupService = new MusicBrainzDecentDbWarmupService(Logger, musicBrainzDbContextFactory); var repositoryMock = new Mock<IMusicBrainzRepository>(); MusicBrainzImportRequest? capturedRequest = null; repositoryMock.Setup(repository => repository.ImportData( @@ -195,8 +195,7 @@ public async Task Execute_WhenImportSucceeds_CheckpointsImportedDatabaseBeforePr .Callback<MusicBrainzImportRequest, ImportProgressCallback?, CancellationToken>((request, _, _) => { capturedRequest = request; - File.WriteAllText(request.TargetDatabasePath!, "imported"); - File.WriteAllBytes($"{request.TargetDatabasePath}.wal", new byte[4096]); + SeedImportedDecentDb(request.TargetDatabasePath!); }) .ReturnsAsync(new OperationResult<bool> { @@ -209,7 +208,8 @@ public async Task Execute_WhenImportSucceeds_CheckpointsImportedDatabaseBeforePr settingService, CreateHttpClientFactory(latestVersion), musicBrainzDbContextFactory, - repositoryMock.Object); + repositoryMock.Object, + warmupService); var context = new MelodeeJobExecutionContext(CancellationToken.None); await job.Execute(context); @@ -217,40 +217,20 @@ public async Task Execute_WhenImportSucceeds_CheckpointsImportedDatabaseBeforePr context.JobResult.Should().NotBeNull(); context.JobResult!.Status.Should().Be(JobResultStatus.Success); capturedRequest.Should().NotBeNull(); - File.Exists(checkpointArgsFile).Should().BeTrue(); - var checkpointArgs = await File.ReadAllLinesAsync(checkpointArgsFile); - checkpointArgs.Should().Equal("checkpoint", "--db", capturedRequest!.TargetDatabasePath); File.Exists(databasePath).Should().BeTrue(); - File.Exists($"{databasePath}.wal").Should().BeFalse(); + CountCheckpointProbeRows(databasePath).Should().Be(1); File.Exists(capturedRequest.TargetDatabasePath!).Should().BeFalse(); - File.Exists($"{capturedRequest.TargetDatabasePath}.wal").Should().BeFalse(); } finally { - Environment.SetEnvironmentVariable("DECENTDB_CLI_PATH", previousCliPath); try { - if (File.Exists(databasePath)) - { - File.Delete(databasePath); - } - if (File.Exists($"{databasePath}.wal")) - { - File.Delete($"{databasePath}.wal"); - } - if (File.Exists($"{databasePath}.shm")) - { - File.Delete($"{databasePath}.shm"); - } + DeleteDatabaseArtifacts(databasePath); if (Directory.Exists(storagePath)) { Directory.Delete(storagePath, true); } - if (Directory.Exists(fakeCliDirectory)) - { - Directory.Delete(fakeCliDirectory, true); - } } catch { @@ -332,59 +312,46 @@ await File.WriteAllTextAsync(Path.Combine(mbDumpPath, "artist"), await File.WriteAllBytesAsync(Path.Combine(stagingPath, "mbdump-derived.tar.bz2"), [1]); } - private static async Task<string> CreateFakeDecentDbCliAsync(string directoryPath, string checkpointArgsFile) + private static void SeedImportedDecentDb(string databasePath) { - if (OperatingSystem.IsWindows()) - { - var scriptPath = Path.Combine(directoryPath, "decentdb.cmd"); - await File.WriteAllTextAsync(scriptPath, $""" - @echo off - echo %* > "{checkpointArgsFile}" - set db= - :loop - if "%~1"=="" goto done - if "%~1"=="--db" ( - shift - set db=%~1 - ) - shift - goto loop - :done - if not "%db%"=="" del "%db%.wal" 2>nul - if not "%db%"=="" del "%db%.shm" 2>nul - exit /b 0 - """); - return scriptPath; - } - else + using var connection = new DecentDBConnection($"Data Source={databasePath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE checkpoint_probe ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL + ); + INSERT INTO checkpoint_probe (id, name) VALUES (1, 'imported'); + """; + command.ExecuteNonQuery(); + } + + private static long CountCheckpointProbeRows(string databasePath) + { + using var connection = new DecentDBConnection($"Data Source={databasePath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM checkpoint_probe"; + return Convert.ToInt64(command.ExecuteScalar()); + } + + private static void DeleteDatabaseArtifacts(string databasePath) + { + foreach (var path in new[] + { + databasePath, + $"{databasePath}.wal", + $"{databasePath}-wal", + $"{databasePath}.shm", + $"{databasePath}-shm", + $"{databasePath}.coord" + }) { - var scriptPath = Path.Combine(directoryPath, "decentdb"); - await File.WriteAllTextAsync(scriptPath, $""" - #!/bin/sh - printf '%s\n' "$@" > '{checkpointArgsFile.Replace("'", "'\"'\"'")}' - db="" - while [ "$#" -gt 0 ]; do - if [ "$1" = "--db" ]; then - shift - db="$1" - fi - shift - done - if [ -n "$db" ]; then - rm -f "$db.wal" "$db.shm" - fi - exit 0 - """); - File.SetUnixFileMode( - scriptPath, - UnixFileMode.UserRead | - UnixFileMode.UserWrite | - UnixFileMode.UserExecute | - UnixFileMode.GroupRead | - UnixFileMode.GroupExecute | - UnixFileMode.OtherRead | - UnixFileMode.OtherExecute); - return scriptPath; + if (File.Exists(path)) + { + File.Delete(path); + } } } diff --git a/tests/Melodee.Tests.Common/Jobs/ScanStepResultTests.cs b/tests/Melodee.Tests.Common/Jobs/ScanStepResultTests.cs new file mode 100644 index 000000000..244348bc7 --- /dev/null +++ b/tests/Melodee.Tests.Common/Jobs/ScanStepResultTests.cs @@ -0,0 +1,34 @@ +using Melodee.Common.Jobs; + +namespace Melodee.Tests.Common.Jobs; + +public class ScanStepResultTests +{ + [Fact] + public void AlbumsHandledByStorageTransfer_WithMovedAndMergedAlbums_ReturnsCombinedCount() + { + var result = new ScanStepResult( + AlbumsMoved: 4, + AlbumsMergedWithExisting: 5); + + Assert.Equal(9, result.AlbumsHandledByStorageTransfer); + } + + [Fact] + public void Constructor_WithSkippedReasonCounts_SetsReasonCounts() + { + var skippedReasons = new Dictionary<string, int> + { + ["HasInvalidArtists"] = 10, + ["HasInvalidArtists, HasNoImages"] = 2 + }; + + var result = new ScanStepResult( + AlbumsSkippedByStatus: 12, + AlbumsSkippedByReason: skippedReasons); + + Assert.Equal(12, result.AlbumsSkippedByStatus); + Assert.Equal(10, result.AlbumsSkippedByReason!["HasInvalidArtists"]); + Assert.Equal(2, result.AlbumsSkippedByReason["HasInvalidArtists, HasNoImages"]); + } +} diff --git a/tests/Melodee.Tests.Common/Jobs/StagingAlbumRevalidationJobTests.cs b/tests/Melodee.Tests.Common/Jobs/StagingAlbumRevalidationJobTests.cs new file mode 100644 index 000000000..413146420 --- /dev/null +++ b/tests/Melodee.Tests.Common/Jobs/StagingAlbumRevalidationJobTests.cs @@ -0,0 +1,266 @@ +using System.Text; +using FluentAssertions; +using Melodee.Common.Configuration; +using Melodee.Common.Constants; +using Melodee.Common.Enums; +using Melodee.Common.Extensions; +using Melodee.Common.Jobs; +using Melodee.Common.Models; +using Melodee.Common.Services.Scanning; +using Melodee.Common.Services.SearchEngines; +using Melodee.Tests.Common.Services; +using Moq; + +namespace Melodee.Tests.Common.Jobs; + +public class StagingAlbumRevalidationJobTests : ServiceTestBase +{ + [Fact] + public async Task Execute_WithTrustedItunesArtistId_RevalidatesAndClearsInvalidArtistReason() + { + const string stagingPath = "/melodee_test/staging"; + const string albumDirectoryName = "iTunes Artist - [2026] Trusted Album"; + var albumDirectoryPath = Path.Combine(stagingPath, albumDirectoryName); + var albumFilePath = Path.Combine(albumDirectoryPath, Album.JsonFileName); + var mockFileSystem = new MockFileSystemService().SetDirectoryExists(stagingPath); + var album = CreateStaleInvalidArtistAlbum(albumDirectoryPath, albumDirectoryName); + mockFileSystem.SetAlbumForFile(albumFilePath, album); + + var albumDiscoveryService = new AlbumDiscoveryService( + Logger, + CacheManager, + MockFactory(), + MockConfigurationFactory(), + mockFileSystem); + var job = new StagingAlbumRevalidationJob( + Logger, + MockConfigurationFactory(), + MockLibraryService(), + albumDiscoveryService, + GetArtistSearchEngineService(), + Serializer, + mockFileSystem, + new AlwaysDueRevalidationStateStore()); + var context = new MelodeeJobExecutionContext(CancellationToken.None); + + await job.Execute(context); + + var result = context.Result.Should().BeOfType<ScanStepResult>().Subject; + result.AlbumsRevalidated.Should().Be(1); + + var persistedBytes = await mockFileSystem.ReadAllBytesAsync(albumFilePath); + var persistedAlbum = Serializer.Deserialize<Album>(Encoding.UTF8.GetString(persistedBytes)); + persistedAlbum.Should().NotBeNull(); + persistedAlbum!.StatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists).Should().BeFalse(); + } + + [Fact] + public async Task Execute_WhenArtistLookupFindsNoMatch_ReportsAttemptAndNoMatch() + { + const string stagingPath = "/melodee_test/staging"; + const string albumDirectoryName = "Unknown Artist - [2026] Missing Artist Album"; + var albumDirectoryPath = Path.Combine(stagingPath, albumDirectoryName); + var albumFilePath = Path.Combine(albumDirectoryPath, Album.JsonFileName); + var mockFileSystem = new MockFileSystemService().SetDirectoryExists(stagingPath); + var album = CreateInvalidArtistAlbumWithoutIdentifiers(albumDirectoryPath, albumDirectoryName); + mockFileSystem.SetAlbumForFile(albumFilePath, album); + + var configFactory = SearchEnginesDisabledConfigurationFactory(); + var albumDiscoveryService = new AlbumDiscoveryService( + Logger, + CacheManager, + MockFactory(), + configFactory, + mockFileSystem); + var job = new StagingAlbumRevalidationJob( + Logger, + configFactory, + MockLibraryService(), + albumDiscoveryService, + CreateArtistSearchEngineService(configFactory), + Serializer, + mockFileSystem, + new AlwaysDueRevalidationStateStore()); + var context = new MelodeeJobExecutionContext(CancellationToken.None); + + await job.Execute(context); + + var result = context.Result.Should().BeOfType<ScanStepResult>().Subject; + result.AlbumsRevalidationLookupsAttempted.Should().Be(1); + result.AlbumsRevalidationNoMatch.Should().Be(1); + result.AlbumsRevalidated.Should().Be(0); + result.AlbumsSkippedRevalidation.Should().Be(0); + result.AlbumsDeferredRevalidation.Should().Be(0); + } + + [Fact] + public void CanAttemptArtistRevalidation_WithBlankArtistName_ReturnsFalse() + { + var album = CreateStaleInvalidArtistAlbum("/melodee_test/staging/Unknown", "Unknown"); + album.Artist = new Artist(string.Empty, string.Empty, string.Empty); + + var result = StagingAlbumRevalidationJob.CanAttemptArtistRevalidation(album); + + result.Should().BeFalse(); + } + + [Fact] + public void CanAttemptArtistRevalidation_WithUnwantedArtistText_ReturnsFalse() + { + var album = CreateStaleInvalidArtistAlbum("/melodee_test/staging/Bad", "Bad"); + album.Artist = new Artist("Artist [WEB]", "ARTISTWEB", "Artist [WEB]"); + album.StatusReasons = AlbumNeedsAttentionReasons.HasInvalidArtists | + AlbumNeedsAttentionReasons.ArtistNameHasUnwantedText; + + var result = StagingAlbumRevalidationJob.CanAttemptArtistRevalidation(album); + + result.Should().BeFalse(); + } + + [Fact] + public void CanAttemptArtistRevalidation_WithTrustedItunesArtist_ReturnsTrue() + { + var album = CreateStaleInvalidArtistAlbum("/melodee_test/staging/Trusted", "Trusted"); + + var result = StagingAlbumRevalidationJob.CanAttemptArtistRevalidation(album); + + result.Should().BeTrue(); + } + + private static Album CreateStaleInvalidArtistAlbum(string albumDirectoryPath, string albumDirectoryName) + { + var artistName = "iTunes Artist"; + return new Album + { + Id = Guid.NewGuid(), + AlbumType = AlbumType.Album, + Artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName) + { + ItunesId = "123456789" + }, + Directory = new FileSystemDirectoryInfo + { + Path = albumDirectoryPath, + Name = albumDirectoryName + }, + OriginalDirectory = new FileSystemDirectoryInfo + { + Path = albumDirectoryPath, + Name = albumDirectoryName + }, + Status = AlbumStatus.Invalid, + StatusReasons = AlbumNeedsAttentionReasons.HasInvalidArtists, + ViaPlugins = [], + Tags = + [ + new MetaTag<object?> { Identifier = MetaTagIdentifier.AlbumArtist, Value = artistName }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.Album, Value = "Trusted Album" }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.RecordingYear, Value = "2026" } + ] + }; + } + + private static Album CreateInvalidArtistAlbumWithoutIdentifiers(string albumDirectoryPath, string albumDirectoryName) + { + const string artistName = "Unknown Revalidation Artist"; + return new Album + { + Id = Guid.NewGuid(), + AlbumType = AlbumType.Album, + Artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName), + Directory = new FileSystemDirectoryInfo + { + Path = albumDirectoryPath, + Name = albumDirectoryName + }, + OriginalDirectory = new FileSystemDirectoryInfo + { + Path = albumDirectoryPath, + Name = albumDirectoryName + }, + Status = AlbumStatus.Invalid, + StatusReasons = AlbumNeedsAttentionReasons.HasInvalidArtists, + ViaPlugins = [], + Tags = + [ + new MetaTag<object?> { Identifier = MetaTagIdentifier.AlbumArtist, Value = artistName }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.Album, Value = "Missing Artist Album" }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.RecordingYear, Value = "2026" } + ] + }; + } + + private ArtistSearchEngineService CreateArtistSearchEngineService(IMelodeeConfigurationFactory configFactory) + { + return new ArtistSearchEngineService( + Logger, + CacheManager, + MockSettingService(), + MockSpotifyClientBuilder(), + configFactory, + MockFactory(), + MockArtistSearchEngineFactory(), + GetMusicBrainzRepository(), + Serializer, + MockHttpClientFactory()); + } + + private static IMelodeeConfigurationFactory SearchEnginesDisabledConfigurationFactory() + { + var settings = TestsBase.NewConfiguration(); + settings[SettingRegistry.SearchEngineMusicBrainzEnabled] = "false"; + settings[SettingRegistry.SearchEngineSpotifyEnabled] = "false"; + settings[SettingRegistry.SearchEngineITunesEnabled] = "false"; + settings[SettingRegistry.SearchEngineLastFmEnabled] = "false"; + settings[SettingRegistry.SearchEngineDiscogsEnabled] = "false"; + settings[SettingRegistry.SearchEngineWikiDataEnabled] = "false"; + + var mock = new Mock<IMelodeeConfigurationFactory>(); + mock.Setup(f => f.GetConfigurationAsync(It.IsAny<CancellationToken>())) + .ReturnsAsync(new MelodeeConfiguration(settings)); + return mock.Object; + } + + private sealed class AlwaysDueRevalidationStateStore : IStagingAlbumRevalidationStateStore + { + public Task<IStagingAlbumRevalidationStateSession> OpenAsync( + string stagingPath, + IReadOnlyCollection<Album> currentAlbums, + CancellationToken cancellationToken) + { + return Task.FromResult<IStagingAlbumRevalidationStateSession>(new AlwaysDueRevalidationStateSession()); + } + } + + private sealed class AlwaysDueRevalidationStateSession : IStagingAlbumRevalidationStateSession + { + public StagingAlbumRevalidationDecision GetDecision(Album album, DateTimeOffset now, bool force) + { + return new StagingAlbumRevalidationDecision(true, Reason: "Test"); + } + + public void RecordAttempt(Album album, DateTimeOffset now, string outcome) + { + } + + public void RecordSuccess(Album album) + { + } + + public Task SaveChangesAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/Melodee.Tests.Common/Melodee.Tests.Common.csproj b/tests/Melodee.Tests.Common/Melodee.Tests.Common.csproj index e211983ce..ff484895e 100644 --- a/tests/Melodee.Tests.Common/Melodee.Tests.Common.csproj +++ b/tests/Melodee.Tests.Common/Melodee.Tests.Common.csproj @@ -15,6 +15,9 @@ </PackageReference> <PackageReference Include="FluentAssertions" /> <PackageReference Include="NBomber" /> + <PackageReference Include="MessagePack"> + <PrivateAssets>all</PrivateAssets> + </PackageReference> <PackageReference Include="NodaTime.Testing" /> <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" /> <PackageReference Include="Microsoft.NET.Test.Sdk" /> diff --git a/tests/Melodee.Tests.Common/Plugins/Conversion/MediaConvertorTests.cs b/tests/Melodee.Tests.Common/Plugins/Conversion/MediaConvertorTests.cs index da5408c13..5aa7ba4e1 100644 --- a/tests/Melodee.Tests.Common/Plugins/Conversion/MediaConvertorTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/Conversion/MediaConvertorTests.cs @@ -10,6 +10,50 @@ namespace Melodee.Tests.Common.Plugins.Conversion; public class MediaConvertorTests { + [Fact] + public async Task IsUsableConvertedMp3Async_WithMp3Header_ReturnsTrue() + { + var testPath = Path.Combine(Path.GetTempPath(), $"melodee-converted-{Guid.NewGuid():N}.mp3"); + + try + { + await File.WriteAllBytesAsync(testPath, [(byte)'I', (byte)'D', (byte)'3', 0]); + + var result = await MediaConvertor.IsUsableConvertedMp3Async(new FileInfo(testPath)); + + Assert.True(result); + } + finally + { + if (File.Exists(testPath)) + { + File.Delete(testPath); + } + } + } + + [Fact] + public async Task IsUsableConvertedMp3Async_WithInvalidMp3File_ReturnsFalse() + { + var testPath = Path.Combine(Path.GetTempPath(), $"melodee-converted-{Guid.NewGuid():N}.mp3"); + + try + { + await File.WriteAllTextAsync(testPath, "not audio"); + + var result = await MediaConvertor.IsUsableConvertedMp3Async(new FileInfo(testPath)); + + Assert.False(result); + } + finally + { + if (File.Exists(testPath)) + { + File.Delete(testPath); + } + } + } + [Fact] public async Task ValidateConvertingFlacToMp3Async() { diff --git a/tests/Melodee.Tests.Common/Plugins/MetaData/BlackbeardTests.cs b/tests/Melodee.Tests.Common/Plugins/MetaData/BlackbeardTests.cs new file mode 100644 index 000000000..fcdda9540 --- /dev/null +++ b/tests/Melodee.Tests.Common/Plugins/MetaData/BlackbeardTests.cs @@ -0,0 +1,243 @@ +using Melodee.Common.Models; +using Melodee.Common.Models.Extensions; +using Melodee.Common.Plugins.MetaData.Directory.Blackbeard; + +namespace Melodee.Tests.Common.Plugins.MetaData; + +public class BlackbeardTests : TestsBase +{ + [Fact] + public async Task ProcessDirectoryAsync_WithBlackbeardProvenance_CreatesAlbumMetadata() + { + var albumDirectory = Path.Combine(Path.GetTempPath(), $"melodee-blackbeard-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(albumDirectory); + await File.WriteAllTextAsync(Path.Combine(albumDirectory, "01 - First Track.mp3"), "track-one"); + await File.WriteAllTextAsync(Path.Combine(albumDirectory, "02 - Second Track.mp3"), "track-two"); + await File.WriteAllTextAsync( + Path.Combine(albumDirectory, Blackbeard.HandlesFileName), + ProvenanceJson()); + + var plugin = new Blackbeard(Serializer, GetAlbumValidator(), NewPluginsConfiguration()); + + var result = await plugin.ProcessDirectoryAsync(new FileSystemDirectoryInfo + { + Path = albumDirectory, + Name = Path.GetFileName(albumDirectory) + }); + + Assert.Equal(1, result.Data); + + var melodeeJson = Directory.GetFiles(albumDirectory, $"*{Album.JsonFileName}") + .Single(x => Path.GetFileName(x) != Blackbeard.HandlesFileName); + var album = await Album.DeserializeAndInitializeAlbumAsync( + Serializer, + melodeeJson); + Assert.NotNull(album); + Assert.Equal("Blackbeard Artist", album.Artist.Name); + Assert.Equal("Blackbeard Album", album.AlbumTitle()); + Assert.Equal(2026, album.AlbumYear()); + Assert.Equal(2, album.SongTotalValue()); + + var songs = album.Songs!.OrderBy(x => x.SortOrder).ToArray(); + Assert.Equal(2, songs.Length); + Assert.Equal("First Track", songs[0].Title()); + Assert.Equal("Second Track", songs[1].Title()); + Assert.Equal(187000, songs[0].Duration()); + Assert.Equal(320, songs[0].BitRate()); + Assert.Equal(44100, songs[0].SamplingRate()); + Assert.Equal(2, songs[0].ChannelCount()); + Assert.Contains(album.Files, x => x.ProcessedByPlugin == nameof(Blackbeard)); + + var serializedAlbum = Serializer.Serialize(album) ?? string.Empty; + Assert.DoesNotContain("apikey", serializedAlbum, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("conversion_command", serializedAlbum, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("/mnt/incoming", serializedAlbum, StringComparison.OrdinalIgnoreCase); + } + finally + { + if (Directory.Exists(albumDirectory)) + { + Directory.Delete(albumDirectory, true); + } + } + } + + [Fact] + public async Task ProcessDirectoryAsync_WithUnsupportedSchemaVersion_ReturnsVisibleWarningAndSkips() + { + var albumDirectory = Path.Combine(Path.GetTempPath(), $"melodee-blackbeard-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(albumDirectory); + await File.WriteAllTextAsync(Path.Combine(albumDirectory, "01 - First Track.mp3"), "track-one"); + await File.WriteAllTextAsync( + Path.Combine(albumDirectory, Blackbeard.HandlesFileName), + ProvenanceJson().Replace("\"schema_version\": 1", "\"schema_version\": 2", StringComparison.Ordinal)); + + var plugin = new Blackbeard(Serializer, GetAlbumValidator(), NewPluginsConfiguration()); + + var result = await plugin.ProcessDirectoryAsync(new FileSystemDirectoryInfo + { + Path = albumDirectory, + Name = Path.GetFileName(albumDirectory) + }); + + Assert.Equal(0, result.Data); + var message = Assert.Single(result.Messages ?? []); + Assert.Contains("unsupported Blackbeard schema version [2]", message); + Assert.Contains($"Supported version is [{Blackbeard.SupportedSchemaVersion}]", message); + Assert.DoesNotContain( + Directory.GetFiles(albumDirectory, $"*{Album.JsonFileName}"), + x => Path.GetFileName(x) != Blackbeard.HandlesFileName); + } + finally + { + if (Directory.Exists(albumDirectory)) + { + Directory.Delete(albumDirectory, true); + } + } + } + + [Fact] + public async Task AlbumForProvenanceFileAsync_WhenTrackPathEscapesDirectory_SkipsUnsafeTrack() + { + var albumDirectory = Path.Combine(Path.GetTempPath(), $"melodee-blackbeard-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(albumDirectory); + await File.WriteAllTextAsync(Path.Combine(albumDirectory, "01 - First Track.mp3"), "track-one"); + var provenancePath = Path.Combine(albumDirectory, Blackbeard.HandlesFileName); + await File.WriteAllTextAsync(provenancePath, ProvenanceJson("../outside.mp3")); + + var plugin = new Blackbeard(Serializer, GetAlbumValidator(), NewPluginsConfiguration()); + + var album = await plugin.AlbumForProvenanceFileAsync( + new FileInfo(provenancePath), + new FileSystemDirectoryInfo + { + Path = albumDirectory, + Name = Path.GetFileName(albumDirectory) + }); + + Assert.NotNull(album); + var song = Assert.Single(album.Songs!); + Assert.Equal("First Track", song.Title()); + } + finally + { + if (Directory.Exists(albumDirectory)) + { + Directory.Delete(albumDirectory, true); + } + } + } + + private static string ProvenanceJson(string secondTrackPath = "02 - Second Track.mp3") + { + return $$""" + { + "schema_version": 1, + "blackbeard_version": "0.1.0", + "release": { + "guid": "https://example.invalid/release", + "canonical_release_key": "blackbeard artist|blackbeard album|2026", + "album_artist": "Blackbeard Artist", + "album_title": "Blackbeard Album", + "release_year": 2026, + "staged_path": "Blackbeard Artist - Blackbeard Album (2026)" + }, + "tracks": [ + { + "nzb_source": "https://example.invalid/api?t=get&apikey=secret", + "staged_track_path": "01 - First Track.mp3", + "track_number": 1, + "track_total": 2, + "disc_number": 1, + "disc_total": 1, + "title": "First Track", + "artist": "Blackbeard Artist", + "album_artist": "Blackbeard Artist", + "album_title": "Blackbeard Album", + "release_year": 2026, + "source": { + "path": "/mnt/incoming/blackbeard/downloads/source.flac" + }, + "output": { + "path": "01 - First Track.mp3", + "conversion_command": "ffmpeg -i /mnt/incoming/blackbeard/downloads/source.flac output.mp3" + }, + "normalization": { + "tags_after": { + "title": "First Track", + "artist": "Blackbeard Artist", + "album_artist": "Blackbeard Artist", + "album_title": "Blackbeard Album", + "release_year": 2026, + "track_number": 1, + "track_total": 2, + "disc_number": 1, + "disc_total": 1, + "genres": ["Ambient"], + "codec": "mp3", + "container": "mp3", + "bitrate": 320, + "sample_rate": 44100, + "channels": 2, + "duration_ms": 187000 + } + }, + "final_validation": { + "passed": true, + "classification": "completed" + } + }, + { + "staged_track_path": "{{secondTrackPath}}", + "track_number": 2, + "track_total": 2, + "disc_number": 1, + "disc_total": 1, + "title": "Second Track", + "artist": "Blackbeard Artist", + "album_artist": "Blackbeard Artist", + "album_title": "Blackbeard Album", + "release_year": 2026, + "output": { + "path": "{{secondTrackPath}}" + }, + "normalization": { + "tags_after": { + "title": "Second Track", + "artist": "Blackbeard Artist", + "album_artist": "Blackbeard Artist", + "album_title": "Blackbeard Album", + "release_year": 2026, + "track_number": 2, + "track_total": 2, + "disc_number": 1, + "disc_total": 1, + "genres": ["Ambient"], + "codec": "mp3", + "container": "mp3", + "bitrate": 256, + "sample_rate": 44100, + "channels": 2, + "duration_ms": 188000 + } + }, + "final_validation": { + "passed": true, + "classification": "completed" + } + } + ] + } + """; + } +} diff --git a/tests/Melodee.Tests.Common/Plugins/MetaData/MetaTagTests.cs b/tests/Melodee.Tests.Common/Plugins/MetaData/MetaTagTests.cs index edc807a8b..9306483ab 100644 --- a/tests/Melodee.Tests.Common/Plugins/MetaData/MetaTagTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/MetaData/MetaTagTests.cs @@ -78,7 +78,7 @@ public async Task ValidateLoadingTagsForBorkedMp3FileAsync() var fileInfo = new FileInfo(testFile); if (fileInfo.Exists) { - var metaTag = new IdSharpMetaTag(new MetaTagsProcessor(NewPluginsConfiguration(), Serializer), NewPluginsConfiguration()); + var metaTag = new NativeId3MetaTag(new MetaTagsProcessor(NewPluginsConfiguration(), Serializer), NewPluginsConfiguration()); var dirInfo = new FileSystemDirectoryInfo { Path = @"/melodee_test/tests/", diff --git a/tests/Melodee.Tests.Common/Plugins/MetaData/NfoTests.cs b/tests/Melodee.Tests.Common/Plugins/MetaData/NfoTests.cs index 0e260fd9c..e48ab73b5 100644 --- a/tests/Melodee.Tests.Common/Plugins/MetaData/NfoTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/MetaData/NfoTests.cs @@ -63,6 +63,59 @@ public void ValidateIsLineForSong(string? input, bool shouldBe) Assert.Equal(shouldBe, Nfo.IsLineForSong(input ?? string.Empty)); } + [Fact] + public async Task AlbumForNfoFileAsync_WithMalformedTrackLines_DoesNotThrow() + { + var testDirectory = Directory.CreateTempSubdirectory("melodee-nfo-malformed-"); + try + { + var nfoPath = Path.Combine(testDirectory.FullName, "malformed.nfo"); + await File.WriteAllLinesAsync(nfoPath, + [ + "Artist: Test Artist", + "Title: Malformed NFO", + "01 00:01", + "02. 00:02" + ]); + var nfo = new Nfo(Serializer, GetAlbumValidator(), NewPluginsConfiguration()); + + var exception = await Record.ExceptionAsync(() => + nfo.AlbumForNfoFileAsync(new FileInfo(nfoPath), testDirectory.ToDirectorySystemInfo())); + + Assert.Null(exception); + } + finally + { + testDirectory.Delete(true); + } + } + + [Fact] + public async Task AlbumForNfoFileAsync_WithMissingArtist_ReturnsNull() + { + var testDirectory = Directory.CreateTempSubdirectory("melodee-nfo-missing-artist-"); + try + { + var mediaPath = Path.Combine(testDirectory.FullName, "01 Test Song.mp3"); + await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]); + var nfoPath = Path.Combine(testDirectory.FullName, "missing-artist.nfo"); + await File.WriteAllLinesAsync(nfoPath, + [ + "Title: Missing Artist NFO", + "01 Test Song 00:01" + ]); + var nfo = new Nfo(Serializer, GetAlbumValidator(), NewPluginsConfiguration()); + + var result = await nfo.AlbumForNfoFileAsync(new FileInfo(nfoPath), testDirectory.ToDirectorySystemInfo()); + + Assert.Null(result); + } + finally + { + testDirectory.Delete(true); + } + } + [Fact] public async Task ParseJellyfinNfoFile() { diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/DecentDBMusicBrainzRepositoryTests.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/DecentDBMusicBrainzRepositoryTests.cs index 5ab3ef816..dee0fbf72 100644 --- a/tests/Melodee.Tests.Common/Plugins/SearchEngine/DecentDBMusicBrainzRepositoryTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/DecentDBMusicBrainzRepositoryTests.cs @@ -216,6 +216,66 @@ public async Task SearchArtist_WithAlbumKeyValues_IncludesAlbumMatching() Assert.NotEmpty(artist.Releases); } + [Fact] + public async Task SearchArtist_WithSingleResultAndAlbumKeyValues_LoadsMatchingReleaseOnly() + { + var artistId = Guid.NewGuid(); + var matchingAlbumId = Guid.NewGuid(); + var otherAlbumId = Guid.NewGuid(); + using var context = new MusicBrainzDbContext(_dbContextOptions); + context.ArtistAliases.RemoveRange(context.ArtistAliases); + context.Artists.RemoveRange(context.Artists); + context.Albums.RemoveRange(context.Albums); + await context.SaveChangesAsync(); + + context.Artists.Add(new Artist + { + Id = 1, + MusicBrainzArtistId = 1, + MusicBrainzIdRaw = artistId.ToString(), + Name = "Compact Artist", + NameNormalized = "Compact Artist".ToNormalizedString() ?? string.Empty, + SortName = "Compact Artist", + AlternateNames = "" + }); + context.Albums.AddRange( + new Album + { + Id = 1, + MusicBrainzIdRaw = matchingAlbumId.ToString(), + Name = "Target Album", + NameNormalized = "Target Album".ToNormalizedString() ?? string.Empty, + SortName = "Target Album", + ReleaseDate = new DateTime(2024, 1, 1), + ReleaseType = 1, + MusicBrainzArtistId = 1, + ReleaseGroupMusicBrainzIdRaw = Guid.NewGuid().ToString() + }, + new Album + { + Id = 2, + MusicBrainzIdRaw = otherAlbumId.ToString(), + Name = "Other Album", + NameNormalized = "Other Album".ToNormalizedString() ?? string.Empty, + SortName = "Other Album", + ReleaseDate = new DateTime(2025, 1, 1), + ReleaseType = 1, + MusicBrainzArtistId = 1, + ReleaseGroupMusicBrainzIdRaw = Guid.NewGuid().ToString() + }); + await context.SaveChangesAsync(); + + var result = await _repository.SearchArtist(new ArtistQuery + { + Name = "Compact Artist", + AlbumKeyValues = [new KeyValue("2024", "Target Album")] + }, 1); + + var artist = Assert.Single(result.Data); + var release = Assert.Single(artist.Releases ?? []); + Assert.Equal("Target Album", release.Name); + } + [Fact] public async Task SearchArtist_WithSpecialCharacters_HandlesCorrectly() { diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/ITunesTests.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/ITunesTests.cs index 59adf6805..8181fa126 100644 --- a/tests/Melodee.Tests.Common/Plugins/SearchEngine/ITunesTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/ITunesTests.cs @@ -1,3 +1,5 @@ +using System.Net; +using System.Text; using Melodee.Common.Models.SearchEngines; using Melodee.Common.Plugins.SearchEngine.ITunes; using Melodee.Common.Services.Caching; @@ -22,4 +24,51 @@ public async Task PerformITunesAlbumSearch() Assert.NotNull(result); } } + + [Fact] + public async Task DoArtistSearchAsync_WithLargeArtistId_DeserializesResultAndUsesRequestedLimit() + { + Uri? requestedUri = null; + var handler = new HttpHandlerStubDelegate((request, _) => + { + requestedUri = request.RequestUri; + const string json = """ + { + "resultCount": 1, + "results": [ + { + "wrapperType": "artist", + "artistType": "Artist", + "artistName": "Large Id Artist", + "artistId": 3000000000, + "amgArtistId": 4000000000, + "primaryGenreId": 5000000000, + "primaryGenreName": "Rock" + } + ] + } + """; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + }); + using var httpClient = new HttpClient(handler); + var itunes = new ITunesSearchEngine( + Logger, + Serializer, + new TestHttpClientFactory(httpClient), + new FakeCacheManager(Logger, TimeSpan.FromMinutes(5), Serializer)); + + var result = await itunes.DoArtistSearchAsync(new ArtistQuery + { + Name = "Large Id Artist", + Country = "US" + }, 5); + + var artist = Assert.Single(result.Data ?? []); + Assert.Equal("3000000000", artist.ItunesId); + Assert.Equal("4000000000", artist.AmgId); + Assert.Contains("limit=5", requestedUri?.ToString()); + } } diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/DecentDBDiagnosticTests.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/DecentDBDiagnosticTests.cs index 175da3d0e..267040083 100644 --- a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/DecentDBDiagnosticTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/DecentDBDiagnosticTests.cs @@ -1,6 +1,7 @@ using System.Data.Common; using FluentAssertions; using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data.Models.Materialized; using Microsoft.EntityFrameworkCore; namespace Melodee.Tests.Common.Plugins.SearchEngine.MusicBrainz; @@ -118,6 +119,47 @@ INSERT INTO ArtistStaging (ArtistId, MusicBrainzIdRaw, Name, NameNormalized, Sor count.Should().Be(2); } + [Fact] + public async Task IndexedStringEquality_OnMusicBrainzIdRaw_RoundTripsCanonicalString() + { + var dbOptions = new DbContextOptionsBuilder<MusicBrainzDbContext>() + .UseDecentDB($"Data Source={_dbFile}") + .Options; + + var targetId = Guid.Parse("395cc503-63b5-4a0b-a20a-604e3fcacea2").ToString(); + + await using (var context = new MusicBrainzDbContext(dbOptions)) + { + await context.Database.EnsureCreatedAsync(); + for (var i = 0; i < 250; i++) + { + context.Artists.Add(new Artist + { + MusicBrainzArtistId = i + 1, + MusicBrainzIdRaw = i == 127 ? targetId : Guid.NewGuid().ToString(), + Name = $"Artist {i}", + NameNormalized = $"artist {i}", + SortName = $"Artist {i}", + AlternateNames = string.Empty + }); + } + + await context.SaveChangesAsync(); + } + + await using (var context = new MusicBrainzDbContext(dbOptions)) + { + var result = await context.Artists + .AsNoTracking() + .Where(a => a.MusicBrainzIdRaw == targetId) + .OrderBy(a => a.Id) + .Select(a => a.MusicBrainzIdRaw) + .SingleAsync(); + + result.Should().Be(targetId); + } + } + private static void AddParameter(DbCommand command, string name, object value) { var parameter = command.CreateParameter(); diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzDecentDbWarmupServiceTests.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzDecentDbWarmupServiceTests.cs new file mode 100644 index 000000000..bd3a9b396 --- /dev/null +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzDecentDbWarmupServiceTests.cs @@ -0,0 +1,151 @@ +using DecentDB.EntityFrameworkCore; +using FluentAssertions; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data.Models.Materialized; +using Melodee.Tests.Common.Services; +using Microsoft.EntityFrameworkCore; +using Moq; + +namespace Melodee.Tests.Common.Plugins.SearchEngine.MusicBrainz; + +public class MusicBrainzDecentDbWarmupServiceTests : ServiceTestBase +{ + [Fact] + public async Task WarmHotIndexesAsync_WhenMaterializedRowsExist_WarmsExpectedIndexedShapes() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"melodee-mb-warmup-{Guid.NewGuid():N}.ddb"); + + try + { + var dbContextFactory = await CreateSeededFactoryAsync(databasePath); + var service = new MusicBrainzDecentDbWarmupService(Logger, dbContextFactory); + + var result = await service.WarmHotIndexesAsync(); + + result.Succeeded.Should().BeTrue(result.Message); + result.Skipped.Should().BeFalse(); + result.WarmedQueryCount.Should().BeGreaterThanOrEqualTo(4); + result.Measurements.Select(measurement => measurement.Name).Should().Contain([ + "exact-normalized-name", + "exact-musicbrainz-id-raw", + "exact-normalized-alias", + "albums-by-artist-id" + ]); + result.Measurements.Single(measurement => measurement.Name == "exact-normalized-name").RowCount + .Should().Be(1); + result.Measurements.Single(measurement => measurement.Name == "exact-musicbrainz-id-raw").RowCount + .Should().Be(1); + result.Measurements.Single(measurement => measurement.Name == "exact-normalized-alias").RowCount + .Should().Be(1); + result.Measurements.Single(measurement => measurement.Name == "albums-by-artist-id").RowCount + .Should().Be(1); + } + finally + { + DeleteDatabaseArtifacts(databasePath); + } + } + + [Fact] + public async Task WarmHotIndexesAsync_WhenNoArtistsExist_ReturnsSkippedResult() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"melodee-mb-warmup-empty-{Guid.NewGuid():N}.ddb"); + + try + { + var dbContextFactory = await CreateEmptyFactoryAsync(databasePath); + var service = new MusicBrainzDecentDbWarmupService(Logger, dbContextFactory); + + var result = await service.WarmHotIndexesAsync(); + + result.Succeeded.Should().BeFalse(); + result.Skipped.Should().BeTrue(); + result.Message.Should().Contain("no materialized artists"); + result.WarmedQueryCount.Should().Be(0); + } + finally + { + DeleteDatabaseArtifacts(databasePath); + } + } + + private static async Task<IDbContextFactory<MusicBrainzDbContext>> CreateSeededFactoryAsync(string databasePath) + { + var dbContextOptions = CreateOptions(databasePath); + await using var context = new MusicBrainzDbContext(dbContextOptions); + await context.Database.EnsureCreatedAsync(); + context.Artists.Add(new Artist + { + MusicBrainzArtistId = 1001, + Name = "Example Artist", + SortName = "Example Artist", + NameNormalized = "exampleartist", + MusicBrainzIdRaw = "11111111-1111-1111-1111-111111111111" + }); + context.ArtistAliases.Add(new ArtistAliasLookup + { + MusicBrainzArtistId = 1001, + NameNormalized = "examplealias" + }); + context.Albums.Add(new Album + { + MusicBrainzArtistId = 1001, + Name = "Example Album", + SortName = "Example Album", + NameNormalized = "examplealbum", + MusicBrainzIdRaw = "22222222-2222-2222-2222-222222222222", + ReleaseGroupMusicBrainzIdRaw = "33333333-3333-3333-3333-333333333333", + ReleaseDate = new DateTime(2026, 1, 1) + }); + await context.SaveChangesAsync(); + + return CreateFactory(dbContextOptions); + } + + private static async Task<IDbContextFactory<MusicBrainzDbContext>> CreateEmptyFactoryAsync(string databasePath) + { + var dbContextOptions = CreateOptions(databasePath); + await using var context = new MusicBrainzDbContext(dbContextOptions); + await context.Database.EnsureCreatedAsync(); + await context.SaveChangesAsync(); + + return CreateFactory(dbContextOptions); + } + + private static DbContextOptions<MusicBrainzDbContext> CreateOptions(string databasePath) + { + return new DbContextOptionsBuilder<MusicBrainzDbContext>() + .UseDecentDB($"Data Source={databasePath}", optionsBuilder => optionsBuilder.UseNodaTime()) + .Options; + } + + private static IDbContextFactory<MusicBrainzDbContext> CreateFactory( + DbContextOptions<MusicBrainzDbContext> dbContextOptions) + { + var dbContextFactory = new Mock<IDbContextFactory<MusicBrainzDbContext>>(); + dbContextFactory.Setup(factory => factory.CreateDbContext()) + .Returns(() => new MusicBrainzDbContext(dbContextOptions)); + dbContextFactory.Setup(factory => factory.CreateDbContextAsync(It.IsAny<CancellationToken>())) + .ReturnsAsync(() => new MusicBrainzDbContext(dbContextOptions)); + return dbContextFactory.Object; + } + + private static void DeleteDatabaseArtifacts(string databasePath) + { + foreach (var path in new[] + { + databasePath, + $"{databasePath}.wal", + $"{databasePath}-wal", + $"{databasePath}.shm", + $"{databasePath}-shm", + $"{databasePath}.coord" + }) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs index 9de1e1f1d..357c49d02 100644 --- a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/MusicBrainzImportBenchmark.cs @@ -92,7 +92,7 @@ void ProgressCallback(string phase, int current, int total, string? message) } } - await importer.ImportAsync( + var summary = await importer.ImportAsync( context, storagePath, ProgressCallback, @@ -104,11 +104,8 @@ await importer.ImportAsync( phaseTimings[currentPhase] = phaseStopwatch.ElapsedMilliseconds; } - var importedArtists = await context.Artists.CountAsync(); - var importedAlbums = await context.Albums.CountAsync(); - - importedArtists.Should().Be(stats.ArtistCount); - importedAlbums.Should().BeGreaterThan(0); + summary.Artists.Should().Be(stats.ArtistCount); + summary.Albums.Should().BeGreaterThan(0); phaseTimings.Should().NotBeEmpty(); } @@ -171,7 +168,7 @@ void ProgressCallback(string phase, int current, int total, string? message) // Run the import var importSw = Stopwatch.StartNew(); - await importer.ImportAsync( + var summary = await importer.ImportAsync( context, storagePath, ProgressCallback, @@ -188,10 +185,11 @@ await importer.ImportAsync( results.TotalImportMs = importSw.ElapsedMilliseconds; results.PhaseTimings = phaseTimings; - // Get final counts - results.ImportedArtists = await context.Artists.CountAsync(); - results.ImportedAlbums = await context.Albums.CountAsync(); - results.ImportedRelations = await context.ArtistRelations.CountAsync(); + // Use the importer's known materialization counts so benchmark results + // do not include extra large-table verification probes. + results.ImportedArtists = summary.Artists; + results.ImportedAlbums = summary.Albums; + results.ImportedRelations = summary.ArtistRelations; // Calculate metrics results.RecordsPerSecond = results.TotalRecordsGenerated / (results.TotalImportMs / 1000.0); diff --git a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/StreamingMusicBrainzImporterTests.cs b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/StreamingMusicBrainzImporterTests.cs index 5f6f9b766..60e9afa3e 100644 --- a/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/StreamingMusicBrainzImporterTests.cs +++ b/tests/Melodee.Tests.Common/Plugins/SearchEngine/MusicBrainz/StreamingMusicBrainzImporterTests.cs @@ -108,7 +108,7 @@ public async Task ImportAsync_WithSmallTestData_ImportsSuccessfully() var importer = new DecentDBStreamingMusicBrainzImporter(_logger); var progressMessages = new List<string>(); - await importer.ImportAsync( + var summary = await importer.ImportAsync( context, _testDataPath, (phase, current, total, msg) => progressMessages.Add($"{phase}: {msg}"), @@ -121,6 +121,10 @@ await importer.ImportAsync( artistCount.Should().Be(stats.ArtistCount); aliasCount.Should().BeGreaterThan(0); albumCount.Should().BeGreaterThan(0); + summary.Artists.Should().Be(stats.ArtistCount); + summary.ArtistAliases.Should().Be(aliasCount); + summary.Albums.Should().Be(albumCount); + summary.HasMaterializedData.Should().BeTrue(); progressMessages.Should().NotBeEmpty(); } @@ -244,7 +248,7 @@ public async Task ImportAsync_WithMissingFiles_HandlesGracefully() var importer = new DecentDBStreamingMusicBrainzImporter(_logger); - await importer.ImportAsync( + var summary = await importer.ImportAsync( context, Path.Combine(_testDataPath, "empty"), null, @@ -252,6 +256,7 @@ await importer.ImportAsync( var artistCount = await context.Artists.CountAsync(); artistCount.Should().Be(0); + summary.HasMaterializedData.Should().BeFalse(); } [Fact] diff --git a/tests/Melodee.Tests.Common/Services/ArtistServiceTests.cs b/tests/Melodee.Tests.Common/Services/ArtistServiceTests.cs index 59b72fc57..91c6c12ed 100644 --- a/tests/Melodee.Tests.Common/Services/ArtistServiceTests.cs +++ b/tests/Melodee.Tests.Common/Services/ArtistServiceTests.cs @@ -1913,6 +1913,47 @@ public async Task FindArtistAsync_BySpotifyId_ReturnsArtist() Assert.Equal(spotifyId, result.Data.SpotifyId); } + [Fact] + public async Task FindArtistAsync_ByItunesId_ReturnsArtist() + { + var itunesId = "123456789"; + var artistName = "iTunes Find Test"; + + await using (var context = await MockFactory().CreateDbContextAsync()) + { + var library = new Library + { + Name = "Test Library", + Path = "/test/path", + Type = (int)LibraryType.Storage, + CreatedAt = Instant.FromDateTimeUtc(DateTime.UtcNow) + }; + context.Libraries.Add(library); + await context.SaveChangesAsync(); + + var artist = new Artist + { + ApiKey = Guid.NewGuid(), + ItunesId = itunesId, + Directory = "find-itunes-test", + CreatedAt = Instant.FromDateTimeUtc(DateTime.UtcNow), + LibraryId = library.Id, + Name = artistName, + NameNormalized = artistName.ToNormalizedString()!, + Library = library + }; + context.Artists.Add(artist); + await context.SaveChangesAsync(); + } + + var result = await GetArtistService().FindArtistAsync(null, Guid.Empty, null, null, null, itunesId); + + AssertResultIsSuccessful(result); + Assert.NotNull(result.Data); + Assert.Equal(artistName, result.Data.Name); + Assert.Equal(itunesId, result.Data.ItunesId); + } + #endregion #region Additional Helper Methods diff --git a/tests/Melodee.Tests.Common/Services/Scanning/DirectoryProcessorToStagingServiceTests.cs b/tests/Melodee.Tests.Common/Services/Scanning/DirectoryProcessorToStagingServiceTests.cs index a7ccac622..bd5259ae9 100644 --- a/tests/Melodee.Tests.Common/Services/Scanning/DirectoryProcessorToStagingServiceTests.cs +++ b/tests/Melodee.Tests.Common/Services/Scanning/DirectoryProcessorToStagingServiceTests.cs @@ -1,9 +1,12 @@ +using FluentAssertions; using Melodee.Common.Data.Models; using Melodee.Common.Enums; using Melodee.Common.Imaging; using Melodee.Common.Models; +using Melodee.Common.Models.Scripting; using Melodee.Common.Services; using Melodee.Common.Services.Scanning; +using Melodee.Common.Services.ScriptEvaluation; using Microsoft.EntityFrameworkCore; using Moq; using NodaTime; @@ -14,6 +17,21 @@ public class DirectoryProcessorToStagingServiceTests : ServiceTestBase { #region Helper Methods + [Fact] + public void FormatProcessingEventMessage_WithException_ReturnsSingleLineSummary() + { + var exception = new InvalidOperationException("Non-negative number required.\nStack detail"); + + var result = DirectoryProcessorToStagingService.FormatProcessingEventMessage( + "Processing Directory [{0}]", + exception, + "Album Name"); + + result.Should().Be("Error: Processing Directory [Album Name]: Non-negative number required. Stack detail"); + result.Should().NotContain("System.InvalidOperationException"); + result.Should().NotContain(" at "); + } + private DirectoryProcessorToStagingService GetDirectoryProcessorService() { return new DirectoryProcessorToStagingService( @@ -54,6 +72,29 @@ private DirectoryProcessorToStagingService GetDirectoryProcessorService(IFileSys new ImageProcessor()); } + private DirectoryProcessorToStagingService GetDirectoryProcessorService( + IFileSystemService fileSystemService, + IScriptOrchestrationService scriptOrchestrationService, + DenyActionHandlerFactory denyActionHandlerFactory) + { + return new DirectoryProcessorToStagingService( + Logger, + CacheManager, + MockFactory(), + MockConfigurationFactory(), + GetLibraryService(), + Serializer, + GetMediaEditService(), + GetArtistSearchEngineService(), + GetAlbumImageSearchEngineService(), + MockHttpClientFactory(), + fileSystemService, + scriptOrchestrationService, + MockDirectoryContextProvider(), + denyActionHandlerFactory, + new ImageProcessor()); + } + private async Task CreateStagingLibraryInDb() { await using var context = await MockFactory().CreateDbContextAsync(); @@ -295,6 +336,486 @@ public async Task ProcessDirectoryAsync_WithLastProcessDate_FiltersOlderFiles() #endregion + #region Source Metadata Cleanup Tests + + [Fact] + public void IsSourceMetadataOnlyDirectory_WithOnlyKnownSidecars_ReturnsTrue() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-sidecars-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "release.nfo"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "release.sfv"), "metadata"); + + var result = DirectoryProcessorToStagingService.IsSourceMetadataOnlyDirectory( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }); + + Assert.True(result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void IsSourceMetadataOnlyDirectory_WithUnknownFile_ReturnsFalse() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-sidecars-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "release.nfo"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "keep.txt"), "not sidecar metadata"); + + var result = DirectoryProcessorToStagingService.IsSourceMetadataOnlyDirectory( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }); + + Assert.False(result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void DeleteSourceSidecarMetadataFiles_DeletesKnownSidecarsOnly() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-sidecars-{Guid.NewGuid():N}"); + var keepFilePath = Path.Combine(releasePath, "keep.txt"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "release.cue"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "release.m3u"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "release.nfo"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "release.sfv"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, ".blackbeard.provenance.json"), "{}"); + File.WriteAllText(keepFilePath, "keep"); + + var deletedCount = DirectoryProcessorToStagingService.DeleteSourceSidecarMetadataFiles( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }, + Logger); + + Assert.Equal(5, deletedCount); + Assert.True(File.Exists(keepFilePath)); + var remainingFiles = Directory.EnumerateFiles(releasePath).ToArray(); + Assert.Single(remainingFiles); + Assert.Equal(keepFilePath, remainingFiles[0]); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void IsSourceResidueOnlyDirectory_WithImageTextAndSidecarsNoMedia_ReturnsTrue() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-residue-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "release.nfo"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "release.sfv"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "i-01-Front.jpg"), "image"); + File.WriteAllText(Path.Combine(releasePath, "foo_dr.txt"), "text"); + + var result = DirectoryProcessorToStagingService.IsSourceResidueOnlyDirectory( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }); + + Assert.True(result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void IsSourceResidueOnlyDirectory_WithMediaFile_ReturnsFalse() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-residue-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "release.sfv"), "metadata"); + File.WriteAllText(Path.Combine(releasePath, "i-01-Front.jpg"), "image"); + File.WriteAllText(Path.Combine(releasePath, "01-track.mp3"), "media"); + + var result = DirectoryProcessorToStagingService.IsSourceResidueOnlyDirectory( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }); + + Assert.False(result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void DeleteSourceResidueOnlyDirectoryFiles_WithNestedResidueDirectories_RemovesLeafThenParentResidue() + { + var rootPath = Path.Combine(Path.GetTempPath(), $"melodee-residue-root-{Guid.NewGuid():N}"); + var releasePath = Path.Combine(rootPath, "Artist - Album (part "); + var childPath = Path.Combine(releasePath, " two) (2026)"); + + try + { + Directory.CreateDirectory(childPath); + File.WriteAllText(Path.Combine(releasePath, "cover.jpg"), "image"); + File.WriteAllText(Path.Combine(childPath, "i-01-Front.jpg"), "image"); + + var deletedCount = DirectoryProcessorToStagingService.DeleteSourceResidueOnlyDirectoryFiles( + new FileSystemDirectoryInfo + { + Path = rootPath, + Name = Path.GetFileName(rootPath) + }, + Logger); + + Assert.Equal(2, deletedCount); + Assert.False(Directory.Exists(releasePath)); + Assert.True(Directory.Exists(rootPath)); + } + finally + { + if (Directory.Exists(rootPath)) + { + Directory.Delete(rootPath, true); + } + } + } + + [Fact] + public async Task FindUnstableSourceFileAsync_WhenFileChanges_ReturnsFilePath() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-unstable-{Guid.NewGuid():N}"); + var filePath = Path.Combine(releasePath, "01-track.mp3"); + + try + { + Directory.CreateDirectory(releasePath); + await File.WriteAllTextAsync(filePath, "media"); + + var stabilityTask = DirectoryProcessorToStagingService.FindUnstableSourceFileAsync( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }, + delayMs: 300); + + await Task.Delay(50); + await File.AppendAllTextAsync(filePath, " changed"); + + var result = await stabilityTask; + + Assert.Equal(filePath, result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public async Task FindUnstableSourceFileAsync_WhenFilesAreStable_ReturnsNull() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-stable-{Guid.NewGuid():N}"); + + try + { + Directory.CreateDirectory(releasePath); + await File.WriteAllTextAsync(Path.Combine(releasePath, "01-track.mp3"), "media"); + + var result = await DirectoryProcessorToStagingService.FindUnstableSourceFileAsync( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }, + delayMs: 10); + + Assert.Null(result); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + #endregion + + #region Converted Source Resolution Tests + + [Fact] + public void TryResolveSourceFileForStaging_WithOriginalFile_ReturnsOriginalFile() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-converted-{Guid.NewGuid():N}"); + var sourceFilePath = Path.Combine(releasePath, "01-track.flac"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(sourceFilePath, "source"); + + var result = DirectoryProcessorToStagingService.TryResolveSourceFileForStaging( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }, + new FileSystemFileInfo + { + Name = "01-track.flac", + OriginalName = "01-track.flac", + Size = 6 + }, + new Dictionary<string, FileSystemFileInfo>(StringComparer.OrdinalIgnoreCase), + new FileSystemService(Serializer), + out var sourceFile, + out var sourcePath); + + Assert.True(result); + Assert.Equal("01-track.flac", sourceFile.Name); + Assert.Equal(sourceFilePath, sourcePath); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + [Fact] + public void TryResolveSourceFileForStaging_WithConvertedFile_ReturnsConvertedFile() + { + var releasePath = Path.Combine(Path.GetTempPath(), $"melodee-converted-{Guid.NewGuid():N}"); + var convertedFilePath = Path.Combine(releasePath, "01-track.mp3"); + + try + { + Directory.CreateDirectory(releasePath); + File.WriteAllText(Path.Combine(releasePath, "01-track.flac"), "source"); + File.WriteAllText(convertedFilePath, "converted"); + + var convertedFile = new FileSystemFileInfo + { + Name = "01-track.mp3", + OriginalName = "01-track.mp3", + Size = 9 + }; + var result = DirectoryProcessorToStagingService.TryResolveSourceFileForStaging( + new FileSystemDirectoryInfo + { + Path = releasePath, + Name = Path.GetFileName(releasePath) + }, + new FileSystemFileInfo + { + Name = "01-track.flac", + OriginalName = "01-track.flac", + Size = 6 + }, + new Dictionary<string, FileSystemFileInfo>(StringComparer.OrdinalIgnoreCase) + { + ["01-track.flac"] = convertedFile + }, + new FileSystemService(Serializer), + out var sourceFile, + out var sourcePath); + + Assert.True(result); + Assert.Equal("01-track.mp3", sourceFile.Name); + Assert.Equal(convertedFilePath, sourcePath); + } + finally + { + if (Directory.Exists(releasePath)) + { + Directory.Delete(releasePath, true); + } + } + } + + #endregion + + #region Script Safety Tests + + [Fact] + public async Task ProcessDirectoryAsync_WhenDirectoryDeleteScriptWouldDelete_DoesNotEvaluateDeleteEvent() + { + var rootPath = Path.Combine(Path.GetTempPath(), $"melodee-inbound-{Guid.NewGuid():N}"); + var stagingPath = Path.Combine(Path.GetTempPath(), $"melodee-staging-{Guid.NewGuid():N}"); + var releasePath = Path.Combine(rootPath, "Artist - Album"); + + try + { + Directory.CreateDirectory(releasePath); + Directory.CreateDirectory(stagingPath); + await File.WriteAllTextAsync(Path.Combine(releasePath, "01-track.mp3"), "not real audio"); + + var scriptOrchestrationService = new Mock<IScriptOrchestrationService>(); + scriptOrchestrationService + .Setup(x => x.EvaluateScriptForEventAsync( + It.IsAny<string>(), + It.IsAny<object>(), + It.IsAny<CancellationToken>())) + .ReturnsAsync(new ScriptEvaluationResult { Result = true, IsDefault = true }); + scriptOrchestrationService + .Setup(x => x.EvaluateScriptForEventAsync( + ScriptEventNames.DirectoryProcessingDelete, + It.IsAny<object>(), + It.IsAny<CancellationToken>())) + .ReturnsAsync(new ScriptEvaluationResult { Result = true, IsDefault = false, OnDeny = "delete" }); + + var safeDeleteService = new Mock<ISafeDeleteService>(MockBehavior.Strict); + var service = GetDirectoryProcessorService( + new FileSystemService(Serializer), + scriptOrchestrationService.Object, + new DenyActionHandlerFactory(safeDeleteService.Object, new SettingService(), Logger)); + + await service.InitializeAsync(TestsBase.NewPluginsConfiguration(), stagingPath, CancellationToken.None); + + await service.ProcessDirectoryAsync(new FileSystemDirectoryInfo + { + Path = rootPath, + Name = "inbound" + }, null, null); + + Assert.True(Directory.Exists(releasePath)); + scriptOrchestrationService.Verify(x => x.EvaluateScriptForEventAsync( + ScriptEventNames.DirectoryProcessingDelete, + It.IsAny<object>(), + It.IsAny<CancellationToken>()), Times.Never); + safeDeleteService.Verify(x => x.DeleteDirectoryAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never); + } + finally + { + if (Directory.Exists(rootPath)) + { + Directory.Delete(rootPath, true); + } + + if (Directory.Exists(stagingPath)) + { + Directory.Delete(stagingPath, true); + } + } + } + + [Fact] + public async Task ProcessDirectoryAsync_WhenStartScriptRequestsDelete_SkipsWithoutDeletingDirectory() + { + var rootPath = Path.Combine(Path.GetTempPath(), $"melodee-inbound-{Guid.NewGuid():N}"); + var stagingPath = Path.Combine(Path.GetTempPath(), $"melodee-staging-{Guid.NewGuid():N}"); + var releasePath = Path.Combine(rootPath, "Artist - Album"); + + try + { + Directory.CreateDirectory(releasePath); + Directory.CreateDirectory(stagingPath); + await File.WriteAllTextAsync(Path.Combine(releasePath, "01-track.mp3"), "not real audio"); + + var scriptOrchestrationService = new Mock<IScriptOrchestrationService>(); + scriptOrchestrationService + .Setup(x => x.EvaluateScriptForEventAsync( + It.IsAny<string>(), + It.IsAny<object>(), + It.IsAny<CancellationToken>())) + .ReturnsAsync(new ScriptEvaluationResult { Result = true, IsDefault = true }); + scriptOrchestrationService + .Setup(x => x.EvaluateScriptForEventAsync( + ScriptEventNames.DirectoryProcessingStart, + It.IsAny<object>(), + It.IsAny<CancellationToken>())) + .ReturnsAsync(new ScriptEvaluationResult { Result = false, IsDefault = false, OnDeny = "delete" }); + + var safeDeleteService = new Mock<ISafeDeleteService>(MockBehavior.Strict); + var service = GetDirectoryProcessorService( + new FileSystemService(Serializer), + scriptOrchestrationService.Object, + new DenyActionHandlerFactory(safeDeleteService.Object, new SettingService(), Logger)); + + await service.InitializeAsync(TestsBase.NewPluginsConfiguration(), stagingPath, CancellationToken.None); + + await service.ProcessDirectoryAsync(new FileSystemDirectoryInfo + { + Path = rootPath, + Name = "inbound" + }, null, null); + + Assert.True(Directory.Exists(releasePath)); + safeDeleteService.Verify(x => x.DeleteDirectoryAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never); + } + finally + { + if (Directory.Exists(rootPath)) + { + Directory.Delete(rootPath, true); + } + + if (Directory.Exists(stagingPath)) + { + Directory.Delete(stagingPath, true); + } + } + } + + #endregion + #region Dispose Tests [Fact] @@ -380,6 +901,25 @@ public async Task OnProcessingStart_WhenProcessing_RaisesStartEvent() #endregion + #region Conversion Throttle Tests + + [Theory] + [InlineData(0, 1)] + [InlineData(1, 1)] + [InlineData(2, 1)] + [InlineData(4, 2)] + [InlineData(32, 2)] + public void CalculateMaxConcurrentConversions_WithProcessorCount_ReturnsBoundedConcurrency( + int processorCount, + int expectedConcurrency) + { + var result = DirectoryProcessorToStagingService.CalculateMaxConcurrentConversions(processorCount); + + Assert.Equal(expectedConcurrency, result); + } + + #endregion + #region DirectoryProcessorResult Tests [Fact] diff --git a/tests/Melodee.Tests.Common/Services/Scanning/DirectoryRunContextTests.cs b/tests/Melodee.Tests.Common/Services/Scanning/DirectoryRunContextTests.cs index b0a1783cc..e618380e4 100644 --- a/tests/Melodee.Tests.Common/Services/Scanning/DirectoryRunContextTests.cs +++ b/tests/Melodee.Tests.Common/Services/Scanning/DirectoryRunContextTests.cs @@ -145,10 +145,23 @@ public void Constructor_CreatesValidCaches() using var context = new DirectoryRunContext(); Assert.NotNull(context.ArtistSearchCache); + Assert.NotNull(context.ForcedArtistSearchCache); Assert.NotNull(context.AlbumImageCache); Assert.NotNull(context.ApiThrottler); } + [Fact] + public void ForcedArtistSearchCache_WhenNormalCacheHasNegativeResult_RemainsSeparate() + { + using var context = new DirectoryRunContext(); + var query = new ArtistQuery { Name = "Shared Artist" }; + + context.ArtistSearchCache.AddNegative(query); + var found = context.ForcedArtistSearchCache.TryGet(query, out _, out _); + + Assert.False(found); + } + [Fact] public void AddPluginTime_AccumulatesTime() { @@ -158,7 +171,9 @@ public void AddPluginTime_AccumulatesTime() context.AddPluginTime(50); context.AddPluginTime(25); - // Implicitly tested via LogSummary + var summary = context.GetPerformanceSummary(); + + Assert.Equal(175, summary.PluginTimeMs); } [Fact] @@ -168,6 +183,39 @@ public void IncrementDirectoriesProcessed_IncrementsConcurrently() Parallel.For(0, 100, _ => context.IncrementDirectoriesProcessed()); - // Implicitly tested via LogSummary + var summary = context.GetPerformanceSummary(); + + Assert.Equal(100, summary.DirectoriesProcessed); + } + + [Fact] + public void GetPerformanceSummary_WithRecordedCounters_ReturnsSnapshot() + { + using var context = new DirectoryRunContext(); + + context.AddConversionTime(125); + context.AddCopyTime(75); + context.AddEnrichmentTime(250); + context.RecordArtistSearchPersistenceConflict(); + context.RecordArtistSearchPersistenceRetry(); + context.RecordArtistSearchPersistenceCorruption(); + context.RecordArtistSearchReadError(); + context.RecordArtistSearchReadCorruption(); + context.RecordAlbumSkippedRevalidation(); + context.RecordAlbumDeferredRevalidation(); + + var summary = context.GetPerformanceSummary(); + + Assert.Equal(125, summary.ConversionTimeMs); + Assert.Equal(1, summary.ConversionFilesProcessed); + Assert.Equal(75, summary.CopyTimeMs); + Assert.Equal(250, summary.EnrichmentTimeMs); + Assert.Equal(1, summary.ArtistSearchPersistenceConflicts); + Assert.Equal(1, summary.ArtistSearchPersistenceRetries); + Assert.Equal(1, summary.ArtistSearchPersistenceCorruptions); + Assert.Equal(1, summary.ArtistSearchReadErrors); + Assert.Equal(1, summary.ArtistSearchReadCorruptions); + Assert.Equal(1, summary.AlbumsSkippedRevalidation); + Assert.Equal(1, summary.AlbumsDeferredRevalidation); } } diff --git a/tests/Melodee.Tests.Common/Services/Scanning/StagingAlbumRevalidationStateStoreTests.cs b/tests/Melodee.Tests.Common/Services/Scanning/StagingAlbumRevalidationStateStoreTests.cs new file mode 100644 index 000000000..3738a76c4 --- /dev/null +++ b/tests/Melodee.Tests.Common/Services/Scanning/StagingAlbumRevalidationStateStoreTests.cs @@ -0,0 +1,177 @@ +using FluentAssertions; +using Melodee.Common.Enums; +using Melodee.Common.Models; +using Melodee.Common.Services.Scanning; +using Serilog; + +namespace Melodee.Tests.Common.Services.Scanning; + +public sealed class StagingAlbumRevalidationStateStoreTests : IDisposable +{ + private readonly string _stagingPath; + private readonly StagingAlbumRevalidationStateStore _store; + + public StagingAlbumRevalidationStateStoreTests() + { + _stagingPath = Path.Combine(Path.GetTempPath(), $"melodee-revalidation-state-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_stagingPath); + _store = new StagingAlbumRevalidationStateStore(Log.Logger); + } + + [Fact] + public async Task OpenAsync_AfterAttemptRecorded_DefersAlbumUntilNextAttempt() + { + var now = new DateTimeOffset(2026, 5, 25, 12, 0, 0, TimeSpan.Zero); + var album = CreateAlbum(_stagingPath); + + await using (var session = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None)) + { + session.GetDecision(album, now, force: false).IsDue.Should().BeTrue(); + session.RecordAttempt(album, now, "ArtistLookupNoMatch"); + await session.SaveChangesAsync(CancellationToken.None); + } + + await using var reopened = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None); + var decision = reopened.GetDecision(album, now.AddMinutes(5), force: false); + + decision.IsDue.Should().BeFalse(); + decision.AttemptCount.Should().Be(1); + decision.NextAttemptAt.Should().Be(StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(1, now)); + } + + [Fact] + public async Task OpenAsync_CreatesHiddenDatabaseFileInStagingRoot() + { + var album = CreateAlbum(_stagingPath); + + await using (var session = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None)) + { + await session.SaveChangesAsync(CancellationToken.None); + } + + var databasePath = StagingAlbumRevalidationStateStore.GetDatabasePath(_stagingPath); + + File.Exists(databasePath).Should().BeTrue(); + Path.GetFileName(databasePath).Should().Be(StagingAlbumRevalidationStateStore.DatabaseFileName); + } + + [Fact] + public async Task GetDecision_WhenAlbumFingerprintChanged_ReturnsDue() + { + var now = new DateTimeOffset(2026, 5, 25, 12, 0, 0, TimeSpan.Zero); + var album = CreateAlbum(_stagingPath); + + await using (var session = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None)) + { + session.RecordAttempt(album, now, "ArtistLookupNoMatch"); + await session.SaveChangesAsync(CancellationToken.None); + } + + album.Artist = new Artist("Corrected Artist", "CORRECTEDARTIST", "Corrected Artist"); + + await using var reopened = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None); + var decision = reopened.GetDecision(album, now.AddMinutes(5), force: false); + + decision.IsDue.Should().BeTrue(); + decision.Reason.Should().Be("AlbumChanged"); + } + + [Fact] + public void CalculateNextAttemptAt_UsesBoundedBackoff() + { + var now = new DateTimeOffset(2026, 5, 25, 12, 0, 0, TimeSpan.Zero); + + StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(1, now).Should().Be(now.AddHours(6)); + StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(2, now).Should().Be(now.AddHours(12)); + StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(3, now).Should().Be(now.AddDays(1)); + StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(4, now).Should().Be(now.AddDays(3)); + StagingAlbumRevalidationStateStore.CalculateNextAttemptAt(99, now).Should().Be(now.AddDays(7)); + } + + [Fact] + public async Task OpenAsync_RemovesStatesForAlbumsNoLongerInStaging() + { + var now = new DateTimeOffset(2026, 5, 25, 12, 0, 0, TimeSpan.Zero); + var album = CreateAlbum(_stagingPath, "Artist - [2026] Album One"); + var remainingAlbum = CreateAlbum(_stagingPath, "Artist - [2026] Album Two"); + + await using (var session = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None)) + { + session.RecordAttempt(album, now, "ArtistLookupNoMatch"); + await session.SaveChangesAsync(CancellationToken.None); + } + + await using (var session = await _store.OpenAsync(_stagingPath, [remainingAlbum], CancellationToken.None)) + { + await session.SaveChangesAsync(CancellationToken.None); + } + + await using var reopened = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None); + var decision = reopened.GetDecision(album, now.AddMinutes(5), force: false); + + decision.IsDue.Should().BeTrue(); + decision.Reason.Should().Be("NoState"); + } + + [Fact] + public async Task OpenAsync_WhenStateDatabaseIsInvalid_RecreatesDatabase() + { + var album = CreateAlbum(_stagingPath); + var databasePath = StagingAlbumRevalidationStateStore.GetDatabasePath(_stagingPath); + await File.WriteAllTextAsync(databasePath, "this is not a valid decentdb file"); + + await using var session = await _store.OpenAsync(_stagingPath, [album], CancellationToken.None); + var decision = session.GetDecision( + album, + new DateTimeOffset(2026, 5, 25, 12, 0, 0, TimeSpan.Zero), + force: false); + + decision.IsDue.Should().BeTrue(); + decision.Reason.Should().Be("NoState"); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_stagingPath)) + { + Directory.Delete(_stagingPath, recursive: true); + } + } + catch + { + // Best effort cleanup. + } + } + + private static Album CreateAlbum(string stagingPath, string directoryName = "Artist - [2026] Album") + { + var albumPath = Path.Combine(stagingPath, directoryName); + return new Album + { + Id = Guid.NewGuid(), + AlbumType = AlbumType.Album, + Artist = new Artist("Artist", "ARTIST", "Artist"), + Directory = new FileSystemDirectoryInfo + { + Path = albumPath, + Name = directoryName + }, + OriginalDirectory = new FileSystemDirectoryInfo + { + Path = albumPath, + Name = directoryName + }, + Status = AlbumStatus.Invalid, + StatusReasons = AlbumNeedsAttentionReasons.HasInvalidArtists, + ViaPlugins = [], + Tags = + [ + new MetaTag<object?> { Identifier = MetaTagIdentifier.AlbumArtist, Value = "Artist" }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.Album, Value = "Album" }, + new MetaTag<object?> { Identifier = MetaTagIdentifier.RecordingYear, Value = "2026" } + ] + }; + } +} diff --git a/tests/Melodee.Tests.Common/Services/ScriptEvaluation/DirectoryDeletionScriptTests.cs b/tests/Melodee.Tests.Common/Services/ScriptEvaluation/DirectoryDeletionScriptTests.cs index 0025a0827..a37ffcfb0 100644 --- a/tests/Melodee.Tests.Common/Services/ScriptEvaluation/DirectoryDeletionScriptTests.cs +++ b/tests/Melodee.Tests.Common/Services/ScriptEvaluation/DirectoryDeletionScriptTests.cs @@ -9,11 +9,9 @@ namespace Melodee.Tests.Common.Services.ScriptEvaluation; /// <summary> -/// Critical tests for directory deletion script logic. -/// These tests ensure that directories are ONLY deleted when the script explicitly returns true -/// and that directories are preserved in all other cases (errors, disabled scripts, default behavior). -/// -/// PRODUCTION SAFETY: These tests are essential to prevent accidental data loss. +/// Tests for legacy directoryProcessingDelete script evaluation. +/// The ingestion processor must not physically delete release directories based on this event. +/// These tests keep the script evaluator behavior covered for existing configuration compatibility. /// </summary> [Collection("ScriptEvaluation")] public class DirectoryDeletionScriptTests @@ -35,7 +33,7 @@ private static ScriptEvaluationService CreateEvaluationService(ILogger logger) } /// <summary> - /// The actual production delete script that checks for insufficient files or media. + /// Legacy delete script shape that checks for insufficient files or media. /// </summary> private const string ProductionDeleteScript = @" function check(ctx, scriptConfig) { @@ -520,31 +518,6 @@ public async Task Orchestration_WithScriptError_IsDefaultShouldBeTrue() #endregion - #region Delete Decision Logic Tests - - /// <summary> - /// This test simulates the exact condition in DirectoryProcessorToStagingService: - /// if (deleteResult.Result && !deleteResult.IsDefault) - /// </summary> - [Theory] - [InlineData(true, false, true, "Script returns true, not default = DELETE")] - [InlineData(true, true, false, "Script returns true but is default = DO NOT DELETE")] - [InlineData(false, false, false, "Script returns false = DO NOT DELETE")] - [InlineData(false, true, false, "Script returns false and is default = DO NOT DELETE")] - public void DeleteDecisionLogic_CorrectlyDeterminesWhetherToDelete( - bool scriptResult, - bool isDefault, - bool expectedShouldDelete, - string scenario) - { - // This is the exact condition from DirectoryProcessorToStagingService.cs line 1215 - var shouldDelete = scriptResult && !isDefault; - - shouldDelete.Should().Be(expectedShouldDelete, scenario); - } - - #endregion - #region Context Property Access Tests [Fact] diff --git a/tests/Melodee.Tests.Common/Services/SearchEngines/AlbumImageSearchEngineServiceTests.cs b/tests/Melodee.Tests.Common/Services/SearchEngines/AlbumImageSearchEngineServiceTests.cs index b9c5994c3..502169a09 100644 --- a/tests/Melodee.Tests.Common/Services/SearchEngines/AlbumImageSearchEngineServiceTests.cs +++ b/tests/Melodee.Tests.Common/Services/SearchEngines/AlbumImageSearchEngineServiceTests.cs @@ -1,4 +1,16 @@ +using Melodee.Common.Configuration; +using Melodee.Common.Data; +using Melodee.Common.Models; using Melodee.Common.Models.SearchEngines; +using Melodee.Common.Plugins.SearchEngine; +using Melodee.Common.Plugins.SearchEngine.MusicBrainz.Data; +using Melodee.Common.Plugins.SearchEngine.Spotify; +using Melodee.Common.Serialization; +using Melodee.Common.Services; +using Melodee.Common.Services.Caching; +using Melodee.Common.Services.SearchEngines; +using Microsoft.EntityFrameworkCore; +using Serilog; namespace Melodee.Tests.Common.Services.SearchEngines; @@ -329,4 +341,122 @@ public async Task DoSearchAsync_WithDifferentCountry_HandlesCountrySpecificSearc Assert.NotNull(result.Data); } + [Fact] + public async Task DoSearchAsync_WithMultipleEnabledPlugins_StartsProvidersConcurrently() + { + var releaseSearches = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously); + var firstPlugin = new BlockingAlbumImageSearchEnginePlugin("First", 10, 1, releaseSearches); + var secondPlugin = new BlockingAlbumImageSearchEnginePlugin("Second", 20, 2, releaseSearches); + var service = GetAlbumImageSearchEngineServiceWithPlugins(firstPlugin, secondPlugin); + var query = new AlbumQuery + { + Name = "Parallel Album", + Artist = "Parallel Artist", + Year = 2024 + }; + + var searchTask = service.DoSearchAsync(query, 1); + + await Task.WhenAll(firstPlugin.Started, secondPlugin.Started).WaitAsync(TimeSpan.FromSeconds(1)); + releaseSearches.SetResult(true); + + var result = await searchTask; + + Assert.True(result.IsSuccess); + Assert.NotNull(result.Data); + Assert.Single(result.Data); + Assert.Equal("Second", result.Data[0].FromPlugin); + } + + private AlbumImageSearchEngineService GetAlbumImageSearchEngineServiceWithPlugins( + params IAlbumImageSearchEnginePlugin[] searchEngines) + { + return new TestAlbumImageSearchEngineService( + Logger, + CacheManager, + Serializer, + MockSettingService(), + MockConfigurationFactory(), + MockFactory(), + GetMusicBrainzRepository(), + MockSpotifyClientBuilder(), + MockHttpClientFactory(), + searchEngines); + } + + private sealed class TestAlbumImageSearchEngineService( + ILogger logger, + ICacheManager cacheManager, + ISerializer serializer, + SettingService settingService, + IMelodeeConfigurationFactory configurationFactory, + IDbContextFactory<MelodeeDbContext> contextFactory, + IMusicBrainzRepository musicBrainzRepository, + ISpotifyClientBuilder spotifyClientBuilder, + IHttpClientFactory httpClientFactory, + IReadOnlyCollection<IAlbumImageSearchEnginePlugin> searchEngines) + : AlbumImageSearchEngineService( + logger, + cacheManager, + serializer, + settingService, + configurationFactory, + contextFactory, + musicBrainzRepository, + spotifyClientBuilder, + httpClientFactory) + { + protected override IReadOnlyCollection<IAlbumImageSearchEnginePlugin> CreateSearchEngines(IMelodeeConfiguration configuration) + { + return searchEngines; + } + } + + private sealed class BlockingAlbumImageSearchEnginePlugin( + string displayName, + short rank, + long uniqueId, + TaskCompletionSource<bool> releaseSearch) : IAlbumImageSearchEnginePlugin + { + private readonly TaskCompletionSource<bool> _started = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Started => _started.Task; + + public bool StopProcessing => false; + + public string Id => uniqueId.ToString(); + + public string DisplayName => displayName; + + public bool IsEnabled { get; set; } = true; + + public int SortOrder { get; } = 1; + + public async Task<OperationResult<ImageSearchResult[]?>> DoAlbumImageSearch( + AlbumQuery query, + int maxResults, + CancellationToken cancellationToken = default) + { + _started.SetResult(true); + await releaseSearch.Task.WaitAsync(cancellationToken); + + return new OperationResult<ImageSearchResult[]?> + { + Data = + [ + new ImageSearchResult + { + FromPlugin = displayName, + Rank = rank, + UniqueId = uniqueId, + Width = 600, + Height = 600, + ThumbnailUrl = $"https://example.com/{uniqueId}.jpg", + MediaUrl = $"https://example.com/{uniqueId}.jpg", + Title = displayName + } + ] + }; + } + } } diff --git a/tests/Melodee.Tests.Common/Services/SearchEngines/ArtistSearchEngineServiceTests.cs b/tests/Melodee.Tests.Common/Services/SearchEngines/ArtistSearchEngineServiceTests.cs index a593c197f..1915013df 100644 --- a/tests/Melodee.Tests.Common/Services/SearchEngines/ArtistSearchEngineServiceTests.cs +++ b/tests/Melodee.Tests.Common/Services/SearchEngines/ArtistSearchEngineServiceTests.cs @@ -1,6 +1,15 @@ +using System.Net; +using System.Text; using Melodee.Blazor.Controllers.Melodee.Models.ArtistLookup; +using Melodee.Common.Configuration; +using Melodee.Common.Constants; +using Melodee.Common.Enums; +using Melodee.Common.Filtering; using Melodee.Common.Models; using Melodee.Common.Models.SearchEngines; +using Melodee.Common.Services.SearchEngines; +using Microsoft.EntityFrameworkCore; +using Moq; using Album = Melodee.Common.Models.SearchEngines.ArtistSearchEngineServiceData.Album; using Artist = Melodee.Common.Models.SearchEngines.ArtistSearchEngineServiceData.Artist; @@ -57,6 +66,384 @@ public async Task DoSearchAsync_WithEmptyQuery_ReturnsResults() Assert.NotNull(result); } + [Fact] + public async Task DoSearchAsync_WithBypassNegativeCache_SearchesAfterCachedMiss() + { + var service = GetArtistSearchEngineService(); + await service.InitializeAsync(); + + var artistName = $"Cache Bypass Artist {Guid.NewGuid():N}"; + var query = new ArtistQuery { Name = artistName }; + + var cachedMiss = await service.DoSearchAsync(query, 10); + Assert.Empty(cachedMiss.Data ?? []); + + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.Add(new Artist + { + Name = artistName, + NameNormalized = query.NameNormalized, + SortName = artistName + }); + await context.SaveChangesAsync(); + } + + var stillCachedMiss = await service.DoSearchAsync(query, 10); + Assert.Empty(stillCachedMiss.Data ?? []); + + var bypassResult = await service.DoSearchAsync(query, 10, bypassNegativeCache: true); + var artist = Assert.Single(bypassResult.Data ?? []); + Assert.Equal(artistName, artist.Name); + } + + [Fact] + public async Task DoSearchAsync_PassesBoundedMaxResultsToExternalProviders() + { + Uri? requestedUri = null; + var handler = new HttpHandlerStubDelegate((request, _) => + { + requestedUri = request.RequestUri; + const string json = """{"resultCount":0,"results":[]}"""; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + }); + using var httpClient = new HttpClient(handler); + var configFactory = SearchConfigurationFactory(new Dictionary<string, object?> + { + [SettingRegistry.SearchEngineMusicBrainzEnabled] = "false", + [SettingRegistry.SearchEngineSpotifyEnabled] = "false", + [SettingRegistry.SearchEngineITunesEnabled] = "true", + [SettingRegistry.SearchEngineLastFmEnabled] = "false", + [SettingRegistry.SearchEngineDiscogsEnabled] = "false", + [SettingRegistry.SearchEngineWikiDataEnabled] = "false" + }); + var service = CreateArtistSearchEngineService( + configFactory, + new TestHttpClientFactory(httpClient)); + await service.InitializeAsync(); + + await service.DoSearchAsync(new ArtistQuery { Name = "Bounded Provider Limit Artist" }, 1, bypassNegativeCache: true); + + Assert.Contains("limit=1", requestedUri?.ToString()); + } + + [Fact] + public async Task DoSearchAsync_WithExactLocalNameHit_DoesNotCallExternalHttpProvider() + { + var requestCount = 0; + var handler = new HttpHandlerStubDelegate((_, _) => + { + requestCount++; + const string json = """{"resultCount":0,"results":[]}"""; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + }); + using var httpClient = new HttpClient(handler); + var configFactory = SearchConfigurationFactory(new Dictionary<string, object?> + { + [SettingRegistry.SearchEngineMusicBrainzEnabled] = "false", + [SettingRegistry.SearchEngineSpotifyEnabled] = "false", + [SettingRegistry.SearchEngineITunesEnabled] = "true", + [SettingRegistry.SearchEngineLastFmEnabled] = "false", + [SettingRegistry.SearchEngineDiscogsEnabled] = "false", + [SettingRegistry.SearchEngineWikiDataEnabled] = "false" + }); + var service = CreateArtistSearchEngineService( + configFactory, + new TestHttpClientFactory(httpClient)); + await service.InitializeAsync(); + + var localArtist = new Artist + { + Name = "Local Exact Artist", + NameNormalized = "LOCALEXACTARTIST", + SortName = "Local Exact Artist" + }; + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.Add(localArtist); + await context.SaveChangesAsync(); + context.Albums.Add(NewAlbum(localArtist, "Local Exact Album", 2026)); + await context.SaveChangesAsync(); + } + + var result = await service.DoSearchAsync( + new ArtistQuery { Name = "Local Exact Artist" }, + 10, + bypassNegativeCache: true); + + var artist = Assert.Single(result.Data ?? []); + Assert.Equal("Local Exact Artist", artist.Name); + Assert.Equal(0, requestCount); + } + + [Fact] + public async Task DoSearchAsync_WithExactLocalAliasHit_DoesNotCallExternalHttpProvider() + { + var requestCount = 0; + var handler = new HttpHandlerStubDelegate((_, _) => + { + requestCount++; + const string json = """{"resultCount":0,"results":[]}"""; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + }); + using var httpClient = new HttpClient(handler); + var configFactory = SearchConfigurationFactory(new Dictionary<string, object?> + { + [SettingRegistry.SearchEngineMusicBrainzEnabled] = "false", + [SettingRegistry.SearchEngineSpotifyEnabled] = "false", + [SettingRegistry.SearchEngineITunesEnabled] = "true", + [SettingRegistry.SearchEngineLastFmEnabled] = "false", + [SettingRegistry.SearchEngineDiscogsEnabled] = "false", + [SettingRegistry.SearchEngineWikiDataEnabled] = "false" + }); + var service = CreateArtistSearchEngineService( + configFactory, + new TestHttpClientFactory(httpClient)); + + var localArtist = new Artist + { + Name = "Canonical Artist", + NameNormalized = "CANONICALARTIST", + SortName = "Canonical Artist", + AlternateNames = "EXACTLOCALALIAS" + }; + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.Add(localArtist); + await context.SaveChangesAsync(); + context.Albums.Add(NewAlbum(localArtist, "Alias Album", 2026)); + await context.SaveChangesAsync(); + } + + await service.InitializeAsync(); + + var result = await service.DoSearchAsync( + new ArtistQuery { Name = "Exact Local Alias" }, + 10, + bypassNegativeCache: true); + + var artist = Assert.Single(result.Data ?? []); + Assert.Equal("Canonical Artist", artist.Name); + Assert.Equal(0, requestCount); + } + + [Fact] + public async Task DoSearchAsync_WithProviderResultMatchingExistingItunesId_DoesNotCreateDuplicateArtist() + { + var handler = new HttpHandlerStubDelegate((_, _) => + { + const string json = """ + { + "resultCount": 1, + "results": [ + { + "artistId": 12345, + "artistName": "Provider Match", + "artistType": "Artist", + "wrapperType": "artist", + "trackCount": 3 + } + ] + } + """; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }); + }); + using var httpClient = new HttpClient(handler); + var configFactory = SearchConfigurationFactory(new Dictionary<string, object?> + { + [SettingRegistry.SearchEngineMusicBrainzEnabled] = "false", + [SettingRegistry.SearchEngineSpotifyEnabled] = "false", + [SettingRegistry.SearchEngineITunesEnabled] = "true", + [SettingRegistry.SearchEngineLastFmEnabled] = "false", + [SettingRegistry.SearchEngineDiscogsEnabled] = "false", + [SettingRegistry.SearchEngineWikiDataEnabled] = "false" + }); + var service = CreateArtistSearchEngineService( + configFactory, + new TestHttpClientFactory(httpClient)); + await service.InitializeAsync(); + + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.Add(new Artist + { + Name = "Existing Provider Artist", + NameNormalized = "EXISTINGPROVIDERARTIST", + SortName = "Existing Provider Artist", + ItunesId = "12345" + }); + await context.SaveChangesAsync(); + } + + var result = await service.DoSearchAsync( + new ArtistQuery { Name = "Provider Match" }, + 10, + bypassNegativeCache: true); + + var artist = Assert.Single(result.Data ?? []); + Assert.Equal("Existing Provider Artist", artist.Name); + + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + Assert.Equal(1, await context.Artists.CountAsync()); + } + } + + #endregion + + #region Performance Helper Tests + + [Fact] + public void GetCompoundArtistFallbackNames_WithCompoundArtist_ReturnsParts() + { + var result = ArtistSearchEngineService.GetCompoundArtistFallbackNames("Artist One feat. Artist Two"); + + Assert.Equal(["Artist One", "Artist Two"], result); + } + + [Fact] + public void GetCompoundArtistFallbackNames_WithSingleArtist_ReturnsEmpty() + { + var result = ArtistSearchEngineService.GetCompoundArtistFallbackNames("Earth Wind Fire"); + + Assert.Empty(result); + } + + [Fact] + public void ShouldUseCompoundArtistFallbackResult_WithTrustedMatchingRelease_ReturnsTrue() + { + var candidate = new ArtistSearchResult + { + Name = "Artist One", + FromPlugin = "Test", + SpotifyId = "spotify:artist-one", + Releases = + [ + new AlbumSearchResult + { + AlbumType = AlbumType.Album, + Name = "Shared Release", + NameNormalized = "SHAREDRELEASE", + SortName = "Shared Release", + ReleaseDate = "2026-01-01" + } + ] + }; + var query = new ArtistQuery + { + Name = "Artist One feat. Artist Two", + AlbumKeyValues = [new KeyValue("2026", "SHAREDRELEASE")] + }; + + var result = ArtistSearchEngineService.ShouldUseCompoundArtistFallbackResult(candidate, query); + + Assert.True(result); + } + + [Fact] + public void ShouldUseCompoundArtistFallbackResult_WithoutTrustedIdentifier_ReturnsFalse() + { + var candidate = new ArtistSearchResult + { + Name = "Artist One", + FromPlugin = "Test", + Releases = + [ + new AlbumSearchResult + { + AlbumType = AlbumType.Album, + Name = "Shared Release", + NameNormalized = "SHAREDRELEASE", + SortName = "Shared Release", + ReleaseDate = "2026-01-01" + } + ] + }; + var query = new ArtistQuery + { + Name = "Artist One feat. Artist Two", + AlbumKeyValues = [new KeyValue("2026", "SHAREDRELEASE")] + }; + + var result = ArtistSearchEngineService.ShouldUseCompoundArtistFallbackResult(candidate, query); + + Assert.False(result); + } + + [Fact] + public void ShouldAttemptCompoundArtistFallback_WithForcedRevalidation_ReturnsFalse() + { + var query = new ArtistQuery + { + Name = "Artist One feat. Artist Two", + AlbumKeyValues = [new KeyValue("2026", "SHAREDRELEASE")] + }; + + var result = ArtistSearchEngineService.ShouldAttemptCompoundArtistFallback( + query, + bypassNegativeCache: true); + + Assert.False(result); + } + + [Fact] + public void ShouldAttemptCompoundArtistFallback_WithAlbumEvidenceAndNormalSearch_ReturnsTrue() + { + var query = new ArtistQuery + { + Name = "Artist One feat. Artist Two", + AlbumKeyValues = [new KeyValue("2026", "SHAREDRELEASE")] + }; + + var result = ArtistSearchEngineService.ShouldAttemptCompoundArtistFallback( + query, + bypassNegativeCache: false); + + Assert.True(result); + } + + [Fact] + public void IsDecentDbTransientTransactionConflict_WithConflictMessage_ReturnsTrue() + { + var exception = new InvalidOperationException("DecentDB error 4: transaction conflict"); + + var result = ArtistSearchEngineService.IsDecentDbTransientTransactionConflict(exception); + + Assert.True(result); + } + + [Fact] + public void IsDecentDbCorruption_WithChecksumMessage_ReturnsTrue() + { + var exception = new InvalidOperationException("DecentDB checksum mismatch detected"); + + var result = ArtistSearchEngineService.IsDecentDbCorruption(exception); + + Assert.True(result); + } + + [Fact] + public void IsDecentDbCorruption_WithDatabaseCorruptionMessage_ReturnsTrue() + { + var exception = new InvalidOperationException("DecentDB error 2: database corruption: page WAL frame used page id 0"); + + var result = ArtistSearchEngineService.IsDecentDbCorruption(exception); + + Assert.True(result); + } + #endregion #region ListAsync Tests @@ -112,6 +499,94 @@ public async Task ListAsync_WithAlbums_ReturnsPagedArtistsWithAlbumCounts() Assert.Equal(1, artists[1].AlbumCount); } + [Fact] + public async Task ListAsync_WithTotalCountOnlyRequest_ReturnsExactCountWithoutData() + { + var service = GetArtistSearchEngineService(); + await service.InitializeAsync(); + + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.AddRange( + new Artist + { + Name = "Count Artist One", + NameNormalized = "COUNTARTISTONE", + SortName = "Count Artist One" + }, + new Artist + { + Name = "Count Artist Two", + NameNormalized = "COUNTARTISTTWO", + SortName = "Count Artist Two" + }, + new Artist + { + Name = "Count Artist Three", + NameNormalized = "COUNTARTISTTHREE", + SortName = "Count Artist Three" + }); + await context.SaveChangesAsync(); + } + + var result = await service.ListAsync(new PagedRequest + { + Page = 1, + PageSize = 2, + IsTotalCountOnlyRequest = true + }); + + Assert.Equal(3, result.TotalCount); + Assert.Empty(result.Data ?? []); + } + + [Fact] + public async Task ListAsync_WithFilteredPage_ReturnsOnlyRequestedPage() + { + var service = GetArtistSearchEngineService(); + await service.InitializeAsync(); + + await using (var context = await MockArtistSearchEngineFactory().CreateDbContextAsync()) + { + context.Artists.AddRange( + new Artist + { + Name = "Alpha One", + NameNormalized = "ALPHAONE", + SortName = "Alpha One" + }, + new Artist + { + Name = "Alpha Two", + NameNormalized = "ALPHATWO", + SortName = "Alpha Two" + }, + new Artist + { + Name = "Beta One", + NameNormalized = "BETAONE", + SortName = "Beta One" + }); + await context.SaveChangesAsync(); + } + + var result = await service.ListAsync(new PagedRequest + { + Page = 1, + PageSize = 1, + FilterBy = + [ + new FilterOperatorInfo(nameof(Artist.NameNormalized), FilterOperator.Contains, "ALPHA") + ], + OrderBy = new Dictionary<string, string> { { nameof(Artist.Name), PagedRequest.OrderAscDirection } } + }); + + var artists = result.Data?.ToArray() ?? []; + Assert.Equal(2, result.TotalCount); + var artist = Assert.Single(artists); + Assert.Equal("Alpha One", artist.Name); + } + private static Album NewAlbum(Artist artist, string name, int year) { return new Album @@ -126,6 +601,38 @@ private static Album NewAlbum(Artist artist, string name, int year) }; } + private ArtistSearchEngineService CreateArtistSearchEngineService( + IMelodeeConfigurationFactory configFactory, + IHttpClientFactory httpClientFactory) + { + return new ArtistSearchEngineService( + Logger, + CacheManager, + MockSettingService(), + MockSpotifyClientBuilder(), + configFactory, + MockFactory(), + MockArtistSearchEngineFactory(), + GetMusicBrainzRepository(), + Serializer, + httpClientFactory); + } + + private static IMelodeeConfigurationFactory SearchConfigurationFactory( + IReadOnlyDictionary<string, object?> overrides) + { + var settings = TestsBase.NewConfiguration(); + foreach (var (key, value) in overrides) + { + settings[key] = value; + } + + var mock = new Mock<IMelodeeConfigurationFactory>(); + mock.Setup(f => f.GetConfigurationAsync(It.IsAny<CancellationToken>())) + .ReturnsAsync(new MelodeeConfiguration(settings)); + return mock.Object; + } + #endregion #region DoArtistTopSongsSearchAsync Tests diff --git a/tests/Melodee.Tests.Common/TestsBase.cs b/tests/Melodee.Tests.Common/TestsBase.cs index a8a0938ac..192b78f62 100644 --- a/tests/Melodee.Tests.Common/TestsBase.cs +++ b/tests/Melodee.Tests.Common/TestsBase.cs @@ -122,6 +122,7 @@ public static PagedResult<Library> TestLibraries() { SettingRegistry.OpenSubsonicServerSupportedVersion, "1.16.1" }, { SettingRegistry.OpenSubsonicServerType, "Melodee" }, { SettingRegistry.SystemBaseUrl, "http://localhost:5000" }, + { SettingRegistry.PluginEnabledBlackbeard, "true" }, { SettingRegistry.PluginEnabledCueSheet, "true" }, { SettingRegistry.PluginEnabledSimpleFileVerification, "true" }, { SettingRegistry.PluginEnabledM3u, "true" }, diff --git a/tests/Melodee.Tests.Common/Validation/AlbumValidatorTests.cs b/tests/Melodee.Tests.Common/Validation/AlbumValidatorTests.cs index c1a8eda47..b150677d6 100644 --- a/tests/Melodee.Tests.Common/Validation/AlbumValidatorTests.cs +++ b/tests/Melodee.Tests.Common/Validation/AlbumValidatorTests.cs @@ -363,6 +363,26 @@ public void ValidateAlbumWithMissingArtist() Assert.Equal(AlbumNeedsAttentionReasons.HasInvalidArtists | AlbumNeedsAttentionReasons.HasInvalidSongs | AlbumNeedsAttentionReasons.HasNoImages, validationResult.Data.AlbumStatusReasons); } + [Fact] + public void ValidateAlbum_WithItunesArtistIdWithoutSearchEngineResult_DoesNotFlagInvalidArtist() + { + var album = NewTestAlbum(); + var artistName = "iTunes Artist"; + album.Artist = new Artist( + artistName, + artistName.ToNormalizedString()!, + artistName) + { + ItunesId = "123456789" + }; + + var validator = new AlbumValidator(NewPluginsConfiguration()); + var validationResult = validator.ValidateAlbum(album); + + Assert.True(validationResult.IsSuccess); + Assert.False(validationResult.Data.AlbumStatusReasons.HasFlag(AlbumNeedsAttentionReasons.HasInvalidArtists)); + } + [Fact] public void ValidateAlbumWithInvalidYear() {