Add option to delete orphaned downloads

Closes #273

app/Desktop/Dialogs/Progress/ProgressDialog.axaml.cs

@@ -2,11 +2,15 @@ using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using Avalonia.Controls;
+using DHT.Desktop.Dialogs.Message;
+using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Dialogs.Progress;
 
 [SuppressMessage("ReSharper", "MemberCanBeInternal")]
 public sealed partial class ProgressDialog : Window {
+	private static readonly Log Log = Log.ForType<ProgressDialog>();
+	
 	internal static async Task Show(Window owner, string title, Func<ProgressDialog, IProgressCallback, Task> action) {
 		var taskCompletionSource = new TaskCompletionSource();
 		var dialog = new ProgressDialog();
@@ -84,6 +88,11 @@ public sealed partial class ProgressDialog : Window {
 	
 	public async Task ShowProgressDialog(Window owner) {
 		await ShowDialog(owner);
-		await progressTask;
+		try {
+			await progressTask;
+		} catch (Exception e) {
+			Log.Error(e);
+			await Dialog.ShowOk(owner, "Unexpected Error", e.Message);
+		}
 	}
 }

app/Desktop/Main/Pages/DownloadsPage.axaml

@@ -13,8 +13,8 @@
     </Design.DataContext>
 
     <UserControl.Styles>
-        <Style Selector="Expander">
-            <Setter Property="Margin" Value="0 5 0 0" />
+        <Style Selector="WrapPanel > Button">
+            <Setter Property="Margin" Value="0 0 10 10" />
         </Style>
         <Style Selector="DataGridColumnHeader">
             <Setter Property="FontWeight" Value="Medium" />
@@ -30,16 +30,17 @@
         </Style>
     </UserControl.Styles>
 
-    <StackPanel Orientation="Vertical" Spacing="20">
-        <StackPanel Orientation="Horizontal" Spacing="10">
+    <StackPanel Orientation="Vertical">
+        <WrapPanel Orientation="Horizontal">
             <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
             <Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed Downloads</Button>
-        </StackPanel>
-        <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
-        <TextBlock TextWrapping="Wrap">
-            Downloading state and filter settings are remembered per-database.
-        </TextBlock>
-        <StackPanel Orientation="Vertical" Spacing="12">
+            <Button Command="{Binding OnClickDeleteOrphanedDownloads}">Delete Orphaned Downloads</Button>
+        </WrapPanel>
+        <StackPanel Orientation="Vertical" Spacing="20" Margin="0 10 0 0">
+            <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
+            <TextBlock TextWrapping="Wrap">
+                Downloading state and filter settings are remembered per-database.
+            </TextBlock>
             <Expander Header="Download Status" IsExpanded="True">
                 <DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
                     <DataGrid.Columns>

app/Desktop/Main/Pages/DownloadsPageModel.cs

@@ -1,12 +1,17 @@
 using System;
+using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
+using Avalonia.Controls;
 using Avalonia.ReactiveUI;
 using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
+using DHT.Desktop.Dialogs.Message;
+using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
 using DHT.Server;
+using DHT.Server.Data;
 using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
 using DHT.Server.Data.Settings;
@@ -48,6 +53,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	
 	public bool IsDownloading => state.Downloader.IsDownloading;
 	
+	private readonly Window window;
 	private readonly State state;
 	private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
 	private readonly IDisposable downloadItemCountSubscription;
@@ -55,9 +61,10 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	private IDisposable? finishedItemsSubscription;
 	private DownloadItemFilter? currentDownloadFilter;
 	
-	public DownloadsPageModel() : this(State.Dummy) {}
+	public DownloadsPageModel() : this(null!, State.Dummy) {}
 	
-	public DownloadsPageModel(State state) {
+	public DownloadsPageModel(Window window, State state) {
+		this.window = window;
 		this.state = state;
 		
 		FilterModel = new DownloadItemFilterPanelModel(state);
@@ -158,6 +165,50 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
 	}
 	
+	private const string DeleteOrphanedDownloadsTitle = "Delete Orphaned Downloads";
+	
+	public async Task OnClickDeleteOrphanedDownloads() {
+		await ProgressDialog.Show(window, DeleteOrphanedDownloadsTitle, DeleteOrphanedDownloads);
+	}
+	
+	private async Task DeleteOrphanedDownloads(ProgressDialog dialog, IProgressCallback callback) {
+		await callback.UpdateIndeterminate("Searching for orphaned downloads...");
+		
+		HashSet<string> reachableNormalizedUrls = [];
+		HashSet<string> orphanedNormalizedUrls = [];
+		
+		await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
+			reachableNormalizedUrls.Add(download.NormalizedUrl);
+		}
+		
+		await foreach (Download download in state.Db.Downloads.Get()) {
+			string normalizedUrl = download.NormalizedUrl;
+			if (!reachableNormalizedUrls.Contains(normalizedUrl)) {
+				orphanedNormalizedUrls.Add(normalizedUrl);
+			}
+		}
+		
+		if (orphanedNormalizedUrls.Count == 0) {
+			await Dialog.ShowOk(window, DeleteOrphanedDownloadsTitle, "No orphaned downloads found.");
+			return;
+		}
+		
+		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
+			return;
+		}
+		
+		await callback.UpdateIndeterminate("Deleting orphaned downloads...");
+		await state.Db.Downloads.Remove(orphanedNormalizedUrls);
+		RecomputeDownloadStatistics();
+		
+		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
+			return;
+		}
+		
+		await callback.UpdateIndeterminate("Vacuuming database...");
+		await state.Db.Vacuum();
+	}
+	
 	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
 		statisticsPending.Items = statusStatistics.PendingCount;
 		statisticsPending.Size = statusStatistics.PendingTotalSize;

app/Desktop/Main/Screens/MainContentScreenModel.cs

@@ -49,7 +49,7 @@ sealed class MainContentScreenModel : IAsyncDisposable {
 		TrackingPageModel = new TrackingPageModel(window);
 		TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
 		
-		DownloadsPageModel = new DownloadsPageModel(state);
+		DownloadsPageModel = new DownloadsPageModel(window, state);
 		DownloadsPage = new DownloadsPage { DataContext = DownloadsPageModel };
 		
 		ViewerPageModel = new ViewerPageModel(window, state);

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -32,6 +32,10 @@ public interface IDownloadRepository {
 	
 	Task<int> RetryFailed(CancellationToken cancellationToken = default);
 	
+	Task Remove(ICollection<string> normalizedUrls);
+	
+	IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken = default);
+	
 	internal sealed class Dummy : IDownloadRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
 		
@@ -70,5 +74,13 @@ public interface IDownloadRepository {
 		public Task<int> RetryFailed(CancellationToken cancellationToken) {
 			return Task.FromResult(0);
 		}
+		
+		public Task Remove(ICollection<string> normalizedUrls) {
+			return Task.CompletedTask;
+		}
+		
+		public IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken) {
+			return AsyncEnumerable.Empty<Data.Download>();
+		}
 	}
 }

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -321,4 +321,62 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 		return await cmd.ExecuteNonQueryAsync(cancellationToken);
 	}
+	
+	public async Task Remove(ICollection<string> normalizedUrls) {
+		await using (var conn = await pool.Take()) {
+			await conn.BeginTransactionAsync();
+			
+			await using (var cmd = conn.Command("DELETE FROM download_metadata WHERE normalized_url = :normalized_url")) {
+				cmd.Add(":normalized_url", SqliteType.Text);
+				
+				foreach (string normalizedUrl in normalizedUrls) {
+					cmd.Set(":normalized_url", normalizedUrl);
+					await cmd.ExecuteNonQueryAsync();
+				}
+			}
+			
+			await conn.CommitTransactionAsync();
+		}
+		
+		UpdateTotalCount();
+	}
+	
+	public async IAsyncEnumerable<Data.Download> FindAllDownloadableUrls([EnumeratorCancellation] CancellationToken cancellationToken = default) {
+		await using var conn = await pool.Take();
+		
+		await using (var cmd = conn.Command("SELECT normalized_url, download_url, type, size FROM attachments")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromAttachment(reader.GetString(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetUint64(3));
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT json FROM message_embeds")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				var result = await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0));
+				if (result is not null) {
+					yield return result;
+				}
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1));
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM message_reactions WHERE emoji_id IS NOT NULL")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1));
+			}
+		}
+	}
 }

app/Server/Download/DownloadLinkExtractor.cs

@@ -28,7 +28,11 @@ static class DownloadLinkExtractor {
 	}
 	
 	public static Data.Download FromAttachment(Attachment attachment) {
-		return new Data.Download(attachment.NormalizedUrl, attachment.DownloadUrl, DownloadStatus.Pending, attachment.Type, attachment.Size);
+		return FromAttachment(attachment.NormalizedUrl, attachment.DownloadUrl, attachment.Type, attachment.Size);
+	}
+	
+	public static Data.Download FromAttachment(string normalizedUrl, string downloadUrl, string? type, ulong size) {
+		return new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Pending, type, size);
 	}
 	
 	public static async Task<Data.Download?> TryFromEmbedJson(Stream jsonStream) {