Fix broken links to image embeds in viewer

Closes #256

app/.idea/.idea.DiscordHistoryTracker/.idea/avalonia.xml

@@ -9,15 +9,15 @@
         <entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Dialogs/TextBox/TextBoxDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
-        <entry key="Desktop/Main/Controls/AttachmentFilterPanel.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Controls/DownloadItemFilterPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
-        <entry key="Desktop/Main/Pages/AttachmentsPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/DebugPage.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Pages/DownloadsPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/TrackingPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/ViewerPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Screens/MainContentScreen.axaml" value="Desktop/Desktop.csproj" />

app/.idea/.idea.DiscordHistoryTracker/.idea/codeStyles/Project.xml

@@ -13,10 +13,10 @@
     <CssCodeStyleSettings>
       <option name="HEX_COLOR_LOWER_CASE" value="true" />
     </CssCodeStyleSettings>
-    <DB2CodeStyleSettings version="6">
+    <DB2CodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </DB2CodeStyleSettings>
-    <DerbyCodeStyleSettings version="6">
+    <DerbyCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </DerbyCodeStyleSettings>
     <GoCodeStyleSettings>
@@ -26,10 +26,10 @@
       <option name="WRAP_FUNC_PARAMS" value="5" />
       <option name="WRAP_FUNC_RESULT" value="5" />
     </GoCodeStyleSettings>
-    <H2CodeStyleSettings version="6">
+    <H2CodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </H2CodeStyleSettings>
-    <HSQLCodeStyleSettings version="6">
+    <HSQLCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </HSQLCodeStyleSettings>
     <HTMLCodeStyleSettings>
@@ -85,10 +85,10 @@
     <LessCodeStyleSettings>
       <option name="HEX_COLOR_LOWER_CASE" value="true" />
     </LessCodeStyleSettings>
-    <MSSQLCodeStyleSettings version="6">
+    <MSSQLCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </MSSQLCodeStyleSettings>
-    <MySQLCodeStyleSettings version="6">
+    <MySQLCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </MySQLCodeStyleSettings>
     <Objective-C>
@@ -98,7 +98,7 @@
       <option name="SPACE_BEFORE_INIT_LIST" value="true" />
       <option name="SPACE_AFTER_DICTIONARY_LITERAL_COLON" value="false" />
     </Objective-C>
-    <OracleCodeStyleSettings version="6">
+    <OracleCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </OracleCodeStyleSettings>
     <PHPCodeStyleSettings>
@@ -116,7 +116,7 @@
       <option name="NEW_LINE_AFTER_PHP_OPENING_TAG" value="true" />
       <option name="SPACE_AROUND_ASSIGNMENT_IN_DECLARE" value="true" />
     </PHPCodeStyleSettings>
-    <PostgresCodeStyleSettings version="6">
+    <PostgresCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </PostgresCodeStyleSettings>
     <Properties>
@@ -143,13 +143,16 @@
       <option name="INDENT_WHEN_CASES" value="true" />
       <option name="CHAIN_CALLS_ALIGNMENT" value="2" />
     </Ruby>
-    <SQLiteCodeStyleSettings version="6">
+    <SQLiteCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </SQLiteCodeStyleSettings>
     <ScssCodeStyleSettings>
       <option name="HEX_COLOR_LOWER_CASE" value="true" />
     </ScssCodeStyleSettings>
-    <SqlCodeStyleSettings version="6">
+    <Shell_Script>
+      <option name="KEEP_COLUMN_ALIGNMENT_PADDING" value="true" />
+    </Shell_Script>
+    <SqlCodeStyleSettings version="7">
       <option name="KEYWORD_CASE" value="2" />
       <option name="TYPE_CASE" value="2" />
       <option name="CUSTOM_TYPE_CASE" value="2" />
@@ -173,9 +176,10 @@
       <option name="CONSTRAINT_WRAP_3" value="true" />
       <option name="CONSTRAINT_WRAP_4" value="true" />
       <option name="VIEW_INDENT_QUERY" value="true" />
+      <option name="IMP_LOOP_LOOP_INDENT" value="false" />
       <option name="EXPR_CASE_WHEN_WRAP" value="false" />
     </SqlCodeStyleSettings>
-    <SybaseCodeStyleSettings version="6">
+    <SybaseCodeStyleSettings version="7">
       <option name="USE_GENERIC_STYLE" value="true" />
     </SybaseCodeStyleSettings>
     <TypeScriptCodeStyleSettings version="0">
@@ -203,6 +207,11 @@
         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
       </indentOptions>
     </codeStyleSettings>
+    <codeStyleSettings language="Gherkin">
+      <indentOptions>
+        <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
+      </indentOptions>
+    </codeStyleSettings>
     <codeStyleSettings language="Groovy">
       <option name="ELSE_ON_NEW_LINE" value="true" />
       <option name="SPACE_WITHIN_ARRAY_INITIALIZER_BRACES" value="true" />
@@ -282,6 +291,13 @@
         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
       </indentOptions>
     </codeStyleSettings>
+    <codeStyleSettings language="Markdown">
+      <indentOptions>
+        <option name="INDENT_SIZE" value="2" />
+        <option name="CONTINUATION_INDENT_SIZE" value="2" />
+        <option name="TAB_SIZE" value="2" />
+      </indentOptions>
+    </codeStyleSettings>
     <codeStyleSettings language="ObjectiveC">
       <option name="LINE_COMMENT_AT_FIRST_COLUMN" value="false" />
       <option name="BLOCK_COMMENT_AT_FIRST_COLUMN" value="false" />
@@ -342,7 +358,6 @@
       </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="Rust">
-      <option name="RIGHT_MARGIN" value="140" />
       <option name="ALIGN_MULTILINE_CHAINED_METHODS" value="true" />
       <indentOptions>
         <option name="USE_TAB_CHARACTER" value="true" />
@@ -357,6 +372,7 @@
     </codeStyleSettings>
     <codeStyleSettings language="SCSS">
       <indentOptions>
+        <option name="CONTINUATION_INDENT_SIZE" value="2" />
         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
       </indentOptions>
     </codeStyleSettings>
@@ -367,13 +383,6 @@
         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
       </indentOptions>
     </codeStyleSettings>
-    <codeStyleSettings language="Shell Script">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="4" />
-        <option name="TAB_SIZE" value="4" />
-        <option name="USE_TAB_CHARACTER" value="true" />
-      </indentOptions>
-    </codeStyleSettings>
     <codeStyleSettings language="TOML">
       <indentOptions>
         <option name="CONTINUATION_INDENT_SIZE" value="4" />
@@ -427,6 +436,9 @@
     </codeStyleSettings>
     <codeStyleSettings language="liquid">
       <indentOptions>
+        <option name="INDENT_SIZE" value="2" />
+        <option name="CONTINUATION_INDENT_SIZE" value="2" />
+        <option name="TAB_SIZE" value="2" />
         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" />
       </indentOptions>
     </codeStyleSettings>

app/.idea/.idea.DiscordHistoryTracker/.idea/inspectionProfiles/Project.xml

@@ -91,7 +91,6 @@
     <inspection_tool class="ComparatorNotSerializable" enabled="true" level="WARNING" enabled_by_default="true" />
     <inspection_tool class="CompareToUsesNonFinalVariable" enabled="true" level="WARNING" enabled_by_default="true" />
     <inspection_tool class="ComparisonOfShortAndChar" enabled="true" level="WARNING" enabled_by_default="true" />
-    <inspection_tool class="ComposeMissingKeys" enabled="true" level="ERROR" enabled_by_default="true" />
     <inspection_tool class="ConditionSignal" enabled="true" level="WARNING" enabled_by_default="true" />
     <inspection_tool class="ConditionalExpression" enabled="true" level="INFORMATION" enabled_by_default="true">
       <option name="ignoreSimpleAssignmentsAndReturns" value="true" />
@@ -362,7 +361,7 @@
     </inspection_tool>
     <inspection_tool class="LocalVariableDeclarationSideOnly" enabled="true" level="ERROR" enabled_by_default="true" />
     <inspection_tool class="LocalVariableNamingConventionJS" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="m_regex" value="[a-z][A-Za-z]*" />
+      <option name="m_regex" value="[a-z][A-Za-z0-9]*" />
       <option name="m_minLength" value="0" />
       <option name="m_maxLength" value="99" />
     </inspection_tool>

app/.idea/.idea.DiscordHistoryTracker/.idea/libraries/Generated_files.xml

@@ -1,22 +0,0 @@
-<component name="libraryTable">
-  <library name="Generated files" type="javaScript">
-    <properties>
-      <sourceFilesUrls>
-        <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/discord.js" />
-        <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/dom.js" />
-        <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/gui.js" />
-        <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/settings.js" />
-        <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/state.js" />
-      </sourceFilesUrls>
-    </properties>
-    <CLASSES>
-      <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/discord.js" />
-      <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/dom.js" />
-      <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/gui.js" />
-      <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/settings.js" />
-      <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/state.js" />
-    </CLASSES>
-    <JAVADOC />
-    <SOURCES />
-  </library>
-</component>

app/Desktop/Arguments.cs

@@ -15,6 +15,7 @@ sealed class Arguments {
 	public string? DatabaseFile { get; }
 	public ushort? ServerPort { get; }
 	public string? ServerToken { get; }
+	public byte? ConcurrentDownloads { get; }
 
 	public Arguments(IReadOnlyList<string> args) {
 		for (int i = FirstArgument; i < args.Count; i++) {
@@ -50,11 +51,11 @@ sealed class Arguments {
 					continue;
 
 				case "-port": {
-					if (ushort.TryParse(value, out var port)) {
+					if (!ushort.TryParse(value, out var port)) {
-						ServerPort = port;
+						Log.Warn("Invalid port number: " + value);
 					}
 					else {
-						Log.Warn("Invalid port number: " + value);
+						ServerPort = port;
 					}
 
 					continue;
@@ -64,6 +65,20 @@ sealed class Arguments {
 					ServerToken = value;
 					continue;
 				
+				case "-concurrentdownloads":
+					if (!ulong.TryParse(value, out var concurrentDownloads) || concurrentDownloads == 0) {
+						Log.Warn("Invalid concurrent downloads count: " + value);
+					}
+					else if (concurrentDownloads > 10) {
+						Log.Warn("Limiting concurrent downloads to 10");
+						ConcurrentDownloads = 10;
+					}
+					else {
+						ConcurrentDownloads = (byte) concurrentDownloads;
+					}
+					
+					continue;
+
 				default:
 					Log.Warn("Unknown command line argument: " + key);
 					break;

app/Desktop/Common/BytesValueConverter.cs

@@ -29,7 +29,7 @@ sealed class BytesValueConverter : IValueConverter {
 
 	private const int Scale = 1000;
 
-	private static string Convert(ulong size) {
+	public static string Convert(ulong size) {
 		int power = size == 0L ? 0 : (int) Math.Log(size, Scale);
 		int unit = power >= Units.Length ? Units.Length - 1 : power;
 		return Units[unit].Format(unit == 0 ? size : size / Math.Pow(Scale, unit));

app/Desktop/Common/DatabaseGui.cs

@@ -9,7 +9,7 @@ using DHT.Desktop.Dialogs.Message;
 using DHT.Server.Database;
 using DHT.Server.Database.Exceptions;
 using DHT.Server.Database.Sqlite;
-using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Database.Sqlite.Schema;
 using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Common;

app/Desktop/Desktop.csproj

@@ -32,10 +32,6 @@
   
   <ItemGroup>
     <Compile Include="..\Version.cs" Link="Version.cs" />
-    <Compile Update="Dialogs\TextBox\TextBoxDialog.axaml.cs">
-      <DependentUpon>CheckBoxDialog.axaml</DependentUpon>
-      <SubType>Code</SubType>
-    </Compile>
   </ItemGroup>
   
   <ItemGroup>
@@ -43,11 +39,6 @@
     <EmbeddedResource Include="Resources/tracker-loader.js">
       <LogicalName>tracker-loader.js</LogicalName>
     </EmbeddedResource>
-    <EmbeddedResource Include="../Resources/Viewer/**">
-      <LogicalName>Viewer\%(RecursiveDir)%(Filename)%(Extension)</LogicalName>
-      <Link>Resources/Viewer/%(RecursiveDir)%(Filename)%(Extension)</Link>
-      <Visible>false</Visible>
-    </EmbeddedResource>
   </ItemGroup>
   
 </Project>

app/Desktop/Main/Controls/DownloadItemFilterPanel.axaml

@@ -4,11 +4,11 @@
              xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
              xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
              mc:Ignorable="d"
-             x:Class="DHT.Desktop.Main.Controls.AttachmentFilterPanel"
+             x:Class="DHT.Desktop.Main.Controls.DownloadItemFilterPanel"
-             x:DataType="controls:AttachmentFilterPanelModel">
+             x:DataType="controls:DownloadItemFilterPanelModel">
 
     <Design.DataContext>
-        <controls:AttachmentFilterPanelModel />
+        <controls:DownloadItemFilterPanelModel />
     </Design.DataContext>
 
     <UserControl.Styles>

app/Desktop/Main/Controls/DownloadItemFilterPanel.axaml.cs

@@ -4,8 +4,8 @@ using Avalonia.Controls;
 namespace DHT.Desktop.Main.Controls;
 
 [SuppressMessage("ReSharper", "MemberCanBeInternal")]
-public sealed partial class AttachmentFilterPanel : UserControl {
+public sealed partial class DownloadItemFilterPanel : UserControl {
-	public AttachmentFilterPanel() {
+	public DownloadItemFilterPanel() {
 		InitializeComponent();
 	}
 }

app/Desktop/Main/Controls/DownloadItemFilterPanelModel.cs

@@ -12,7 +12,7 @@ using DHT.Utils.Tasks;
 
 namespace DHT.Desktop.Main.Controls;
 
-sealed partial class AttachmentFilterPanelModel : ObservableObject, IDisposable {
+sealed partial class DownloadItemFilterPanelModel : ObservableObject, IDisposable {
 	public sealed record Unit(string Name, uint Scale);
 
 	private static readonly Unit[] AllUnits = [
@@ -43,21 +43,21 @@ sealed partial class AttachmentFilterPanelModel : ObservableObject, IDisposable
 	private readonly State state;
 	private readonly string verb;
 
-	private readonly RestartableTask<long> matchingAttachmentCountTask;
+	private readonly RestartableTask<long> downloadItemCountTask;
-	private long? matchingAttachmentCount;
+	private long? matchingItemCount;
 	
-	private readonly IDisposable attachmentCountSubscription;
+	private readonly IDisposable downloadItemCountSubscription;
-	private long? totalAttachmentCount;
+	private long? totalItemCount;
 
 	[Obsolete("Designer")]
-	public AttachmentFilterPanelModel() : this(State.Dummy) {}
+	public DownloadItemFilterPanelModel() : this(State.Dummy) {}
 
-	public AttachmentFilterPanelModel(State state, string verb = "Matches") {
+	public DownloadItemFilterPanelModel(State state, string verb = "Matches") {
 		this.state = state;
 		this.verb = verb;
 
-		this.matchingAttachmentCountTask = new RestartableTask<long>(SetAttachmentCounts, TaskScheduler.FromCurrentSynchronizationContext());
+		this.downloadItemCountTask = new RestartableTask<long>(SetMatchingCount, TaskScheduler.FromCurrentSynchronizationContext());
-		this.attachmentCountSubscription = state.Db.Attachments.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnAttachmentCountChanged);
+		this.downloadItemCountSubscription = state.Db.Downloads.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnDownloadItemCountChanged);
 
 		UpdateFilterStatistics();
 
@@ -65,7 +65,8 @@ sealed partial class AttachmentFilterPanelModel : ObservableObject, IDisposable
 	}
 
 	public void Dispose() {
-		attachmentCountSubscription.Dispose();
+		downloadItemCountTask.Cancel();
+		downloadItemCountSubscription.Dispose();
 	}
 
 	private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
@@ -74,8 +75,8 @@ sealed partial class AttachmentFilterPanelModel : ObservableObject, IDisposable
 		}
 	}
 	
-	private void OnAttachmentCountChanged(long newAttachmentCount) {
+	private void OnDownloadItemCountChanged(long newItemCount) {
-		totalAttachmentCount = newAttachmentCount;
+		totalItemCount = newItemCount;
 		UpdateFilterStatistics();
 	}
 	
@@ -83,32 +84,32 @@ sealed partial class AttachmentFilterPanelModel : ObservableObject, IDisposable
 	private void UpdateFilterStatistics() {
 		var filter = CreateFilter();
 		if (filter.IsEmpty) {
-			matchingAttachmentCountTask.Cancel();
+			downloadItemCountTask.Cancel();
-			matchingAttachmentCount = totalAttachmentCount;
+			matchingItemCount = totalItemCount;
 			UpdateFilterStatisticsText();
 		}
 		else {
-			matchingAttachmentCount = null;
+			matchingItemCount = null;
 			UpdateFilterStatisticsText();
-			matchingAttachmentCountTask.Restart(cancellationToken => state.Db.Attachments.Count(filter, cancellationToken));
+			downloadItemCountTask.Restart(cancellationToken => state.Db.Downloads.Count(filter, cancellationToken));
 		}
 	}
 
-	private void SetAttachmentCounts(long matchingAttachmentCount) {
+	private void SetMatchingCount(long matchingAttachmentCount) {
-		this.matchingAttachmentCount = matchingAttachmentCount;
+		this.matchingItemCount = matchingAttachmentCount;
 		UpdateFilterStatisticsText();
 	}
 
 	private void UpdateFilterStatisticsText() {
-		var matchingAttachmentCountStr = matchingAttachmentCount?.Format() ?? "(...)";
+		var matchingItemCountStr = matchingItemCount?.Format() ?? "(...)";
-		var totalAttachmentCountStr = totalAttachmentCount?.Format() ?? "(...)";
+		var totalItemCountStr = totalItemCount?.Format() ?? "(...)";
 
-		FilterStatisticsText = verb + " " + matchingAttachmentCountStr + " out of " + totalAttachmentCountStr + " attachment" + (totalAttachmentCount is null or 1 ? "." : "s.");
+		FilterStatisticsText = verb + " " + matchingItemCountStr + " out of " + totalItemCountStr + " file" + (totalItemCount is null or 1 ? "." : "s.");
 		OnPropertyChanged(nameof(FilterStatisticsText));
 	}
 
-	public AttachmentFilter CreateFilter() {
+	public DownloadItemFilter CreateFilter() {
-		AttachmentFilter filter = new ();
+		DownloadItemFilter filter = new ();
 
 		if (LimitSize) {
 			try {

app/Desktop/Main/MainWindow.axaml.cs

@@ -1,9 +1,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
-using System.IO;
 using System.Threading.Tasks;
 using Avalonia.Controls;
-using DHT.Desktop.Main.Pages;
 using DHT.Utils.Logging;
 using JetBrains.Annotations;
 
@@ -43,13 +41,5 @@ public sealed partial class MainWindow : Window {
 				Log.Error("Caught exception while disposing window: " + ex);
 			}
 		}
-
-		foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {
-			try {
-				File.Delete(temporaryFile);
-			} catch (Exception) {
-				// ignored
-			}
-		}
 	}
 }

app/Desktop/Main/MainWindowModel.cs

@@ -30,6 +30,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
 	private MainContentScreenModel? mainContentScreenModel;
 
 	private readonly Window window;
+	private readonly int? concurrentDownloads;
 
 	private State? state;
 
@@ -73,6 +74,8 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
 		if (args.ServerToken != null) {
 			ServerConfiguration.Token = args.ServerToken;
 		}
+		
+		concurrentDownloads = args.ConcurrentDownloads;
 	}
 
 	private async void OnDatabaseSelected(object? sender, IDatabaseFile db) {
@@ -80,7 +83,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
 		
 		await DisposeState();
 		
-		state = new State(db);
+		state = new State(db, concurrentDownloads);
 
 		try {
 			await state.Server.Start(ServerConfiguration.Port, ServerConfiguration.Token);

app/Desktop/Main/Pages/AttachmentsPageModel.cs

@@ -1,244 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.Reactive.Linq;
-using System.Threading.Tasks;
-using Avalonia.ReactiveUI;
-using CommunityToolkit.Mvvm.ComponentModel;
-using DHT.Desktop.Common;
-using DHT.Desktop.Main.Controls;
-using DHT.Server;
-using DHT.Server.Data;
-using DHT.Server.Data.Aggregations;
-using DHT.Server.Data.Filters;
-using DHT.Utils.Logging;
-using DHT.Utils.Tasks;
-
-namespace DHT.Desktop.Main.Pages;
-
-sealed partial class AttachmentsPageModel : ObservableObject, IDisposable {
-	private static readonly Log Log = Log.ForType<AttachmentsPageModel>();
-
-	private static readonly DownloadItemFilter EnqueuedItemFilter = new () {
-		IncludeStatuses = new HashSet<DownloadStatus> {
-			DownloadStatus.Enqueued,
-			DownloadStatus.Downloading
-		}
-	};
-
-	[ObservableProperty(Setter = Access.Private)]
-	private bool isToggleDownloadButtonEnabled = true;
-
-	public string ToggleDownloadButtonText => IsDownloading ? "Stop Downloading" : "Start Downloading";
-
-	[ObservableProperty(Setter = Access.Private)]
-	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
-	private bool isRetryingFailedDownloads = false;
-
-	[ObservableProperty(Setter = Access.Private)]
-	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
-	private bool hasFailedDownloads;
-
-	public bool IsRetryFailedOnDownloadsButtonEnabled => !IsRetryingFailedDownloads && hasFailedDownloads;
-
-	[ObservableProperty(Setter = Access.Private)]
-	private string downloadMessage = "";
-
-	public double DownloadProgress => totalItemsToDownloadCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / totalItemsToDownloadCount.Value;
-
-	public AttachmentFilterPanelModel FilterModel { get; }
-
-	private readonly StatisticsRow statisticsEnqueued = new ("Enqueued");
-	private readonly StatisticsRow statisticsDownloaded = new ("Downloaded");
-	private readonly StatisticsRow statisticsFailed = new ("Failed");
-	private readonly StatisticsRow statisticsSkipped = new ("Skipped");
-
-	public ObservableCollection<StatisticsRow> StatisticsRows { get; }
-
-	public bool IsDownloading => state.Downloader.IsDownloading;
-
-	private readonly State state;
-	private readonly ThrottledTask<int> enqueueDownloadItemsTask;
-	private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
-
-	private readonly IDisposable attachmentCountSubscription;
-	private readonly IDisposable downloadCountSubscription;
-
-	private IDisposable? finishedItemsSubscription;
-	private int doneItemsCount;
-	private int totalEnqueuedItemCount;
-	private int? totalItemsToDownloadCount;
-
-	public AttachmentsPageModel() : this(State.Dummy) {}
-
-	public AttachmentsPageModel(State state) {
-		this.state = state;
-
-		FilterModel = new AttachmentFilterPanelModel(state);
-		
-		StatisticsRows = [
-			statisticsEnqueued,
-			statisticsDownloaded,
-			statisticsFailed,
-			statisticsSkipped
-		];
-
-		enqueueDownloadItemsTask = new ThrottledTask<int>(OnItemsEnqueued, TaskScheduler.FromCurrentSynchronizationContext());
-		downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
-
-		attachmentCountSubscription = state.Db.Attachments.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnAttachmentCountChanged);
-		downloadCountSubscription = state.Db.Downloads.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnDownloadCountChanged);
-
-		RecomputeDownloadStatistics();
-	}
-
-	public void Dispose() {
-		attachmentCountSubscription.Dispose();
-		downloadCountSubscription.Dispose();
-		finishedItemsSubscription?.Dispose();
-
-		enqueueDownloadItemsTask.Dispose();
-		downloadStatisticsTask.Dispose();
-
-		FilterModel.Dispose();
-	}
-
-	private void OnAttachmentCountChanged(long newAttachmentCount) {
-		if (IsDownloading) {
-			EnqueueDownloadItemsLater();
-		}
-		else {
-			RecomputeDownloadStatistics();
-		}
-	}
-
-	private void OnDownloadCountChanged(long newDownloadCount) {
-		RecomputeDownloadStatistics();
-	}
-
-	private async Task EnqueueDownloadItems() {
-		OnItemsEnqueued(await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter()));
-	}
-
-	private void EnqueueDownloadItemsLater() {
-		var filter = CreateAttachmentFilter();
-		enqueueDownloadItemsTask.Post(cancellationToken => state.Db.Downloads.EnqueueDownloadItems(filter, cancellationToken));
-	}
-
-	private void OnItemsEnqueued(int itemCount) {
-		totalEnqueuedItemCount += itemCount;
-		totalItemsToDownloadCount = totalEnqueuedItemCount;
-		UpdateDownloadMessage();
-		RecomputeDownloadStatistics();
-	}
-
-	private AttachmentFilter CreateAttachmentFilter() {
-		var filter = FilterModel.CreateFilter();
-		filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
-		return filter;
-	}
-
-	public async Task OnClickToggleDownload() {
-		IsToggleDownloadButtonEnabled = false;
-
-		if (IsDownloading) {
-			await state.Downloader.Stop();
-
-			finishedItemsSubscription?.Dispose();
-			finishedItemsSubscription = null;
-
-			RecomputeDownloadStatistics();
-
-			await state.Db.Downloads.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
-
-			doneItemsCount = 0;
-			totalEnqueuedItemCount = 0;
-			totalItemsToDownloadCount = null;
-			UpdateDownloadMessage();
-		}
-		else {
-			var finishedItems = await state.Downloader.Start();
-
-			finishedItemsSubscription = finishedItems.Select(static _ => true)
-			                                         .Buffer(TimeSpan.FromMilliseconds(100))
-			                                         .Select(static items => items.Count)
-			                                         .Where(static items => items > 0)
-			                                         .ObserveOn(AvaloniaScheduler.Instance)
-			                                         .Subscribe(OnItemsFinished);
-
-			await EnqueueDownloadItems();
-		}
-
-		OnPropertyChanged(nameof(ToggleDownloadButtonText));
-		OnPropertyChanged(nameof(IsDownloading));
-		IsToggleDownloadButtonEnabled = true;
-	}
-
-	private void OnItemsFinished(int finishedItemCount) {
-		doneItemsCount += finishedItemCount;
-		UpdateDownloadMessage();
-	}
-
-	public async Task OnClickRetryFailedDownloads() {
-		IsRetryingFailedDownloads = true;
-
-		try {
-			var allExceptFailedFilter = new DownloadItemFilter {
-				IncludeStatuses = new HashSet<DownloadStatus> {
-					DownloadStatus.Enqueued,
-					DownloadStatus.Downloading,
-					DownloadStatus.Success
-				}
-			};
-
-			await state.Db.Downloads.RemoveDownloadItems(allExceptFailedFilter, FilterRemovalMode.KeepMatching);
-
-			if (IsDownloading) {
-				await EnqueueDownloadItems();
-			}
-		} catch (Exception e) {
-			Log.Error(e);
-		} finally {
-			IsRetryingFailedDownloads = false;
-		}
-	}
-
-	private void RecomputeDownloadStatistics() {
-		downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
-	}
-
-	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
-		statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
-		statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
-
-		statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
-		statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
-
-		statisticsFailed.Items = statusStatistics.FailedCount;
-		statisticsFailed.Size = statusStatistics.FailedSize;
-
-		statisticsSkipped.Items = statusStatistics.SkippedCount;
-		statisticsSkipped.Size = statusStatistics.SkippedSize;
-
-		hasFailedDownloads = statusStatistics.FailedCount > 0;
-
-		UpdateDownloadMessage();
-	}
-
-	private void UpdateDownloadMessage() {
-		DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
-
-		OnPropertyChanged(nameof(DownloadProgress));
-	}
-
-	[ObservableObject]
-	public sealed partial class StatisticsRow(string state) {
-		public string State { get; } = state;
-
-		[ObservableProperty]
-		private int items;
-
-		[ObservableProperty]
-		private ulong? size;
-	}
-}

app/Desktop/Main/Pages/DatabasePageModel.cs

@@ -18,7 +18,7 @@ using DHT.Server;
 using DHT.Server.Data;
 using DHT.Server.Database;
 using DHT.Server.Database.Import;
-using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Database.Sqlite.Schema;
 using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Main.Pages;

app/Desktop/Main/Pages/DebugPageModel.cs

@@ -10,8 +10,9 @@ using DHT.Server;
 using DHT.Server.Data;
 using DHT.Server.Service;
 
-namespace DHT.Desktop.Main.Pages {
+namespace DHT.Desktop.Main.Pages;
-	sealed class DebugPageModel {
+
+sealed class DebugPageModel {
 	public string GenerateChannels { get; set; } = "0";
 	public string GenerateUsers { get; set; } = "0";
 	public string GenerateMessages { get; set; } = "0";
@@ -158,16 +159,15 @@ namespace DHT.Desktop.Main.Pages {
 		int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3));
 		return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)]));
 	}
-	}
 }
 #else
-namespace DHT.Desktop.Main.Pages {
+namespace DHT.Desktop.Main.Pages;
-	sealed class DebugPageModel {
+
+sealed class DebugPageModel {
 	public string GenerateChannels { get; set; } = "0";
 	public string GenerateUsers { get; set; } = "0";
 	public string GenerateMessages { get; set; } = "0";
 
 	public void OnClickAddRandomDataToDatabase() {}
-	}
 }
 #endif

app/Desktop/Main/Pages/DownloadsPage.axaml

@@ -5,11 +5,11 @@
              xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
              xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
              mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
-             x:Class="DHT.Desktop.Main.Pages.AttachmentsPage"
+             x:Class="DHT.Desktop.Main.Pages.DownloadsPage"
-             x:DataType="pages:AttachmentsPageModel">
+             x:DataType="pages:DownloadsPageModel">
 
     <Design.DataContext>
-        <pages:AttachmentsPageModel />
+        <pages:DownloadsPageModel />
     </Design.DataContext>
 
     <UserControl.Styles>
@@ -31,19 +31,15 @@
     </UserControl.Styles>
 
     <StackPanel Orientation="Vertical" Spacing="20">
-        <DockPanel>
+        <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
-            <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" />
+        <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
-            <TextBlock Text="{Binding DownloadMessage}" MinWidth="100" Margin="10 0 0 0" VerticalAlignment="Center" TextAlignment="Right" DockPanel.Dock="Left" />
-            <ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" />
-        </DockPanel>
-        <controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !IsDownloading, RelativeSource={RelativeSource AncestorType=pages:AttachmentsPageModel}}" />
         <StackPanel Orientation="Vertical" Spacing="12">
             <Expander Header="Download Status" IsExpanded="True">
                 <DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
                     <DataGrid.Columns>
-                        <DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" />
+                        <DataGridTextColumn Header="State" Binding="{Binding State, Mode=OneWay}" Width="*" />
-                        <DataGridTextColumn Header="Attachments" Binding="{Binding Items, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
+                        <DataGridTextColumn Header="Files" Binding="{Binding Items, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
-                        <DataGridTextColumn Header="Size" Binding="{Binding Size, Mode=OneWay, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" />
+                        <DataGridTextColumn Header="Size" Binding="{Binding SizeText, Mode=OneWay}" Width="*" CellStyleClasses="right" />
                     </DataGrid.Columns>
                 </DataGrid>
             </Expander>

app/Desktop/Main/Pages/DownloadsPage.axaml.cs

@@ -4,8 +4,8 @@ using Avalonia.Controls;
 namespace DHT.Desktop.Main.Pages;
 
 [SuppressMessage("ReSharper", "MemberCanBeInternal")]
-public sealed partial class AttachmentsPage : UserControl {
+public sealed partial class DownloadsPage : UserControl {
-	public AttachmentsPage() {
+	public DownloadsPage() {
 		InitializeComponent();
 	}
 }

app/Desktop/Main/Pages/DownloadsPageModel.cs

@@ -0,0 +1,186 @@
+using System;
+using System.Collections.ObjectModel;
+using System.Reactive.Linq;
+using System.Threading.Tasks;
+using Avalonia.ReactiveUI;
+using CommunityToolkit.Mvvm.ComponentModel;
+using DHT.Desktop.Common;
+using DHT.Desktop.Main.Controls;
+using DHT.Server;
+using DHT.Server.Data.Aggregations;
+using DHT.Server.Data.Filters;
+using DHT.Server.Download;
+using DHT.Utils.Logging;
+using DHT.Utils.Tasks;
+
+namespace DHT.Desktop.Main.Pages;
+
+sealed partial class DownloadsPageModel : ObservableObject, IDisposable {
+	private static readonly Log Log = Log.ForType<DownloadsPageModel>();
+
+	[ObservableProperty(Setter = Access.Private)]
+	private bool isToggleDownloadButtonEnabled = true;
+
+	public string ToggleDownloadButtonText => IsDownloading ? "Stop Downloading" : "Start Downloading";
+
+	[ObservableProperty(Setter = Access.Private)]
+	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
+	private bool isRetryingFailedDownloads = false;
+
+	[ObservableProperty(Setter = Access.Private)]
+	[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
+	private bool hasFailedDownloads;
+
+	public bool IsRetryFailedOnDownloadsButtonEnabled => !IsRetryingFailedDownloads && HasFailedDownloads;
+
+	[ObservableProperty(Setter = Access.Private)]
+	private string downloadMessage = "";
+
+	public DownloadItemFilterPanelModel FilterModel { get; }
+
+	private readonly StatisticsRow statisticsPending = new ("Pending");
+	private readonly StatisticsRow statisticsDownloaded = new ("Downloaded");
+	private readonly StatisticsRow statisticsFailed = new ("Failed");
+	private readonly StatisticsRow statisticsSkipped = new ("Skipped");
+
+	public ObservableCollection<StatisticsRow> StatisticsRows { get; }
+
+	public bool IsDownloading => state.Downloader.IsDownloading;
+
+	private readonly State state;
+	private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
+	private readonly IDisposable downloadItemCountSubscription;
+	
+	private IDisposable? finishedItemsSubscription;
+	private DownloadItemFilter? currentDownloadFilter;
+
+	public DownloadsPageModel() : this(State.Dummy) {}
+
+	public DownloadsPageModel(State state) {
+		this.state = state;
+
+		FilterModel = new DownloadItemFilterPanelModel(state);
+		
+		StatisticsRows = [
+			statisticsPending,
+			statisticsDownloaded,
+			statisticsFailed,
+			statisticsSkipped
+		];
+
+		downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(Log, UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
+		downloadItemCountSubscription = state.Db.Downloads.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnDownloadCountChanged);
+
+		RecomputeDownloadStatistics();
+	}
+
+	public void Dispose() {
+		finishedItemsSubscription?.Dispose();
+		
+		downloadItemCountSubscription.Dispose();
+		downloadStatisticsTask.Dispose();
+
+		FilterModel.Dispose();
+	}
+
+	private void OnDownloadCountChanged(long newDownloadCount) {
+		RecomputeDownloadStatistics();
+	}
+
+	public async Task OnClickToggleDownload() {
+		IsToggleDownloadButtonEnabled = false;
+
+		if (IsDownloading) {
+			await state.Downloader.Stop();
+			await state.Db.Downloads.MoveDownloadingItemsBackToQueue();
+
+			finishedItemsSubscription?.Dispose();
+			finishedItemsSubscription = null;
+			
+			currentDownloadFilter = null;
+		}
+		else {
+			await state.Db.Downloads.MoveDownloadingItemsBackToQueue();
+			
+			var finishedItems = await state.Downloader.Start(currentDownloadFilter = FilterModel.CreateFilter());
+			finishedItemsSubscription = finishedItems.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnItemFinished);
+		}
+
+		RecomputeDownloadStatistics();
+
+		OnPropertyChanged(nameof(ToggleDownloadButtonText));
+		OnPropertyChanged(nameof(IsDownloading));
+		IsToggleDownloadButtonEnabled = true;
+	}
+
+	private void OnItemFinished(DownloadItem item) {
+		RecomputeDownloadStatistics();
+	}
+
+	public async Task OnClickRetryFailedDownloads() {
+		IsRetryingFailedDownloads = true;
+
+		try {
+			await state.Db.Downloads.RetryFailed();
+			RecomputeDownloadStatistics();
+		} catch (Exception e) {
+			Log.Error(e);
+		} finally {
+			IsRetryingFailedDownloads = false;
+		}
+	}
+
+	private void RecomputeDownloadStatistics() {
+		downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
+	}
+
+	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
+		statisticsPending.Items = statusStatistics.PendingCount;
+		statisticsPending.Size = statusStatistics.PendingTotalSize;
+		statisticsPending.HasFilesWithUnknownSize = statusStatistics.PendingWithUnknownSizeCount > 0;
+
+		statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
+		statisticsDownloaded.Size = statusStatistics.SuccessfulTotalSize;
+		statisticsDownloaded.HasFilesWithUnknownSize = statusStatistics.SuccessfulWithUnknownSizeCount > 0;
+
+		statisticsFailed.Items = statusStatistics.FailedCount;
+		statisticsFailed.Size = statusStatistics.FailedTotalSize;
+		statisticsFailed.HasFilesWithUnknownSize = statusStatistics.FailedWithUnknownSizeCount > 0;
+		
+		statisticsSkipped.Items = statusStatistics.SkippedCount;
+		statisticsSkipped.Size = statusStatistics.SkippedTotalSize;
+		statisticsSkipped.HasFilesWithUnknownSize = statusStatistics.SkippedWithUnknownSizeCount > 0;
+
+		HasFailedDownloads = statusStatistics.FailedCount > 0;
+	}
+
+	[ObservableObject]
+	public sealed partial class StatisticsRow(string state) {
+		public string State { get; } = state;
+
+		[ObservableProperty]
+		private int items;
+
+		[ObservableProperty]
+		[NotifyPropertyChangedFor(nameof(SizeText))]
+		private ulong? size;
+		
+		[ObservableProperty]
+		[NotifyPropertyChangedFor(nameof(SizeText))]
+		private bool hasFilesWithUnknownSize;
+
+		public string SizeText {
+			get {
+				if (size == null) {
+					return "-";
+				}
+				else if (hasFilesWithUnknownSize) {
+					return "\u2265 " + BytesValueConverter.Convert(size.Value);
+				}
+				else {
+					return BytesValueConverter.Convert(size.Value);
+				}
+			}
+		}
+	}
+}

app/Desktop/Main/Pages/ViewerPage.axaml

@@ -21,7 +21,6 @@
     <StackPanel Orientation="Vertical" Spacing="20">
         <StackPanel Orientation="Horizontal" VerticalAlignment="Top">
             <Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
-            <Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
         </StackPanel>
         <controls:MessageFilterPanel DataContext="{Binding FilterModel}" />
         <Expander Header="Database Tools">

app/Desktop/Main/Pages/ViewerPageModel.cs

@@ -1,35 +1,22 @@
 using System;
-using System.Collections.Concurrent;
 using System.ComponentModel;
 using System.Diagnostics;
-using System.IO;
-using System.Text;
 using System.Threading.Tasks;
 using System.Web;
 using Avalonia.Controls;
-using Avalonia.Platform.Storage;
 using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
-using DHT.Desktop.Dialogs.File;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
 using DHT.Desktop.Server;
 using DHT.Server;
 using DHT.Server.Data.Filters;
-using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
-using DHT.Server.Database.Export.Strategy;
-using static DHT.Desktop.Program;
 
 namespace DHT.Desktop.Main.Pages;
 
 sealed partial class ViewerPageModel : ObservableObject, IDisposable {
-	public static readonly ConcurrentBag<string> TemporaryFiles = [];
-
-	private static readonly FilePickerFileType[] ViewerFileTypes = [
-		FileDialogs.CreateFilter("Discord History Viewer", ["html"])
-	];
-
 	public bool DatabaseToolFilterModeKeep { get; set; } = true;
 	public bool DatabaseToolFilterModeRemove { get; set; } = false;
 
@@ -62,96 +49,18 @@ sealed partial class ViewerPageModel : ObservableObject, IDisposable {
 
 	public async void OnClickOpenViewer() {
 		try {
-			var fullPath = await PrepareTemporaryViewerFile();
+			string serverUrl = "http://127.0.0.1:" + ServerConfiguration.Port;
-			var strategy = new LiveViewerExportStrategy(ServerConfiguration.Port, ServerConfiguration.Token);
+			string serverToken = ServerConfiguration.Token;
+			string sessionId = state.ViewerSessions.Register(new ViewerSession(FilterModel.CreateFilter())).ToString();
 			
-			await ProgressDialog.ShowIndeterminate(window, "Open Viewer", "Creating viewer...", _ => Task.Run(() => WriteViewerFile(fullPath, strategy)));
+			Process.Start(new ProcessStartInfo(serverUrl + "/viewer/?token=" + HttpUtility.UrlEncode(serverToken) + "&session=" + HttpUtility.UrlEncode(sessionId)) {
-			
-			Process.Start(new ProcessStartInfo(fullPath) {
 				UseShellExecute = true
 			});
 		} catch (Exception e) {
-			await Dialog.ShowOk(window, "Open Viewer", "Could not create or save viewer: " + e.Message);
+			await Dialog.ShowOk(window, "Open Viewer", "Could not open viewer: " + e.Message);
 		}
 	}
 
-	private async Task<string> PrepareTemporaryViewerFile() {
-		return await Task.Run(() => {
-			string rootPath = Path.Combine(Path.GetTempPath(), "DiscordHistoryTracker");
-			string filenameBase = Path.GetFileNameWithoutExtension(state.Db.Path) + "-" + DateTime.Now.ToString("yyyy-MM-dd");
-			string fullPath = Path.Combine(rootPath, filenameBase + ".html");
-			
-			int counter = 0;
-
-			while (File.Exists(fullPath)) {
-				++counter;
-				fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html");
-			}
-
-			TemporaryFiles.Add(fullPath);
-
-			Directory.CreateDirectory(rootPath);
-
-			return fullPath;
-		});
-	}
-
-	public async void OnClickSaveViewer() {
-		string? path = await window.StorageProvider.SaveFile(new FilePickerSaveOptions {
-			Title = "Save Viewer",
-			FileTypeChoices = ViewerFileTypes,
-			SuggestedFileName = Path.GetFileNameWithoutExtension(state.Db.Path) + ".html",
-			SuggestedStartLocation = await FileDialogs.GetSuggestedStartLocation(window, Path.GetDirectoryName(state.Db.Path)),
-		});
-
-		if (path == null) {
-			return;
-		}
-
-		try {
-			await ProgressDialog.ShowIndeterminate(window, "Save Viewer", "Creating viewer...", _ => Task.Run(() => WriteViewerFile(path, StandaloneViewerExportStrategy.Instance)));
-		} catch (Exception e) {
-			await Dialog.ShowOk(window, "Save Viewer", "Could not create or save viewer: " + e.Message);
-		}
-	}
-
-	private async Task WriteViewerFile(string path, IViewerExportStrategy strategy) {
-		const string ArchiveTag = "/*[ARCHIVE]*/";
-
-		string indexFile = await Resources.ReadTextAsync("Viewer/index.html");
-		string viewerTemplate = indexFile.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
-		                                 .Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'));
-
-		int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag);
-		int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length;
-
-		string jsonTempFile = path + ".tmp";
-
-		await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) {
-			await ViewerJsonExport.Generate(jsonStream, strategy, state.Db, FilterModel.CreateFilter());
-
-			char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)];
-			jsonStream.Position = 0;
-
-			await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
-			await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) {
-				await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]);
-
-				using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) {
-					int readBytes;
-					while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) {
-						string jsonChunk = new string(jsonBuffer, 0, readBytes);
-						await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk));
-					}
-				}
-
-				await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]);
-			}
-		}
-
-		File.Delete(jsonTempFile);
-	}
-
 	public async Task OnClickApplyFiltersToDatabase() {
 		var filter = FilterModel.CreateFilter();
 		var messageCount = await ProgressDialog.ShowIndeterminate(window, "Apply Filters", "Counting matching messages...", _ => state.Db.Messages.Count(filter));

app/Desktop/Main/Screens/MainContentScreen.axaml

@@ -74,7 +74,7 @@
     <DockPanel>
         <Border Classes="statusBar" DockPanel.Dock="Bottom">
             <DockPanel>
-                <TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Attachments</TextBlock>
+                <TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Downloads</TextBlock>
                 <controls:StatusBar DataContext="{Binding StatusBarModel}" DockPanel.Dock="Right" />
             </DockPanel>
         </Border>
@@ -94,9 +94,9 @@
                     <ContentPresenter Content="{Binding TrackingPage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
-            <TabItem x:Name="TabAttachments" Header="Attachments" Grid.Row="2">
+            <TabItem x:Name="TabDownloads" Header="Downloads" Grid.Row="2">
                 <ScrollViewer>
-                    <ContentPresenter Content="{Binding AttachmentsPage}" Classes="page" />
+                    <ContentPresenter Content="{Binding DownloadsPage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
             <TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="3">

app/Desktop/Main/Screens/MainContentScreenModel.cs

@@ -13,8 +13,8 @@ sealed class MainContentScreenModel : IDisposable {
 	public TrackingPage TrackingPage { get; }
 	private TrackingPageModel TrackingPageModel { get; }
 
-	public AttachmentsPage AttachmentsPage { get; }
+	public DownloadsPage DownloadsPage { get; }
-	private AttachmentsPageModel AttachmentsPageModel { get; }
+	private DownloadsPageModel DownloadsPageModel { get; }
 
 	public ViewerPage ViewerPage { get; }
 	private ViewerPageModel ViewerPageModel { get; }
@@ -52,8 +52,8 @@ sealed class MainContentScreenModel : IDisposable {
 		TrackingPageModel = new TrackingPageModel(window);
 		TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
 
-		AttachmentsPageModel = new AttachmentsPageModel(state);
+		DownloadsPageModel = new DownloadsPageModel(state);
-		AttachmentsPage = new AttachmentsPage { DataContext = AttachmentsPageModel };
+		DownloadsPage = new DownloadsPage { DataContext = DownloadsPageModel };
 
 		ViewerPageModel = new ViewerPageModel(window, state);
 		ViewerPage = new ViewerPage { DataContext = ViewerPageModel };
@@ -72,7 +72,7 @@ sealed class MainContentScreenModel : IDisposable {
 	}
 
 	public void Dispose() {
-		AttachmentsPageModel.Dispose();
+		DownloadsPageModel.Dispose();
 		ViewerPageModel.Dispose();
 		AdvancedPageModel.Dispose();
 		StatusBarModel.Dispose();

app/Desktop/Main/Screens/WelcomeScreenModel.cs

@@ -8,7 +8,7 @@ using DHT.Desktop.Common;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
 using DHT.Server.Database;
-using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Database.Sqlite.Schema;
 
 namespace DHT.Desktop.Main.Screens;
 

app/Resources/Tracker/scripts/state.js

@@ -48,6 +48,16 @@ const STATE = (function() {
 		});
 	};
 	
+	const getDate = function(date) {
+		if (date instanceof Date) {
+			return date;
+		}
+		else {
+			// noinspection JSUnresolvedReference
+			return date.toDate();
+		}
+	};
+	
 	const trackingStateChangedListeners = [];
 	let isTracking = false;
 	
@@ -69,8 +79,8 @@ const STATE = (function() {
 	 * @property {String} channel_id
 	 * @property {DiscordUser} author
 	 * @property {String} content
-	 * @property {Timestamp} timestamp
+	 * @property {Date} timestamp
-	 * @property {Timestamp|null} editedTimestamp
+	 * @property {Date|null} editedTimestamp
 	 * @property {DiscordAttachment[]} attachments
 	 * @property {Object[]} embeds
 	 * @property {DiscordMessageReaction[]} [reactions]
@@ -106,11 +116,6 @@ const STATE = (function() {
 	 * @property {Boolean} animated
 	 */
 	
-	/**
-	 * @name Timestamp
-	 * @property {Function} toDate
-	 */
-	
 	return {
 		setup(port, token) {
 			serverPort = port;
@@ -223,12 +228,12 @@ const STATE = (function() {
 					sender: msg.author.id,
 					channel: msg.channel_id,
 					text: msg.content,
-					timestamp: msg.timestamp.toDate().getTime()
+					timestamp: getDate(msg.timestamp).getTime()
 				};
 				
 				if (msg.editedTimestamp !== null) {
 					// noinspection JSUnusedGlobalSymbols
-					obj.editTimestamp = msg.editedTimestamp.toDate().getTime();
+					obj.editTimestamp = getDate(msg.editedTimestamp).getTime();
 				}
 				
 				if (msg.messageReference !== null) {

app/Resources/Viewer/index.html

@@ -2,15 +2,24 @@
 <html lang="en">
   <head>
     <meta charset="utf-8">
+    <meta name="referrer" content="no-referrer">
+    
     <title>Discord Offline History</title>
     
+    <link rel="icon" href="favicon.ico">
+    
+    <link rel="stylesheet" href="styles/main.css">
+    <link rel="stylesheet" href="styles/menu.css">
+    <link rel="stylesheet" href="styles/channels.css">
+    <link rel="stylesheet" href="styles/messages.css">
+    <link rel="stylesheet" href="styles/modal.css">
+    
     <script type="text/javascript">
-		window.DHT_EMBEDDED = "/*[ARCHIVE]*/";
+		const query = new URLSearchParams(location.search);
-		/*[JS]*/
+		window.DHT_SERVER_TOKEN = query.get("token");
+		window.DHT_SERVER_SESSION = query.get("session");
     </script>
-    <style>
+    <script type="module" src="scripts/bootstrap.mjs"></script>
-      /*[CSS]*/
-    </style>
   </head>
   <body>
     <div id="menu">
@@ -64,7 +73,9 @@
     </div>
     
     <div id="app">
-      <div id="channels"></div>
+      <div id="channels">
+        <div class="loading"></div>
+      </div>
       <div id="messages"></div>
     </div>
     

app/Resources/Viewer/scripts/bootstrap.js

@@ -1,39 +0,0 @@
-document.addEventListener("DOMContentLoaded", () => {
-	DISCORD.setup();
-	GUI.setup();
-	
-	GUI.onOptionMessagesPerPageChanged(() => {
-		STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage());
-	});
-	
-	STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage());
-	
-	GUI.onOptMessageFilterChanged(filter => {
-		STATE.setActiveFilter(filter);
-	});
-	
-	GUI.onNavigationButtonClicked(action => {
-		STATE.updateCurrentPage(action);
-	});
-	
-	STATE.onUsersRefreshed(users => {
-		GUI.updateUserList(users);
-	});
-	
-	STATE.onChannelsRefreshed((channels, selected) => {
-		GUI.updateChannelList(channels, selected, STATE.selectChannel);
-	});
-	
-	STATE.onMessagesRefreshed(messages => {
-		GUI.updateNavigation(STATE.getCurrentPage(), STATE.getPageCount());
-		GUI.updateMessageList(messages);
-		GUI.scrollMessagesToTop();
-	});
-	
-	try {
-		STATE.uploadFile(JSON.parse(window.DHT_EMBEDDED));
-	} catch (e) {
-		console.error(e);
-		alert("Could not parse embedded file, see console for details.");
-	}
-});

app/Resources/Viewer/scripts/bootstrap.mjs

@@ -0,0 +1,110 @@
+import discord from "./discord.mjs";
+import gui from "./gui.mjs";
+import state from "./state.mjs";
+
+window.DISCORD = discord;
+
+document.addEventListener("DOMContentLoaded", () => {
+	discord.setup();
+	gui.setup();
+	
+	gui.onOptionMessagesPerPageChanged(() => {
+		state.setMessagesPerPage(gui.getOptionMessagesPerPage());
+	});
+	
+	state.setMessagesPerPage(gui.getOptionMessagesPerPage());
+	
+	gui.onOptMessageFilterChanged(filter => {
+		state.setActiveFilter(filter);
+	});
+	
+	gui.onNavigationButtonClicked(action => {
+		state.updateCurrentPage(action);
+	});
+	
+	state.onUsersRefreshed(users => {
+		gui.updateUserList(users);
+	});
+	
+	state.onChannelsRefreshed((channels, selected) => {
+		gui.updateChannelList(channels, selected, state.selectChannel);
+	});
+	
+	state.onMessagesRefreshed(messages => {
+		gui.updateNavigation(state.getCurrentPage(), state.getPageCount());
+		gui.updateMessageList(messages);
+		gui.scrollMessagesToTop();
+	});
+	
+	async function fetchUrl(path, contentType) {
+		const response = await fetch("/" + path + "?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), {
+			method: "GET",
+			headers: {
+				"Content-Type": contentType,
+			},
+			credentials: "omit",
+			redirect: "error",
+		});
+		
+		if (!response.ok) {
+			throw "Unexpected response status: " + response.statusText;
+		}
+		
+		return response;
+	}
+	
+	async function processLines(response, callback) {
+		let body = "";
+		
+		for await (const chunk of response.body.pipeThrough(new TextDecoderStream("utf-8"))) {
+			body += chunk;
+			
+			let startIndex = 0;
+			
+			while (true) {
+				const endIndex = body.indexOf("\n", startIndex);
+				if (endIndex === -1) {
+					break;
+				}
+				
+				callback(body.substring(startIndex, endIndex));
+				startIndex = endIndex + 1;
+			}
+			
+			body = body.substring(startIndex);
+		}
+		
+		if (body !== "") {
+			callback(body);
+		}
+	}
+	
+	async function loadData() {
+		try {
+			const metadataResponse = await fetchUrl("get-viewer-metadata", "application/json");
+			const metadataJson = await metadataResponse.json();
+			
+			const messagesResponse = await fetchUrl("get-viewer-messages", "application/x-ndjson");
+			const messages = {};
+			
+			await processLines(messagesResponse, line => {
+				const message = JSON.parse(line);
+				const channel = message.c;
+				
+				const channelMessages = messages[channel] || (messages[channel] = {});
+				channelMessages[message.id] = message;
+				
+				delete message.id;
+				delete message.c;
+			});
+			
+			state.uploadFile(metadataJson, messages);
+		} catch (e) {
+			console.error(e);
+			alert("Could not load data, see console for details.");
+			document.querySelector("#channels > div.loading").remove();
+		}
+	}
+	
+	loadData();
+});

app/Resources/Viewer/scripts/discord.mjs

@@ -1,4 +1,10 @@
-const DISCORD = (function() {
+import discord from "./discord.mjs";
+import dom from "./dom.mjs";
+import template from "./template.mjs";
+import settings from "./settings.mjs";
+import state from "./state.mjs";
+
+export default (function() {
 	const regex = {
 		formatBold: /\*\*([\s\S]+?)\*\*(?!\*)/g,
 		formatItalic1: /\*([\s\S]+?)\*(?!\*)/g,
@@ -35,10 +41,24 @@ const DISCORD = (function() {
 	let templateReaction;
 	let templateReactionCustom;
 	
-	const processMessageContents = function(contents) {
+	const fileUrlProcessor = function(serverToken) {
-		let processed = DOM.escapeHTML(contents.replace(regex.formatUrlNoEmbed, "$1"));
+		if (typeof serverToken === "string") {
+			return url => "/get-downloaded-file/" + encodeURIComponent(url) + "?token=" + encodeURIComponent(serverToken);
+		}
+		else {
+			return url => url;
+		}
+	}(window.DHT_SERVER_TOKEN);
 	
-		if (SETTINGS.enableFormatting) {
+	const getEmoji = function(name, id, extension) {
+		const tag = ":" + name + ":";
+		return "<img src='" + fileUrlProcessor("https://cdn.discordapp.com/emojis/" + id + "." + extension) + "' alt='" + tag + "' title='" + tag + "' class='emoji'>";
+	};
+	
+	const processMessageContents = function(contents) {
+		let processed = dom.escapeHTML(contents.replace(regex.formatUrlNoEmbed, "$1"));
+		
+		if (settings.enableFormatting) {
 			const escapeHtmlMatch = (full, match) => "&#" + match.charCodeAt(0) + ";";
 			
 			processed = processed
@@ -54,29 +74,33 @@ const DISCORD = (function() {
 				.replace(regex.formatStrike, "<s>$1</s>");
 		}
 		
-		const animatedEmojiExtension = SETTINGS.enableAnimatedEmoji ? "gif" : "png";
+		const animatedEmojiExtension = settings.enableAnimatedEmoji ? "gif" : "webp";
 		
 		// noinspection HtmlUnknownTarget
 		processed = processed
 			.replace(regex.formatUrl, "<a href='$1' target='_blank' rel='noreferrer'>$1</a>")
-			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + STATE.getChannelName(match) + "</span>")
+			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + state.getChannelName(match) + "</span>")
-			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (STATE.getUserTag(match) || "????") + "'>@" + STATE.getUserName(match) + "</span>")
+			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (state.getUserTag(match) || "????") + "'>@" + state.getUserName(match) + "</span>")
-			.replace(regex.customEmojiStatic, "<img src='https://cdn.discordapp.com/emojis/$2.png' alt=':$1:' title=':$1:' class='emoji'>")
+			.replace(regex.customEmojiStatic, (full, m1, m2) => getEmoji(m1, m2, "webp"))
-			.replace(regex.customEmojiAnimated, "<img src='https://cdn.discordapp.com/emojis/$2." + animatedEmojiExtension + "' alt=':$1:' title=':$1:' class='emoji'>");
+			.replace(regex.customEmojiAnimated, (full, m1, m2) => getEmoji(m1, m2, animatedEmojiExtension));
 		
 		return "<p>" + processed + "</p>";
 	};
 	
+	const getAvatarUrlObject = function(avatar) {
+		return { url: fileUrlProcessor("https://cdn.discordapp.com/avatars/" + avatar.id + "/" + avatar.path + ".webp") };
+	};
+	
 	const getImageEmbed = function(url, image) {
-		if (!SETTINGS.enableImagePreviews) {
+		if (!settings.enableImagePreviews) {
 			return "";
 		}
 		
 		if (image.width && image.height) {
-			return templateEmbedImageWithSize.apply({ url, src: image.url, width: image.width, height: image.height });
+			return templateEmbedImageWithSize.apply({ url: fileUrlProcessor(url), src: fileUrlProcessor(image.url), width: image.width, height: image.height });
 		}
 		else {
-			return templateEmbedImage.apply({ url, src: image.url });
+			return templateEmbedImage.apply({ url: fileUrlProcessor(url), src: fileUrlProcessor(image.url) });
 		}
 	};
 	
@@ -88,21 +112,21 @@ const DISCORD = (function() {
 	
 	return {
 		setup() {
-			templateChannelServer = new TEMPLATE([
+			templateChannelServer = new template([
-				"<div data-channel='{id}'>",
+				"<div class='channel' data-channel='{id}'>",
 				"<div class='info' title='{topic}'><strong class='name'>#{name}</strong>{nsfw}<span class='tag'>{msgcount}</span></div>",
 				"<span class='server'>{server.name} ({server.type})</span>",
 				"</div>"
 			].join(""));
 			
-			templateChannelPrivate = new TEMPLATE([
+			templateChannelPrivate = new template([
-				"<div data-channel='{id}'>",
+				"<div class='channel' data-channel='{id}'>",
 				"<div class='info'><strong class='name'>{name}</strong><span class='tag'>{msgcount}</span></div>",
 				"<span class='server'>({server.type})</span>",
 				"</div>"
 			].join(""));
 			
-			templateMessageNoAvatar = new TEMPLATE([
+			templateMessageNoAvatar = new template([
 				"<div>",
 				"<div class='reply-message'>{reply}</div>",
 				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
@@ -111,7 +135,7 @@ const DISCORD = (function() {
 				"</div>"
 			].join(""));
 			
-			templateMessageWithAvatar = new TEMPLATE([
+			templateMessageWithAvatar = new template([
 				"<div>",
 				"<div class='reply-message reply-message-with-avatar'>{reply}</div>",
 				"<div class='avatar-wrapper'>",
@@ -125,50 +149,52 @@ const DISCORD = (function() {
 				"</div>"
 			].join(""));
 			
-			templateUserAvatar = new TEMPLATE([
+			// noinspection HtmlUnknownTarget
-				"<img src='https://cdn.discordapp.com/avatars/{id}/{path}.webp?size=128' alt=''>"
+			templateUserAvatar = new template([
+				"<img src='{url}' alt=''>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateAttachmentDownload = new TEMPLATE([
+			templateAttachmentDownload = new template([
 				"<a href='{url}' class='embed download'>Download {name}</a>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateEmbedImage = new TEMPLATE([
+			templateEmbedImage = new template([
-				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
+				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' alt='' onload='window.DISCORD.handleImageLoad(this)' onerror='window.DISCORD.handleImageLoadError(this)'></a><br>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateEmbedImageWithSize = new TEMPLATE([
+			templateEmbedImageWithSize = new template([
-				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' width='{width}' height='{height}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
+				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' width='{width}' height='{height}' alt='' onload='window.DISCORD.handleImageLoad(this)' onerror='window.DISCORD.handleImageLoadError(this)'></a><br>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateEmbedRich = new TEMPLATE([
+			templateEmbedRich = new template([
 				"<div class='embed download'><a href='{url}' class='title'>{title}</a><p class='desc'>{description}</p></div>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateEmbedRichNoDescription = new TEMPLATE([
+			templateEmbedRichNoDescription = new template([
 				"<div class='embed download'><a href='{url}' class='title'>{title}</a></div>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
-			templateEmbedUrl = new TEMPLATE([
+			templateEmbedUrl = new template([
 				"<a href='{url}' class='embed download'>{url}</a>"
 			].join(""));
 			
-			templateEmbedUnsupported = new TEMPLATE([
+			templateEmbedUnsupported = new template([
 				"<div class='embed download'><p>(Unsupported embed)</p></div>"
 			].join(""));
 			
-			templateReaction = new TEMPLATE([
+			templateReaction = new template([
 				"<span class='reaction-wrapper'><span class='reaction-emoji'>{n}</span><span class='count'>{c}</span></span>"
 			].join(""));
 			
-			templateReactionCustom = new TEMPLATE([
+			// noinspection HtmlUnknownTarget
-				"<span class='reaction-wrapper'><img src='https://cdn.discordapp.com/emojis/{id}.{ext}' alt=':{n}:' title=':{n}:' class='reaction-emoji-custom'><span class='count'>{c}</span></span>"
+			templateReactionCustom = new template([
+				"<span class='reaction-wrapper'><img src='{url}' alt=':{n}:' title=':{n}:' class='reaction-emoji-custom'><span class='count'>{c}</span></span>"
 			].join(""));
 		},
 		
@@ -184,7 +210,7 @@ const DISCORD = (function() {
 		},
 		
 		isImageAttachment(attachment) {
-			const url = DOM.tryParseUrl(attachment.url);
+			const url = dom.tryParseUrl(attachment.url);
 			return url != null && isImageUrl(url);
 		},
 		
@@ -197,15 +223,15 @@ const DISCORD = (function() {
 		},
 		
 		getMessageHTML(message) { // noinspection FunctionWithInconsistentReturnsJS
-			return (SETTINGS.enableUserAvatars ? templateMessageWithAvatar : templateMessageNoAvatar).apply(message, (property, value) => {
+			return (settings.enableUserAvatars ? templateMessageWithAvatar : templateMessageNoAvatar).apply(message, (property, value) => {
 				if (property === "avatar") {
-					return value ? templateUserAvatar.apply(value) : "";
+					return value ? templateUserAvatar.apply(getAvatarUrlObject(value)) : "";
 				}
 				else if (property === "user.tag") {
 					return value ? value : "????";
 				}
 				else if (property === "timestamp") {
-					return DOM.getHumanReadableTime(value);
+					return dom.getHumanReadableTime(value);
 				}
 				else if (property === "contents") {
 					return value && value.length > 0 ? processMessageContents(value) : "";
@@ -242,22 +268,24 @@ const DISCORD = (function() {
 					}
 					
 					return value.map(attachment => {
-						if (!DISCORD.isImageAttachment(attachment) || !SETTINGS.enableImagePreviews) {
+						const url = fileUrlProcessor(attachment.url);
-							return templateAttachmentDownload.apply(attachment);
+						
+						if (!discord.isImageAttachment(attachment) || !settings.enableImagePreviews) {
+							return templateAttachmentDownload.apply({ url, name: attachment.name });
 						}
 						else if ("width" in attachment && "height" in attachment) {
-							return templateEmbedImageWithSize.apply({ url: attachment.url, src: attachment.url, width: attachment.width, height: attachment.height });
+							return templateEmbedImageWithSize.apply({ url, src: url, width: attachment.width, height: attachment.height });
 						}
 						else {
-							return templateEmbedImage.apply({ url: attachment.url, src: attachment.url });
+							return templateEmbedImage.apply({ url, src: url });
 						}
 					}).join("");
 				}
 				else if (property === "edit") {
-					return value ? "<span class='info edited'>Edited " + DOM.getHumanReadableTime(value) + "</span>" : "";
+					return value ? "<span class='info edited'>Edited " + dom.getHumanReadableTime(value) + "</span>" : "";
 				}
 				else if (property === "jump") {
-					return STATE.hasActiveFilter ? "<span class='info jump' data-jump='" + value + "'>Jump to message</span>" : "";
+					return state.hasActiveFilter ? "<span class='info jump' data-jump='" + value + "'>Jump to message</span>" : "";
 				}
 				else if (property === "reply") {
 					if (!value) {
@@ -265,7 +293,7 @@ const DISCORD = (function() {
 					}
 					
 					const user = "<span class='reply-username' title='#" + (value.user.tag ? value.user.tag : "????") + "'>" + value.user.name + "</span>";
-					const avatar = SETTINGS.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(value.avatar) + "</span>" : "";
+					const avatar = settings.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(getAvatarUrlObject(value.avatar)) + "</span>" : "";
 					const contents = value.contents ? "<span class='reply-contents'>" + processMessageContents(value.contents) + "</span>" : "";
 					
 					return "<span class='jump' data-jump='" + value.id + "'>Jump to reply</span><span class='user'>" + avatar + user + "</span>" + contents;
@@ -277,9 +305,10 @@ const DISCORD = (function() {
 					
 					return "<div class='reactions'>" + value.map(reaction => {
 						if ("id" in reaction){
-							// noinspection JSUnusedGlobalSymbols, JSUnresolvedVariable
+							const ext = reaction.a && settings.enableAnimatedEmoji ? "gif" : "webp";
-							reaction.ext = reaction.a && SETTINGS.enableAnimatedEmoji ? "gif" : "png";
+							const url = fileUrlProcessor("https://cdn.discordapp.com/emojis/" + reaction.id + "." + ext);
-							return templateReactionCustom.apply(reaction);
+							// noinspection JSUnusedGlobalSymbols
+							return templateReactionCustom.apply({ url, n: reaction.n, c: reaction.c });
 						}
 						else {
 							return templateReaction.apply(reaction);

app/Resources/Viewer/scripts/dom.mjs

@@ -8,9 +8,12 @@ const HTML_ENTITY_MAP = {
 
 const HTML_ENTITY_REGEX = /[&<>"']/g;
 
-class DOM {
+export default class {
 	/**
 	 * Returns a child element by its ID. Parent defaults to the entire document.
+	 * @param {string} id
+	 * @param {HTMLElement} [parent]
+	 * @returns {HTMLElement|null}
 	 */
 	static id(id, parent) {
 		return (parent || document).getElementById(id);
@@ -18,6 +21,9 @@ class DOM {
 	
 	/**
 	 * Returns an array of all child elements containing the specified class. Parent defaults to the entire document.
+	 * @param {string} cls
+	 * @param {HTMLElement} [parent]
+	 * @returns {HTMLElement[]}
 	 */
 	static cls(cls, parent) {
 		return Array.prototype.slice.call((parent || document).getElementsByClassName(cls));
@@ -25,6 +31,9 @@ class DOM {
 	
 	/**
 	 * Returns an array of all child elements that have the specified tag. Parent defaults to the entire document.
+	 * @param {string} tag
+	 * @param {HTMLElement} [parent]
+	 * @returns {HTMLElement[]}
 	 */
 	static tag(tag, parent) {
 		return Array.prototype.slice.call((parent || document).getElementsByTagName(tag));
@@ -32,6 +41,9 @@ class DOM {
 	
 	/**
 	 * Returns the first child element containing the specified class. Parent defaults to the entire document.
+	 * @param {string} cls
+	 * @param {HTMLElement} [parent]
+	 * @returns {HTMLElement}
 	 */
 	static fcls(cls, parent) {
 		return (parent || document).getElementsByClassName(cls)[0];

app/Resources/Viewer/scripts/gui.mjs

@@ -1,10 +1,16 @@
-const GUI = (function() {
+import dom from "./dom.mjs";
+import discord from "./discord.mjs";
+import settings from "./settings.mjs";
+import state from "./state.mjs";
+
+export default (function() {
 	let eventOnOptMessagesPerPageChanged;
 	let eventOnOptMessageFilterChanged;
 	let eventOnNavButtonClicked;
 	
 	const getActiveFilter = function() {
-		const active = DOM.fcls("active", DOM.id("opt-filter-list"));
+		/** @type HTMLSelectElement */
+		const active = dom.fcls("active", dom.id("opt-filter-list"));
 		
 		return active && active.value !== "" ? {
 			"type": active.getAttribute("data-filter-type"),
@@ -17,12 +23,12 @@ const GUI = (function() {
 	};
 	
 	const showModal = function(width, html) {
-		const dialog = DOM.id("dialog");
+		const dialog = dom.id("dialog");
 		dialog.innerHTML = html;
 		dialog.style.width = width + "px";
 		dialog.style.marginLeft = (-width / 2) + "px";
 		
-		DOM.id("modal").classList.add("visible");
+		dom.id("modal").classList.add("visible");
 		return dialog;
 	};
 	
@@ -38,9 +44,9 @@ const GUI = (function() {
 <label><input id='dht-cfg-animemoji' type='checkbox'> Animated Emoji</label><br>`);
 		
 		const setupCheckBox = function(id, settingName) {
-			const ele = DOM.id(id);
+			const ele = dom.id(id);
-			ele.checked = SETTINGS[settingName];
+			ele.checked = settings[settingName];
-			ele.addEventListener("change", () => SETTINGS[settingName] = ele.checked);
+			ele.addEventListener("change", () => settings[settingName] = ele.checked);
 		};
 		
 		setupCheckBox("dht-cfg-imgpreviews", "enableImagePreviews");
@@ -65,20 +71,20 @@ const GUI = (function() {
 		// ---------
 		
 		setup() {
-			const inputMessageFilter = DOM.id("opt-messages-filter");
+			const inputMessageFilter = dom.id("opt-messages-filter");
-			const containerFilterList = DOM.id("opt-filter-list");
+			const containerFilterList = dom.id("opt-filter-list");
 			
 			const resetActiveFilter = function() {
 				inputMessageFilter.value = "";
 				inputMessageFilter.dispatchEvent(new Event("change"));
 				
-				DOM.id("opt-filter-contents").value = "";
+				dom.id("opt-filter-contents").value = "";
 			};
 			
 			inputMessageFilter.value = ""; // required to prevent browsers from remembering old value
 			
 			inputMessageFilter.addEventListener("change", () => {
-				DOM.cls("active", containerFilterList).forEach(ele => ele.classList.remove("active"));
+				dom.cls("active", containerFilterList).forEach(ele => ele.classList.remove("active"));
 				
 				if (inputMessageFilter.value) {
 					containerFilterList.querySelector("[data-filter-type='" + inputMessageFilter.value + "']").classList.add("active");
@@ -91,11 +97,11 @@ const GUI = (function() {
 				ele.addEventListener(ele.tagName === "SELECT" ? "change" : "input", () => triggerFilterChanged());
 			});
 			
-			DOM.id("opt-messages-per-page").addEventListener("change", () => {
+			dom.id("opt-messages-per-page").addEventListener("change", () => {
 				eventOnOptMessagesPerPageChanged && eventOnOptMessagesPerPageChanged();
 			});
 			
-			DOM.tag("button", DOM.fcls("nav")).forEach(button => {
+			dom.tag("button", dom.fcls("nav")).forEach(button => {
 				button.disabled = true;
 				
 				button.addEventListener("click", () => {
@@ -103,34 +109,34 @@ const GUI = (function() {
 				});
 			});
 			
-			DOM.id("btn-settings").addEventListener("click", () => {
+			dom.id("btn-settings").addEventListener("click", () => {
 				showSettingsModal();
 			});
 			
-			DOM.id("btn-about").addEventListener("click", () => {
+			dom.id("btn-about").addEventListener("click", () => {
 				showInfoModal();
 			});
 			
-			DOM.id("messages").addEventListener("click", e => {
+			dom.id("messages").addEventListener("click", e => {
 				const jump = e.target.getAttribute("data-jump");
 				
 				if (jump) {
 					resetActiveFilter();
 					
-					const index = STATE.navigateToMessage(jump);
+					const index = state.navigateToMessage(jump);
 					
 					if (index === -1) {
 						alert("Message not found.");
 					}
 					else {
-						DOM.id("messages").children[index].scrollIntoView();
+						dom.id("messages").children[index].scrollIntoView();
 					}
 				}
 			});
 			
-			DOM.id("overlay").addEventListener("click", () => {
+			dom.id("overlay").addEventListener("click", () => {
-				DOM.id("modal").classList.remove("visible");
+				dom.id("modal").classList.remove("visible");
-				DOM.id("dialog").innerHTML = "";
+				dom.id("dialog").innerHTML = "";
 			});
 		},
 		
@@ -168,19 +174,19 @@ const GUI = (function() {
 		 */
 		getOptionMessagesPerPage() {
 			/** @type HTMLInputElement */
-			const messagesPerPage = DOM.id("opt-messages-per-page");
+			const messagesPerPage = dom.id("opt-messages-per-page");
 			return parseInt(messagesPerPage.value, 10);
 		},
 		
 		updateNavigation(currentPage, totalPages) {
-			DOM.id("nav-page-current").innerHTML = currentPage;
+			dom.id("nav-page-current").innerHTML = currentPage;
-			DOM.id("nav-page-total").innerHTML = totalPages || "?";
+			dom.id("nav-page-total").innerHTML = totalPages || "?";
 			
-			DOM.id("nav-first").disabled = currentPage === 1;
+			dom.id("nav-first").disabled = currentPage === 1;
-			DOM.id("nav-prev").disabled = currentPage === 1;
+			dom.id("nav-prev").disabled = currentPage === 1;
-			DOM.id("nav-pick").disabled = (totalPages || 0) <= 1;
+			dom.id("nav-pick").disabled = (totalPages || 0) <= 1;
-			DOM.id("nav-next").disabled = currentPage === (totalPages || 1);
+			dom.id("nav-next").disabled = currentPage === (totalPages || 1);
-			DOM.id("nav-last").disabled = currentPage === (totalPages || 1);
+			dom.id("nav-last").disabled = currentPage === (totalPages || 1);
 		},
 		
 		// --------------
@@ -191,7 +197,7 @@ const GUI = (function() {
 		 * Updates the channel list and sets up their click events. The callback is triggered whenever a channel is selected, and takes the channel ID as its argument.
 		 */
 		updateChannelList(channels, selected, callback) {
-			const eleChannels = DOM.id("channels");
+			const eleChannels = dom.id("channels");
 			
 			if (!channels) {
 				eleChannels.innerHTML = "";
@@ -201,11 +207,11 @@ const GUI = (function() {
 					channels = channels.filter(channel => channel.msgcount > 0);
 				}
 				
-				eleChannels.innerHTML = channels.map(channel => DISCORD.getChannelHTML(channel)).join("");
+				eleChannels.innerHTML = channels.map(channel => discord.getChannelHTML(channel)).join("");
 				
 				Array.prototype.forEach.call(eleChannels.children, ele => {
 					ele.addEventListener("click", () => {
-						const currentChannel = DOM.fcls("active", eleChannels);
+						const currentChannel = dom.fcls("active", eleChannels);
 						
 						if (currentChannel) {
 							currentChannel.classList.remove("active");
@@ -224,12 +230,12 @@ const GUI = (function() {
 		},
 		
 		updateMessageList(messages) {
-			DOM.id("messages").innerHTML = messages ? messages.map(message => DISCORD.getMessageHTML(message)).join("") : "";
+			dom.id("messages").innerHTML = messages ? messages.map(message => discord.getMessageHTML(message)).join("") : "";
 		},
 		
 		updateUserList(users) {
 			/** @type HTMLSelectElement */
-			const eleSelect = DOM.id("opt-filter-user");
+			const eleSelect = dom.id("opt-filter-user");
 			
 			while (eleSelect.length > 1) {
 				eleSelect.remove(1);
@@ -249,7 +255,7 @@ const GUI = (function() {
 		},
 		
 		scrollMessagesToTop() {
-			DOM.id("messages").scrollTop = 0;
+			dom.id("messages").scrollTop = 0;
 		}
 	};
 })();

app/Resources/Viewer/scripts/processor.mjs

@@ -1,16 +1,16 @@
-const PROCESSOR = {};
+import discord from "./discord.mjs";
 
 // ------------------------
 // Global filter generators
 // ------------------------
 
-PROCESSOR.FILTER = {
+const filter = {
 	byUser: ((userindex) => message => message.u === userindex),
 	byTime: ((timeStart, timeEnd) => message => message.t >= timeStart && message.t <= timeEnd),
 	byContents: ((substr) => message => ("m" in message ? message.m : "").indexOf(substr) !== -1),
 	byRegex: ((regex) => message => regex.test("m" in message ? message.m : "")),
-	withImages: (() => message => (message.e && message.e.some(embed => embed.type === "image")) || (message.a && message.a.some(DISCORD.isImageAttachment))),
+	withImages: (() => message => (message.e && message.e.some(embed => embed.type === "image")) || (message.a && message.a.some(discord.isImageAttachment))),
-	withDownloads: (() => message => message.a && message.a.some(attachment => !DISCORD.isImageAttachment(attachment))),
+	withDownloads: (() => message => message.a && message.a.some(attachment => !discord.isImageAttachment(attachment))),
 	withEmbeds: (() => message => message.e && message.e.length > 0),
 	withAttachments: (() => message => message.a && message.a.length > 0),
 	isEdited: (() => message => ("te" in message) ? message.te : (message.f & 1) === 1)
@@ -20,7 +20,7 @@ PROCESSOR.FILTER = {
 // Global sorters
 // --------------
 
-PROCESSOR.SORTER = {
+const sorter = {
 	oldestToNewest: (key1, key2) => {
 		if (key1.length === key2.length) {
 			return key1 > key2 ? 1 : key1 < key2 ? -1 : 0;
@@ -39,3 +39,8 @@ PROCESSOR.SORTER = {
 		}
 	}
 };
+
+export default {
+	FILTER: filter,
+	SORTER: sorter
+};

app/Resources/Viewer/scripts/settings.mjs

@@ -1,4 +1,4 @@
-const SETTINGS = (function() {
+export default (function() {
 	/**
 	 * @type {{}}
 	 * @property {Function} onSettingsChanged

app/Resources/Viewer/scripts/state.mjs

@@ -1,10 +1,12 @@
+import settings from "./settings.mjs";
+import processor from "./processor.mjs";
+
 // noinspection FunctionWithInconsistentReturnsJS
-const STATE = (function() {
+export default (function() {
 	/**
 	 * @type {{}}
 	 * @property {{}} users
-	 * @property {String[]} userindex
+	 * @property {{}} servers
-	 * @property {{}[]} servers
 	 * @property {{}} channels
 	 */
 	let loadedFileMeta;
@@ -17,20 +19,16 @@ const STATE = (function() {
 	let currentPage;
 	let messagesPerPage;
 	
-	const getUser = function(index) {
+	const getUser = function(id) {
-		return loadedFileMeta.users[loadedFileMeta.userindex[index]] || { "name": "<unknown>" };
+		return loadedFileMeta.users[id] || { "name": "<unknown>" };
-	};
-	
-	const getUserId = function(index) {
-		return loadedFileMeta.userindex[index];
 	};
 	
 	const getUserList = function() {
 		return loadedFileMeta ? loadedFileMeta.users : [];
 	};
 	
-	const getServer = function(index) {
+	const getServer = function(id) {
-		return loadedFileMeta.servers[index] || { "name": "<unknown>", "type": "unknown" };
+		return loadedFileMeta.servers[id] || { "name": "<unknown>", "type": "unknown" };
 	};
 	
 	const generateChannelHierarchy = function() {
@@ -204,7 +202,7 @@ const STATE = (function() {
 			 */
 			const message = messages[key];
 			const user = getUser(message.u);
-			const avatar = user.avatar ? { id: getUserId(message.u), path: user.avatar } : null;
+			const avatar = user.avatar ? { id: message.u, path: user.avatar } : null;
 			
 			const obj = {
 				user,
@@ -232,7 +230,7 @@ const STATE = (function() {
 			if ("r" in message) {
 				const replyMessage = getMessageById(message.r);
 				const replyUser = replyMessage ? getUser(replyMessage.u) : null;
-				const replyAvatar = replyUser && replyUser.avatar ? { id: getUserId(replyMessage.u), path: replyUser.avatar } : null;
+				const replyAvatar = replyUser && replyUser.avatar ? { id: replyMessage.u, path: replyUser.avatar } : null;
 				
 				obj["reply"] = replyMessage ? {
 					"id": message.r,
@@ -290,20 +288,17 @@ const STATE = (function() {
 			eventOnUsersRefreshed = callback;
 		},
 		
-		/**
+		uploadFile(meta, data) {
-		 * @param {{ meta, data }} file
-		 */
-		uploadFile(file) {
 			if (loadedFileMeta != null) {
 				throw "A file is already loaded!";
 			}
 			
-			if (!file || typeof file.meta !== "object" || typeof file.data !== "object") {
+			if (typeof meta !== "object" || typeof data !== "object") {
 				throw "Invalid file format!";
 			}
 			
-			loadedFileMeta = file.meta;
+			loadedFileMeta = meta;
-			loadedFileData = file.data;
+			loadedFileData = data;
 			loadedMessages = null;
 			
 			selectedChannel = null;
@@ -313,7 +308,7 @@ const STATE = (function() {
 			triggerChannelsRefreshed();
 			triggerMessagesRefreshed();
 			
-			SETTINGS.onSettingsChanged(() => triggerMessagesRefreshed());
+			settings.onSettingsChanged(() => triggerMessagesRefreshed());
 		},
 		
 		getChannelName(channel) {
@@ -335,7 +330,7 @@ const STATE = (function() {
 			currentPage = 1;
 			selectedChannel = channel;
 			
-			loadedMessages = getFilteredMessageKeys(channel).sort(PROCESSOR.SORTER.oldestToNewest);
+			loadedMessages = getFilteredMessageKeys(channel).sort(processor.SORTER.oldestToNewest);
 			triggerMessagesRefreshed();
 		},
 		
@@ -416,23 +411,23 @@ const STATE = (function() {
 		setActiveFilter(filter) {
 			switch (filter ? filter.type : "") {
 				case "user":
-					filterFunction = PROCESSOR.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value));
+					filterFunction = processor.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value));
 					break;
 				
 				case "contents":
-					filterFunction = PROCESSOR.FILTER.byContents(filter.value);
+					filterFunction = processor.FILTER.byContents(filter.value);
 					break;
 				
 				case "withimages":
-					filterFunction = PROCESSOR.FILTER.withImages();
+					filterFunction = processor.FILTER.withImages();
 					break;
 				
 				case "withdownloads":
-					filterFunction = PROCESSOR.FILTER.withDownloads();
+					filterFunction = processor.FILTER.withDownloads();
 					break;
 				
 				case "edited":
-					filterFunction = PROCESSOR.FILTER.isEdited();
+					filterFunction = processor.FILTER.isEdited();
 					break;
 				
 				default:

app/Resources/Viewer/scripts/template.mjs

@@ -1,6 +1,8 @@
+import dom from "./dom.mjs";
+
 const TEMPLATE_REGEX = /{([^{}]+?)}/g;
 
-class TEMPLATE {
+export default class {
 	constructor(contents) {
 		this.contents = contents;
 	};
@@ -11,10 +13,10 @@ class TEMPLATE {
 			
 			if (processor) {
 				const updated = processor(match, value);
-				return typeof updated === "undefined" ? DOM.escapeHTML(value) : updated;
+				return typeof updated === "undefined" ? dom.escapeHTML(value) : updated;
 			}
 			
-			return DOM.escapeHTML(value);
+			return dom.escapeHTML(value);
 		});
 	}
 }

app/Resources/Viewer/styles/channels.css

@@ -3,18 +3,24 @@
   min-width: 215px;
   max-width: 300px;
   overflow-y: auto;
+  color: #eee;
   background-color: #1c1e22;
+  font-size: 15px;
 }
 
-#channels > div {
+#channels > div.loading {
+  margin: 0 auto;
+  width: 150px;
+  height: 150px;
+}
+
+#channels > div.channel {
   cursor: pointer;
   padding: 10px 12px;
-  color: #eee;
-  font-size: 15px;
   border-bottom: 1px solid #333333;
 }
 
-#channels > div:hover, #channels > div.active {
+#channels > div.channel:hover, #channels > div.channel.active {
   background-color: #282b30;
 }
 

app/Resources/Viewer/styles/main.css

@@ -11,3 +11,20 @@ body {
   display: flex;
   flex-direction: row;
 }
+
+.loading {
+  position: relative;
+  --loading-backdrop: rgba(0, 0, 0, 0);
+}
+
+.loading::after {
+  content: "";
+  background: var(--loading-backdrop)
+  url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 300' preserveAspectRatio='xMidYMid'%3E %3Ccircle cx='150' cy='150' fill='none' stroke='%237983f5' stroke-width='8' r='42' stroke-dasharray='198 68'%3E %3CanimateTransform attributeName='transform' type='rotate' repeatCount='indefinite' dur='1.25s' values='0 150 150;360 150 150' keyTimes='0;1' /%3E %3C/circle%3E %3C/svg%3E")
+  no-repeat center center;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}

app/Resources/Viewer/styles/messages.css

@@ -107,23 +107,11 @@
 }
 
 .message .thumbnail {
-  position: relative;
+  --loading-backdrop: rgba(0, 0, 0, 0.75);
   max-width: calc(100% - 20px);
   max-height: 320px;
 }
 
-.message .thumbnail.loading::after {
-  content: "";
-  background: rgba(0, 0, 0, 0.75)
-              url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 300' preserveAspectRatio='xMidYMid'%3E %3Ccircle cx='150' cy='150' fill='none' stroke='%237983f5' stroke-width='8' r='42' stroke-dasharray='198 68'%3E %3CanimateTransform attributeName='transform' type='rotate' repeatCount='indefinite' dur='1.25s' values='0 150 150;360 150 150' keyTimes='0;1' /%3E %3C/circle%3E %3C/svg%3E")
-              no-repeat center center;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  height: 100%;
-}
-
 .message .thumbnail img {
   width: auto;
   max-width: 100%;

app/Server/Data/Aggregations/DownloadStatusStatistics.cs

@@ -1,15 +1,19 @@
 namespace DHT.Server.Data.Aggregations;
 
 public sealed class DownloadStatusStatistics {
-	public int EnqueuedCount { get; internal set; }
+	public int PendingCount { get; internal init; }
-	public ulong EnqueuedSize { get; internal set; }
+	public ulong PendingTotalSize { get; internal init; }
+	public int PendingWithUnknownSizeCount { get; internal init; }
 
-	public int SuccessfulCount { get; internal set; }
+	public int SuccessfulCount { get; internal init; }
-	public ulong SuccessfulSize { get; internal set; }
+	public ulong SuccessfulTotalSize { get; internal init; }
+	public int SuccessfulWithUnknownSizeCount { get; internal init; }
 
-	public int FailedCount { get; internal set; }
+	public int FailedCount { get; internal init; }
-	public ulong FailedSize { get; internal set; }
+	public ulong FailedTotalSize { get; internal init; }
+	public int FailedWithUnknownSizeCount { get; internal init; }
 	
-	public int SkippedCount { get; internal set; }
+	public int SkippedCount { get; internal init; }
-	public ulong SkippedSize { get; internal set; }
+	public ulong SkippedTotalSize { get; internal init; }
+	public int SkippedWithUnknownSizeCount { get; internal init; }
 }

app/Server/Data/Download.cs

@@ -1,33 +1,17 @@
-using System;
-using System.Net;
-using DHT.Server.Download;
-
 namespace DHT.Server.Data;
 
-public readonly struct Download {
+public sealed class Download {
-	internal static Download NewSuccess(DownloadItem item, byte[] data) {
-		return new Download(item.NormalizedUrl, item.DownloadUrl, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
-	}
-
-	internal static Download NewFailure(DownloadItem item, HttpStatusCode? statusCode, ulong size) {
-		return new Download(item.NormalizedUrl, item.DownloadUrl, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
-	}
-
 	public string NormalizedUrl { get; }
 	public string DownloadUrl { get; }
 	public DownloadStatus Status { get; }
-	public ulong Size { get; }
+	public string? Type { get; }
-	public byte[]? Data { get; }
+	public ulong? Size { get; }
 
-	internal Download(string normalizedUrl, string downloadUrl, DownloadStatus status, ulong size, byte[]? data = null) {
+	internal Download(string normalizedUrl, string downloadUrl, DownloadStatus status, string? type, ulong? size) {
 		NormalizedUrl = normalizedUrl;
 		DownloadUrl = downloadUrl;
 		Status = status;
+		Type = type;
 		Size = size;
-		Data = data;
-	}
-
-	internal Download WithData(byte[] data) {
-		return new Download(NormalizedUrl, DownloadUrl, Status, Size, data);
 	}
 }

app/Server/Data/DownloadStatus.cs

@@ -6,8 +6,9 @@ namespace DHT.Server.Data;
 /// Extends <see cref="HttpStatusCode"/> with custom status codes in the range 0-99.
 /// </summary>
 public enum DownloadStatus {
-	Enqueued = 0,
+	Pending = 0,
 	GenericError = 1,
 	Downloading = 2,
+	LastCustomCode = 99,
 	Success = HttpStatusCode.OK
 }

app/Server/Data/DownloadedAttachment.cs

@@ -1,6 +0,0 @@
-namespace DHT.Server.Data;
-
-public readonly struct DownloadedAttachment {
-	public string? Type { get; internal init; }
-	public byte[] Data { get; internal init; }
-}

app/Server/Data/Embeds/DiscordEmbedJson.cs

@@ -0,0 +1,17 @@
+namespace DHT.Server.Data.Embeds;
+
+sealed class DiscordEmbedJson {
+	public string? Type { get; set; }
+	public string? Url { get; set; }
+	
+	public JsonImage? Image { get; set; }
+	public JsonImage? Thumbnail { get; set; }
+	public JsonImage? Video { get; set; }
+
+	public sealed class JsonImage {
+		public string? Url { get; set; }
+		public string? ProxyUrl { get; set; }
+		public int? Width { get; set; }
+		public int? Height { get; set; }
+	}
+}

app/Server/Data/Embeds/DiscordEmbedJsonContext.cs

@@ -0,0 +1,7 @@
+using System.Text.Json.Serialization;
+
+namespace DHT.Server.Data.Embeds;
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, GenerationMode = JsonSourceGenerationMode.Default)]
+[JsonSerializable(typeof(DiscordEmbedJson))]
+sealed partial class DiscordEmbedJsonContext : JsonSerializerContext;

app/Server/Data/Filters/AttachmentFilter.cs

@@ -1,15 +0,0 @@
-namespace DHT.Server.Data.Filters;
-
-public sealed class AttachmentFilter {
-	public ulong? MaxBytes { get; set; } = null;
-
-	public DownloadItemRules? DownloadItemRule { get; set; } = null;
-
-	public bool IsEmpty => MaxBytes == null &&
-	                       DownloadItemRule == null;
-
-	public enum DownloadItemRules {
-		OnlyNotPresent,
-		OnlyPresent
-	}
-}

app/Server/Data/Filters/DownloadItemFilter.cs

@@ -3,8 +3,10 @@ using System.Collections.Generic;
 namespace DHT.Server.Data.Filters;
 
 public sealed class DownloadItemFilter {
-	public HashSet<DownloadStatus>? IncludeStatuses { get; init; } = null;
+	public HashSet<DownloadStatus>? IncludeStatuses { get; set; } = null;
-	public HashSet<DownloadStatus>? ExcludeStatuses { get; init; } = null;
+	public HashSet<DownloadStatus>? ExcludeStatuses { get; set; } = null;
 	
-	public bool IsEmpty => IncludeStatuses == null && ExcludeStatuses == null;
+	public ulong? MaxBytes { get; set; } = null;
+
+	public bool IsEmpty => IncludeStatuses == null && ExcludeStatuses == null && MaxBytes == null;
 }

app/Server/Database/DatabaseExtensions.cs

@@ -25,8 +25,10 @@ public static class DatabaseExtensions {
 		
 		await target.Messages.Add(batchedMessages);
 
-		await foreach (var download in source.Downloads.GetWithoutData()) {
+		await foreach (var download in source.Downloads.Get()) {
-			await target.Downloads.AddDownload(download.Status == DownloadStatus.Success ? await source.Downloads.HydrateWithData(download) : download);
+			if (download.Status != DownloadStatus.Success || !await source.Downloads.GetDownloadData(download.NormalizedUrl, stream => target.Downloads.AddDownload(download, stream))) {
+				await target.Downloads.AddDownload(download, stream: null);
+			}
 		}
 	}
 }

app/Server/Database/DummyDatabaseFile.cs

@@ -14,7 +14,6 @@ sealed class DummyDatabaseFile : IDatabaseFile {
 	public IServerRepository Servers { get; } = new IServerRepository.Dummy();
 	public IChannelRepository Channels { get; } = new IChannelRepository.Dummy();
 	public IMessageRepository Messages { get; } = new IMessageRepository.Dummy();
-	public IAttachmentRepository Attachments { get; } = new IAttachmentRepository.Dummy();
 	public IDownloadRepository Downloads { get; } = new IDownloadRepository.Dummy();
 	
 	private DummyDatabaseFile() {}

app/Server/Database/Exceptions/DatabaseTooNewException.cs

@@ -5,9 +5,9 @@ namespace DHT.Server.Database.Exceptions;
 
 public sealed class DatabaseTooNewException : Exception {
 	public int DatabaseVersion { get; }
-	public int CurrentVersion => Schema.Version;
+	public int CurrentVersion => SqliteSchema.Version;
 
-	internal DatabaseTooNewException(int databaseVersion) : base("Database is too new: " + databaseVersion + " > " + Schema.Version) {
+	internal DatabaseTooNewException(int databaseVersion) : base("Database is too new: " + databaseVersion + " > " + SqliteSchema.Version) {
 		this.DatabaseVersion = databaseVersion;
 	}
 }

app/Server/Database/Export/Snowflake.cs

@@ -1,3 +1,5 @@
 namespace DHT.Server.Database.Export;
 
-readonly record struct Snowflake(ulong Id);
+readonly record struct Snowflake(ulong Id) {
+	public static implicit operator Snowflake(ulong id) => new (id);
+}

app/Server/Database/Export/Strategy/IViewerExportStrategy.cs

@@ -1,7 +0,0 @@
-using DHT.Server.Data;
-
-namespace DHT.Server.Database.Export.Strategy;
-
-public interface IViewerExportStrategy {
-	string GetAttachmentUrl(Attachment attachment);
-}

app/Server/Database/Export/Strategy/LiveViewerExportStrategy.cs

@@ -1,18 +0,0 @@
-using System.Net;
-using DHT.Server.Data;
-
-namespace DHT.Server.Database.Export.Strategy;
-
-public sealed class LiveViewerExportStrategy : IViewerExportStrategy {
-	private readonly string safePort;
-	private readonly string safeToken;
-
-	public LiveViewerExportStrategy(ushort port, string token) {
-		this.safePort = port.ToString();
-		this.safeToken = WebUtility.UrlEncode(token);
-	}
-
-	public string GetAttachmentUrl(Attachment attachment) {
-		return "http://127.0.0.1:" + safePort + "/get-attachment/" + WebUtility.UrlEncode(attachment.NormalizedUrl) + "?token=" + safeToken;
-	}
-}

app/Server/Database/Export/Strategy/StandaloneViewerExportStrategy.cs

@@ -1,18 +0,0 @@
-using DHT.Server.Data;
-
-namespace DHT.Server.Database.Export.Strategy;
-
-public sealed class StandaloneViewerExportStrategy : IViewerExportStrategy {
-	public static StandaloneViewerExportStrategy Instance { get; } = new ();
-
-	private StandaloneViewerExportStrategy() {}
-
-	public string GetAttachmentUrl(Attachment attachment) {
-		// The normalized URL will not load files from Discord CDN once the time limit is enforced.
-		
-		// The downloaded URL would work, but only for a limited time, so it is better for the links to not work
-		// rather than give users a false sense of security.
-		
-		return attachment.NormalizedUrl;
-	}
-}

app/Server/Database/Export/ViewerJson.cs

@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
 
 namespace DHT.Server.Database.Export;
 
-sealed class ViewerJson {
+static class ViewerJson {
-	public required JsonMeta Meta { get; init; }
-	public required Dictionary<Snowflake, Dictionary<Snowflake, JsonMessage>> Data { get; init; }
-
 	public sealed class JsonMeta {
 		public required Dictionary<Snowflake, JsonUser> Users { get; init; }
-		public required List<Snowflake> Userindex { get; init; }
+		public required Dictionary<Snowflake, JsonServer> Servers { get; init; }
-		public required List<JsonServer> Servers { get; init; }
 		public required Dictionary<Snowflake, JsonChannel> Channels { get; init; }
 	}
 	
@@ -30,7 +26,7 @@ sealed class ViewerJson {
 	}
 
 	public sealed class JsonChannel {
-		public required int Server { get; init; }
+		public required Snowflake Server { get; init; }
 		public required string Name { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
@@ -47,7 +43,9 @@ sealed class ViewerJson {
 	}
 
 	public sealed class JsonMessage {
-		public required int U { get; init; }
+		public required Snowflake Id { get; init; }
+		public required Snowflake C { get; init; }
+		public required Snowflake U { get; init; }
 		public required long T { get; init; }
 
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

app/Server/Database/Export/ViewerJsonExport.cs

@@ -2,118 +2,103 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
-using DHT.Server.Database.Export.Strategy;
 using DHT.Utils.Logging;
 
 namespace DHT.Server.Database.Export;
 
-public static class ViewerJsonExport {
+static class ViewerJsonExport {
 	private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
 
-	public static async Task Generate(Stream stream, IViewerExportStrategy strategy, IDatabaseFile db, MessageFilter? filter = null) {
+	public static async Task GetMetadata(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
 		var perf = Log.Start();
 
-		var includedUserIds = new HashSet<ulong>();
+		var includedChannels = new List<Channel>();
-		var includedChannelIds = new HashSet<ulong>();
 		var includedServerIds = new HashSet<ulong>();
 
-		var includedMessages = await db.Messages.Get(filter).ToListAsync();
+		var channelIdFilter = filter?.ChannelIds;
-		var includedChannels = new List<Channel>();
 
-		foreach (var message in includedMessages) {
+		await foreach (var channel in db.Channels.Get(cancellationToken)) {
-			includedUserIds.Add(message.Sender);
+			if (channelIdFilter == null || channelIdFilter.Contains(channel.Id)) {
-			includedChannelIds.Add(message.Channel);
-		}
-
-		await foreach (var channel in db.Channels.Get()) {
-			if (includedChannelIds.Contains(channel.Id)) {
 				includedChannels.Add(channel);
 				includedServerIds.Add(channel.Server);
 			}
 		}
 
-		var (users, userIndex, userIndices) = await GenerateUserList(db, includedUserIds);
+		var users = await GenerateUserList(db, cancellationToken);
-		var (servers, serverIndices) = await GenerateServerList(db, includedServerIds);
+		var servers = await GenerateServerList(db, includedServerIds, cancellationToken);
-		var channels = GenerateChannelList(includedChannels, serverIndices);
+		var channels = GenerateChannelList(includedChannels);
+
+		var meta = new ViewerJson.JsonMeta {
+			Users = users,
+			Servers = servers,
+			Channels = channels
+		};
 
 		perf.Step("Collect database data");
 
-		var value = new ViewerJson {
+		await JsonSerializer.SerializeAsync(stream, meta, ViewerJsonMetadataContext.Default.JsonMeta, cancellationToken);
-			Meta = new ViewerJson.JsonMeta {
-				Users = users,
-				Userindex = userIndex,
-				Servers = servers,
-				Channels = channels
-			},
-			Data = GenerateMessageList(includedMessages, userIndices, strategy)
-		};
-
-		perf.Step("Generate value object");
-
-		await JsonSerializer.SerializeAsync(stream, value, ViewerJsonContext.Default.ViewerJson);
 
 		perf.Step("Serialize to JSON");
 		perf.End();
 	}
 
-	private static async Task<(Dictionary<Snowflake, ViewerJson.JsonUser> Users, List<Snowflake> UserIndex, Dictionary<ulong, int> UserIndices)> GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds) {
+	public static async Task GetMessages(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
-		var users = new Dictionary<Snowflake, ViewerJson.JsonUser>();
+		var perf = Log.Start();
-		var userIndex = new List<Snowflake>();
-		var userIndices = new Dictionary<ulong, int>();
 
-		await foreach (var user in db.Users.Get()) {
+		ReadOnlyMemory<byte> newLine = "\n"u8.ToArray();
-			var id = user.Id;
+		
-			if (!userIds.Contains(id)) {
+		await foreach(var message in GenerateMessageList(db, filter, cancellationToken)) {
-				continue;
+			await JsonSerializer.SerializeAsync(stream, message, ViewerJsonMessageContext.Default.JsonMessage, cancellationToken);
+			await stream.WriteAsync(newLine, cancellationToken);
 		}
 
-			var idSnowflake = new Snowflake(id);
+		perf.Step("Generate and serialize messages to JSON");
-			userIndices[id] = users.Count;
+		perf.End();
-			userIndex.Add(idSnowflake);
+	}
 
-			users[idSnowflake] = new ViewerJson.JsonUser {
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonUser>> GenerateUserList(IDatabaseFile db, CancellationToken cancellationToken) {
+		var users = new Dictionary<Snowflake, ViewerJson.JsonUser>();
+
+		await foreach (var user in db.Users.Get(cancellationToken)) {
+			users[user.Id] = new ViewerJson.JsonUser {
 				Name = user.Name,
 				Avatar = user.AvatarUrl,
 				Tag = user.Discriminator
 			};
 		}
 
-		return (users, userIndex, userIndices);
+		return users;
 	}
 
-	private static async Task<(List<ViewerJson.JsonServer> Servers, Dictionary<ulong, int> ServerIndices)> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds) {
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonServer>> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds, CancellationToken cancellationToken) {
-		var servers = new List<ViewerJson.JsonServer>();
+		var servers = new Dictionary<Snowflake, ViewerJson.JsonServer>();
-		var serverIndices = new Dictionary<ulong, int>();
 
-		await foreach (var server in db.Servers.Get()) {
+		await foreach (var server in db.Servers.Get(cancellationToken)) {
-			var id = server.Id;
+			if (!serverIds.Contains(server.Id)) {
-			if (!serverIds.Contains(id)) {
 				continue;
 			}
 
-			serverIndices[id] = servers.Count;
+			servers[server.Id] = new ViewerJson.JsonServer {
-			
-			servers.Add(new ViewerJson.JsonServer {
 				Name = server.Name,
 				Type = ServerTypes.ToJsonViewerString(server.Type)
-			});
+			};
 		}
 
-		return (servers, serverIndices);
+		return servers;
 	}
 
-	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels, Dictionary<ulong, int> serverIndices) {
+	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels) {
 		var channels = new Dictionary<Snowflake, ViewerJson.JsonChannel>();
 
 		foreach (var channel in includedChannels) {
-			var channelIdSnowflake = new Snowflake(channel.Id);
+			channels[channel.Id] = new ViewerJson.JsonChannel {
-			
+				Server = channel.Server,
-			channels[channelIdSnowflake] = new ViewerJson.JsonChannel {
-				Server = serverIndices[channel.Server],
 				Name = channel.Name,
 				Parent = channel.ParentId?.ToString(),
 				Position = channel.Position,
@@ -125,26 +110,20 @@ public static class ViewerJsonExport {
 		return channels;
 	}
 
-	private static Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>> GenerateMessageList(List<Message> includedMessages, Dictionary<ulong, int> userIndices, IViewerExportStrategy strategy) {
+	private static async IAsyncEnumerable<ViewerJson.JsonMessage> GenerateMessageList(IDatabaseFile db, MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
-		var data = new Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>>();
+		await foreach (var message in db.Messages.Get(filter, cancellationToken)) {
-
+			yield return new ViewerJson.JsonMessage {
-		foreach (var grouping in includedMessages.GroupBy(static message => message.Channel)) {
+				Id = message.Id,
-			var channelIdSnowflake = new Snowflake(grouping.Key);
+				C = message.Channel,
-			var channelData = new Dictionary<Snowflake, ViewerJson.JsonMessage>();
+				U = message.Sender,
-
-			foreach (var message in grouping) {
-				var messageIdSnowflake = new Snowflake(message.Id);
-				
-				channelData[messageIdSnowflake] = new ViewerJson.JsonMessage {
-					U = userIndices[message.Sender],
 				T = message.Timestamp,
 				M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
 				Te = message.EditTimestamp,
 				R = message.RepliedToId?.ToString(),
 
-					A = message.Attachments.IsEmpty ? null : message.Attachments.Select(attachment => {
+				A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => {
 					var a = new ViewerJson.JsonMessageAttachment {
-							Url = strategy.GetAttachmentUrl(attachment),
+						Url = attachment.DownloadUrl,
 						Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl
 					};
 
@@ -166,10 +145,5 @@ public static class ViewerJsonExport {
 				}).ToArray()
 			};
 		}
-
-			data[channelIdSnowflake] = channelData;
-		}
-
-		return data;
 	}
 }

app/Server/Database/Export/ViewerJsonMessageContext.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace DHT.Server.Database.Export;
+
+[JsonSourceGenerationOptions(
+	Converters = [typeof(SnowflakeJsonSerializer)],
+	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
+	GenerationMode = JsonSourceGenerationMode.Default
+)]
+[JsonSerializable(typeof(ViewerJson.JsonMessage))]
+sealed partial class ViewerJsonMessageContext : JsonSerializerContext;

app/Server/Database/Export/ViewerJsonMetadataContext.cs

@@ -7,5 +7,5 @@ namespace DHT.Server.Database.Export;
 	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
 	GenerationMode = JsonSourceGenerationMode.Default
 )]
-[JsonSerializable(typeof(ViewerJson))]
+[JsonSerializable(typeof(ViewerJson.JsonMeta))]
-sealed partial class ViewerJsonContext : JsonSerializerContext;
+sealed partial class ViewerJsonMetadataContext : JsonSerializerContext;

app/Server/Database/IDatabaseFile.cs

@@ -11,7 +11,6 @@ public interface IDatabaseFile : IAsyncDisposable {
 	IServerRepository Servers { get; }
 	IChannelRepository Channels { get; }
 	IMessageRepository Messages { get; }
-	IAttachmentRepository Attachments { get; }
 	IDownloadRepository Downloads { get; }
 
 	Task Vacuum();

app/Server/Database/Import/LegacyArchiveImport.cs

@@ -161,7 +161,7 @@ public static class LegacyArchiveImport {
 
 		var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null;
 		if (messagesObj == null) {
-			return Array.Empty<Message>();
+			return [];
 		}
 
 		return messagesObj.Value.EnumerateObject().Select(item => {

app/Server/Database/Repositories/IAttachmentRepository.cs

@@ -1,21 +0,0 @@
-using System;
-using System.Reactive.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using DHT.Server.Data.Filters;
-
-namespace DHT.Server.Database.Repositories;
-
-public interface IAttachmentRepository {
-	IObservable<long> TotalCount { get; }
-
-	Task<long> Count(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
-
-	internal sealed class Dummy : IAttachmentRepository {
-		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
-		
-		public Task<long> Count(AttachmentFilter? filter = null, CancellationToken cancellationToken = default) {
-			return Task.FromResult(0L);
-		}
-	}
-}

app/Server/Database/Repositories/IChannelRepository.cs

@@ -15,7 +15,7 @@ public interface IChannelRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Channel> Get();
+	IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IChannelRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -28,7 +28,7 @@ public interface IChannelRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Channel> Get() {
+		public IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Channel>();
 		}
 	}

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -1,10 +1,10 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using DHT.Server.Data;
 using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
 using DHT.Server.Download;
@@ -14,55 +14,61 @@ namespace DHT.Server.Database.Repositories;
 public interface IDownloadRepository {
 	IObservable<long> TotalCount { get; }
 
-	Task AddDownload(Data.Download download);
+	Task AddDownload(Data.Download item, Stream? stream);
 
-	Task<DownloadStatusStatistics> GetStatistics(CancellationToken cancellationToken = default);
+	Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Data.Download> GetWithoutData();
+	Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken = default);
 	
-	Task<Data.Download> HydrateWithData(Data.Download download);
+	IAsyncEnumerable<Data.Download> Get();
 
-	Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl);
+	Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor);
 	
-	Task<int> EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
+	Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken = default);
 
-	IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken = default);
+	IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default);
 	
-	Task RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);
+	Task MoveDownloadingItemsBackToQueue(CancellationToken cancellationToken = default);
+	
+	Task<int> RetryFailed(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IDownloadRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
 
-		public Task AddDownload(Data.Download download) {
+		public Task AddDownload(Data.Download item, Stream? stream) {
 			return Task.CompletedTask;
 		}
 
-		public Task<DownloadStatusStatistics> GetStatistics(CancellationToken cancellationToken) {
+		public Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken) {
+			return Task.FromResult(0L);
+		}
+
+		public Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken) {
 			return Task.FromResult(new DownloadStatusStatistics());
 		}
 
-		public IAsyncEnumerable<Data.Download> GetWithoutData() {
+		public IAsyncEnumerable<Data.Download> Get() {
 			return AsyncEnumerable.Empty<Data.Download>();
 		}
 
-		public Task<Data.Download> HydrateWithData(Data.Download download) {
+		public Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) {
-			return Task.FromResult(download);
+			return Task.FromResult(false);
 		}
 
-		public Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl) {
+		public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken) {
-			return Task.FromResult<DownloadedAttachment?>(null);
+			return Task.FromResult(false);
 		}
 
-		public Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
+		public IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken) {
-			return Task.FromResult(0);
-		}
-
-		public IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<DownloadItem>();
 		}
 
-		public Task RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
+		public Task MoveDownloadingItemsBackToQueue(CancellationToken cancellationToken) {
 			return Task.CompletedTask;
 		}
+
+		public Task<int> RetryFailed(CancellationToken cancellationToken) {
+			return Task.FromResult(0);
+		}
 	}
 }

app/Server/Database/Repositories/IMessageRepository.cs

@@ -16,7 +16,7 @@ public interface IMessageRepository {
 	
 	Task<long> Count(MessageFilter? filter = null, CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Message> Get(MessageFilter? filter = null);
+	IAsyncEnumerable<Message> Get(MessageFilter? filter = null, CancellationToken cancellationToken = default);
 	
 	IAsyncEnumerable<ulong> GetIds(MessageFilter? filter = null);
 	
@@ -33,7 +33,7 @@ public interface IMessageRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Message> Get(MessageFilter? filter) {
+		public IAsyncEnumerable<Message> Get(MessageFilter? filter, CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Message>();
 		}
 

app/Server/Database/Repositories/IServerRepository.cs

@@ -14,7 +14,7 @@ public interface IServerRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Data.Server> Get();
+	IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IServerRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -27,7 +27,7 @@ public interface IServerRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Data.Server> Get() {
+		public IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Data.Server>();
 		}
 	}

app/Server/Database/Repositories/IUserRepository.cs

@@ -15,7 +15,7 @@ public interface IUserRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<User> Get();
+	IAsyncEnumerable<User> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IUserRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -28,7 +28,7 @@ public interface IUserRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<User> Get() {
+		public IAsyncEnumerable<User> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<User>();
 		}
 	}

app/Server/Database/Sqlite/Repositories/BaseSqliteRepository.cs

@@ -1,16 +1,20 @@
 using System;
+using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using DHT.Utils.Logging;
 using DHT.Utils.Tasks;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
 abstract class BaseSqliteRepository : IDisposable {
-	private readonly ObservableThrottledTask<long> totalCountTask = new (TaskScheduler.Default);
+	private readonly ObservableThrottledTask<long> totalCountTask;
 
-	public IObservable<long> TotalCount => totalCountTask;
+	public IObservable<long> TotalCount { get; }
 
-	protected BaseSqliteRepository() {
+	protected BaseSqliteRepository(Log log) {
+		totalCountTask = new ObservableThrottledTask<long>(log, TaskScheduler.Default);
+		TotalCount = totalCountTask.DistinctUntilChanged();
 		UpdateTotalCount();
 	}
 

app/Server/Database/Sqlite/Repositories/SqliteAttachmentRepository.cs

@@ -1,28 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using DHT.Server.Data.Filters;
-using DHT.Server.Database.Repositories;
-using DHT.Server.Database.Sqlite.Utils;
-
-namespace DHT.Server.Database.Sqlite.Repositories;
-
-sealed class SqliteAttachmentRepository : BaseSqliteRepository, IAttachmentRepository {
-	private readonly SqliteConnectionPool pool;
-
-	public SqliteAttachmentRepository(SqliteConnectionPool pool) {
-		this.pool = pool;
-	}
-
-	internal new void UpdateTotalCount() {
-		base.UpdateTotalCount();
-	}
-	
-	public override Task<long> Count(CancellationToken cancellationToken) {
-		return Count(filter: null, cancellationToken);
-	}
-
-	public async Task<long> Count(AttachmentFilter? filter, CancellationToken cancellationToken) {
-		await using var conn = await pool.Take();
-		return await conn.ExecuteReaderAsync("SELECT COUNT(DISTINCT normalized_url) FROM attachments a" + filter.GenerateWhereClause("a"), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
-	}
-}

app/Server/Database/Sqlite/Repositories/SqliteChannelRepository.cs

@@ -1,24 +1,28 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Utils.Logging;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
 sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository {
+	private static readonly Log Log = Log.ForType<SqliteChannelRepository>();
+	
 	private readonly SqliteConnectionPool pool;
 
-	public SqliteChannelRepository(SqliteConnectionPool pool) {
+	public SqliteChannelRepository(SqliteConnectionPool pool) : base(Log) {
 		this.pool = pool;
 	}
 
 	public async Task Add(IReadOnlyList<Channel> channels) {
-		await using var conn = await pool.Take();
+		await using (var conn = await pool.Take()) {
+			await conn.BeginTransactionAsync();
 			
-		await using (var tx = await conn.BeginTransactionAsync()) {
 			await using var cmd = conn.Upsert("channels", [
 				("id", SqliteType.Integer),
 				("server", SqliteType.Integer),
@@ -40,7 +44,7 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
 				await cmd.ExecuteNonQueryAsync();
 			}
 
-			await tx.CommitAsync();
+			await conn.CommitTransactionAsync();
 		}
 
 		UpdateTotalCount();
@@ -51,13 +55,13 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM channels", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<Channel> Get() {
+	public async IAsyncEnumerable<Channel> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, server, name, parent_id, position, topic, nsfw FROM channels");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (reader.Read()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new Channel {
 				Id = reader.GetUint64(0),
 				Server = reader.GetUint64(1),

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -9,198 +10,274 @@ using DHT.Server.Data.Filters;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Utils;
 using DHT.Server.Download;
+using DHT.Utils.Logging;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
-sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepository {
+sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository {
-	private readonly SqliteConnectionPool pool;
+	private static readonly Log Log = Log.ForType<SqliteDownloadRepository>();
 
-	public SqliteDownloadRepository(SqliteConnectionPool pool) {
+	internal sealed class NewDownloadCollector : IAsyncDisposable {
-		this.pool = pool;
+		private readonly SqliteDownloadRepository repository;
+		private bool hasAdded = false;
+
+		private readonly SqliteCommand metadataCmd;
+
+		public NewDownloadCollector(SqliteDownloadRepository repository, ISqliteConnection conn) {
+			this.repository = repository;
+
+			metadataCmd = conn.Command(
+				"""
+				INSERT INTO download_metadata (normalized_url, download_url, status, type, size)
+				VALUES (:normalized_url, :download_url, :status, :type, :size)
+				ON CONFLICT DO NOTHING
+				"""
+			);
+			metadataCmd.Add(":normalized_url", SqliteType.Text);
+			metadataCmd.Add(":download_url", SqliteType.Text);
+			metadataCmd.Add(":status", SqliteType.Integer);
+			metadataCmd.Add(":type", SqliteType.Text);
+			metadataCmd.Add(":size", SqliteType.Integer);
 		}
 
-	public async Task AddDownload(Data.Download download) {
+		public async Task Add(Data.Download download) {
+			metadataCmd.Set(":normalized_url", download.NormalizedUrl);
+			metadataCmd.Set(":download_url", download.DownloadUrl);
+			metadataCmd.Set(":status", (int) download.Status);
+			metadataCmd.Set(":type", download.Type);
+			metadataCmd.Set(":size", download.Size);
+			hasAdded |= await metadataCmd.ExecuteNonQueryAsync() > 0;
+		}
+
+		public void OnCommitted() {
+			if (hasAdded) {
+				repository.UpdateTotalCount();
+			}
+		}
+
+		public async ValueTask DisposeAsync() {
+			await metadataCmd.DisposeAsync();
+		}
+	}
+
+	public async Task AddDownload(Data.Download item, Stream? stream) {
 		await using (var conn = await pool.Take()) {
-			await using var cmd = conn.Upsert("downloads", [
+			await conn.BeginTransactionAsync();
+			
+			await using var metadataCmd = conn.Upsert("download_metadata", [
 				("normalized_url", SqliteType.Text),
 				("download_url", SqliteType.Text),
 				("status", SqliteType.Integer),
+				("type", SqliteType.Text),
 				("size", SqliteType.Integer),
-				("blob", SqliteType.Blob)
 			]);
 
-			cmd.Set(":normalized_url", download.NormalizedUrl);
+			metadataCmd.Set(":normalized_url", item.NormalizedUrl);
-			cmd.Set(":download_url", download.DownloadUrl);
+			metadataCmd.Set(":download_url", item.DownloadUrl);
-			cmd.Set(":status", (int) download.Status);
+			metadataCmd.Set(":status", (int) item.Status);
-			cmd.Set(":size", download.Size);
+			metadataCmd.Set(":type", item.Type);
-			cmd.Set(":blob", download.Data);
+			metadataCmd.Set(":size", item.Size);
-			await cmd.ExecuteNonQueryAsync();
+			await metadataCmd.ExecuteNonQueryAsync();
+
+			if (stream == null) {
+				await using var deleteBlobCmd = conn.Command("DELETE FROM download_blobs WHERE normalized_url = :normalized_url");
+				deleteBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl);
+				await deleteBlobCmd.ExecuteNonQueryAsync();
+			}
+			else {
+				await using var upsertBlobCmd = conn.Command(
+					"""
+					INSERT INTO download_blobs (normalized_url, blob)
+					VALUES (:normalized_url, ZEROBLOB(:blob_length))
+					ON CONFLICT (normalized_url) DO UPDATE SET blob = excluded.blob
+					RETURNING rowid
+					"""
+				);
+				
+				upsertBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl);
+				upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, item.Size);
+				long rowid = await upsertBlobCmd.ExecuteLongScalarAsync();
+
+				await using var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid);
+				await stream.CopyToAsync(blob);
+			}
+
+			await conn.CommitTransactionAsync();
 		}
 
 		UpdateTotalCount();
 	}
 
-	public override async Task<long> Count(CancellationToken cancellationToken) {
+	public override Task<long> Count(CancellationToken cancellationToken) {
+		return Count(filter: null, cancellationToken);
+	}
+
+	public async Task<long> Count(DownloadItemFilter? filter, CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
-		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM downloads", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
+		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM download_metadata" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async Task<DownloadStatusStatistics> GetStatistics(CancellationToken cancellationToken) {
+	public async Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken) {
-		static async Task LoadUndownloadedStatistics(ISqliteConnection conn, DownloadStatusStatistics result, CancellationToken cancellationToken) {
+		nonSkippedFilter.IncludeStatuses = null;
+		nonSkippedFilter.ExcludeStatuses = null;
+		string nonSkippedFilterConditions = nonSkippedFilter.GenerateConditions().Build();
+
+		await using var conn = await pool.Take();
+
 		await using var cmd = conn.Command(
-				"""
+			$"""
-				SELECT IFNULL(COUNT(size), 0), IFNULL(SUM(size), 0)
-				FROM (SELECT MAX(a.size) size
-				      FROM attachments a
-				      WHERE a.normalized_url NOT IN (SELECT d.normalized_url FROM downloads d)
-				      GROUP BY a.normalized_url)
-				""");
-
-			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
-
-			if (reader.Read()) {
-				result.SkippedCount = reader.GetInt32(0);
-				result.SkippedSize = reader.GetUint64(1);
-			}
-		}
-
-		static async Task LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result, CancellationToken cancellationToken) {
-			await using var cmd = conn.Command(
-				"""
 			 SELECT
-				IFNULL(SUM(CASE WHEN status IN (:enqueued, :downloading) THEN 1 ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN (status = :downloading) OR (status = :pending AND {nonSkippedFilterConditions}) THEN 1 ELSE 0 END), 0),
-				IFNULL(SUM(CASE WHEN status IN (:enqueued, :downloading) THEN size ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN (status = :downloading) OR (status = :pending AND {nonSkippedFilterConditions}) THEN IFNULL(size, 0) ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN ((status = :downloading) OR (status = :pending AND {nonSkippedFilterConditions})) AND size IS NULL THEN 1 ELSE 0 END), 0),
 			 IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
-				IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status = :success THEN IFNULL(size, 0) ELSE 0 END), 0),
-				IFNULL(SUM(CASE WHEN status NOT IN (:enqueued, :downloading) AND status != :success THEN 1 ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status = :success AND size IS NULL THEN 1 ELSE 0 END), 0),
-				IFNULL(SUM(CASE WHEN status NOT IN (:enqueued, :downloading) AND status != :success THEN size ELSE 0 END), 0)
+			 IFNULL(SUM(CASE WHEN status NOT IN (:pending, :downloading, :success) THEN 1 ELSE 0 END), 0),
-				FROM downloads
+			 IFNULL(SUM(CASE WHEN status NOT IN (:pending, :downloading, :success) THEN IFNULL(size, 0) ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status NOT IN (:pending, :downloading, :success) AND size IS NULL THEN 1 ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status = :pending AND NOT ({nonSkippedFilterConditions}) THEN 1 ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status = :pending AND NOT ({nonSkippedFilterConditions}) THEN IFNULL(size, 0) ELSE 0 END), 0),
+			 IFNULL(SUM(CASE WHEN status = :pending AND NOT ({nonSkippedFilterConditions}) AND size IS NULL THEN 1 ELSE 0 END), 0)
+			 FROM download_metadata
 			 """
 		);
 
-			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+		cmd.AddAndSet(":pending", SqliteType.Integer, (int) DownloadStatus.Pending);
 		cmd.AddAndSet(":downloading", SqliteType.Integer, (int) DownloadStatus.Downloading);
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 
 		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-			if (reader.Read()) {
+		if (!await reader.ReadAsync(cancellationToken)) {
-				result.EnqueuedCount = reader.GetInt32(0);
+			return new DownloadStatusStatistics();
-				result.EnqueuedSize = reader.GetUint64(1);
-				result.SuccessfulCount = reader.GetInt32(2);
-				result.SuccessfulSize = reader.GetUint64(3);
-				result.FailedCount = reader.GetInt32(4);
-				result.FailedSize = reader.GetUint64(5);
-			}
 		}
 
-		var result = new DownloadStatusStatistics();
+		return new DownloadStatusStatistics {
-
+			PendingCount = reader.GetInt32(0),
-		await using var conn = await pool.Take();
+			PendingTotalSize = reader.GetUint64(1),
-		await LoadUndownloadedStatistics(conn, result, cancellationToken);
+			PendingWithUnknownSizeCount = reader.GetInt32(2),
-		await LoadSuccessStatistics(conn, result, cancellationToken);
+			SuccessfulCount = reader.GetInt32(3),
-		return result;
+			SuccessfulTotalSize = reader.GetUint64(4),
+			SuccessfulWithUnknownSizeCount = reader.GetInt32(5),
+			FailedCount = reader.GetInt32(6),
+			FailedTotalSize = reader.GetUint64(7),
+			FailedWithUnknownSizeCount = reader.GetInt32(8),
+			SkippedCount = reader.GetInt32(9),
+			SkippedTotalSize = reader.GetUint64(10),
+			SkippedWithUnknownSizeCount = reader.GetInt32(11)
+		};
 	}
 
-	public async IAsyncEnumerable<Data.Download> GetWithoutData() {
+	public async IAsyncEnumerable<Data.Download> Get() {
 		await using var conn = await pool.Take();
 
-		await using var cmd = conn.Command("SELECT normalized_url, download_url, status, size FROM downloads");
+		await using var cmd = conn.Command("SELECT normalized_url, download_url, status, type, size FROM download_metadata");
 		await using var reader = await cmd.ExecuteReaderAsync();
 
-		while (reader.Read()) {
+		while (await reader.ReadAsync()) {
 			string normalizedUrl = reader.GetString(0);
 			string downloadUrl = reader.GetString(1);
 			var status = (DownloadStatus) reader.GetInt32(2);
-			ulong size = reader.GetUint64(3);
+			string? type = reader.IsDBNull(3) ? null : reader.GetString(3);
+			ulong? size = reader.IsDBNull(4) ? null : reader.GetUint64(4);
 
-			yield return new Data.Download(normalizedUrl, downloadUrl, status, size);
+			yield return new Data.Download(normalizedUrl, downloadUrl, status, type, size);
 		}
 	}
 
-	public async Task<Data.Download> HydrateWithData(Data.Download download) {
+	public async Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) {
 		await using var conn = await pool.Take();
 
-		await using var cmd = conn.Command("SELECT blob FROM downloads WHERE normalized_url = :url");
+		await using var cmd = conn.Command("SELECT rowid FROM download_blobs WHERE normalized_url = :normalized_url");
-		cmd.AddAndSet(":url", SqliteType.Text, download.NormalizedUrl);
+		cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl);
 		
-		await using var reader = await cmd.ExecuteReaderAsync();
+		long rowid;
 		
-		if (reader.Read() && !reader.IsDBNull(0)) {
+		await using (var reader = await cmd.ExecuteReaderAsync()) {
-			return download.WithData((byte[]) reader["blob"]);
+			if (!await reader.ReadAsync()) {
-		}
+				return false;
-		else {
-			return download;
-		}
 			}
 
-	public async Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl) {
+			rowid = reader.GetInt64(0);
+		}
+		
+		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
+			await dataProcessor(blob);
+		}
+
+		return true;
+	}
+
+	public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command(
 			"""
-			SELECT a.type, d.blob FROM downloads d
+			SELECT dm.download_url, dm.type, db.rowid FROM download_metadata dm
-			LEFT JOIN attachments a ON d.normalized_url = a.normalized_url
+			JOIN download_blobs db ON dm.normalized_url = db.normalized_url
-			WHERE d.normalized_url = :normalized_url AND d.status = :success AND d.blob IS NOT NULL
+			WHERE dm.normalized_url = :normalized_url AND dm.status = :success IS NOT NULL
 			"""
 		);
 
 		cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl);
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 
-		await using var reader = await cmd.ExecuteReaderAsync();
+		string downloadUrl;
+		string? type;
+		long rowid;
 		
-		if (!reader.Read()) {
+		await using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) {
-			return null;
+			if (!await reader.ReadAsync(cancellationToken)) {
+				return false;
 			}
 
-		return new DownloadedAttachment {
+			downloadUrl = reader.GetString(0);
-			Type = reader.IsDBNull(0) ? null : reader.GetString(0),
+			type = reader.IsDBNull(1) ? null : reader.GetString(1);
-			Data = (byte[]) reader["blob"],
+			rowid = reader.GetInt64(2);
-		};
 		}
 		
-	public async Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
+		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
-		await using var conn = await pool.Take();
+			await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob, cancellationToken);
-
-		await using var cmd = conn.Command(
-			$"""
-			 INSERT INTO downloads (normalized_url, download_url, status, size)
-			 SELECT a.normalized_url, a.download_url, :enqueued, MAX(a.size)
-			 FROM attachments a
-			 {filter.GenerateWhereClause("a")}
-			 GROUP BY a.normalized_url
-			 """
-		);
-
-		cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
-		return await cmd.ExecuteNonQueryAsync(cancellationToken);
 		}
 
-	public async IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, [EnumeratorCancellation] CancellationToken cancellationToken) {
+		return true;
+	}
+
+	public async IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
+		filter.IncludeStatuses = [DownloadStatus.Pending];
+		filter.ExcludeStatuses = null;
+
 		var found = new List<DownloadItem>();
 
 		await using var conn = await pool.Take();
 
-		await using (var cmd = conn.Command("SELECT normalized_url, download_url, size FROM downloads WHERE status = :enqueued LIMIT :limit")) {
+		var sql = $"""
-			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+		           SELECT normalized_url, download_url, type, size
+		           FROM download_metadata
+		           {filter.GenerateConditions().BuildWhereClause()}
+		           LIMIT :limit
+		           """;
+
+		await using (var cmd = conn.Command(sql)) {
 			cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
 
 			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-			while (reader.Read()) {
+			while (await reader.ReadAsync(cancellationToken)) {
 				found.Add(new DownloadItem {
 					NormalizedUrl = reader.GetString(0),
 					DownloadUrl = reader.GetString(1),
-					Size = reader.GetUint64(2),
+					Type = reader.IsDBNull(2) ? null : reader.GetString(2),
+					Size = reader.IsDBNull(3) ? null : reader.GetUint64(3)
 				});
 			}
 		}
 
 		if (found.Count != 0) {
-			await using var cmd = conn.Command("UPDATE downloads SET status = :downloading WHERE normalized_url = :normalized_url AND status = :enqueued");
+			await using var cmd = conn.Command("UPDATE download_metadata SET status = :downloading WHERE normalized_url = :normalized_url AND status = :pending");
-			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+			cmd.AddAndSet(":pending", SqliteType.Integer, (int) DownloadStatus.Pending);
 			cmd.AddAndSet(":downloading", SqliteType.Integer, (int) DownloadStatus.Downloading);
 			cmd.Add(":normalized_url", SqliteType.Text);
 
@@ -214,17 +291,23 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor
 		}
 	}
 
-	public async Task RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
+	public async Task MoveDownloadingItemsBackToQueue(CancellationToken cancellationToken) {
-		await using (var conn = await pool.Take()) {
+		await using var conn = await pool.Take();
-			await conn.ExecuteAsync(
+
-				$"""
+		await using var cmd = conn.Command("UPDATE download_metadata SET status = :pending WHERE status = :downloading");
-				 -- noinspection SqlWithoutWhere
+		cmd.AddAndSet(":pending", SqliteType.Integer, (int) DownloadStatus.Pending);
-				 DELETE FROM downloads
+		cmd.AddAndSet(":downloading", SqliteType.Integer, (int) DownloadStatus.Downloading);
-				 {filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching)}
+		await cmd.ExecuteNonQueryAsync(cancellationToken);
-				 """
-			);
 	}
 
-		UpdateTotalCount();
+	public async Task<int> RetryFailed(CancellationToken cancellationToken) {
+		await using var conn = await pool.Take();
+
+		await using var cmd = conn.Command("UPDATE download_metadata SET status = :pending WHERE status = :generic_error OR (status > :last_custom_code AND status != :success)");
+		cmd.AddAndSet(":pending", SqliteType.Integer, (int) DownloadStatus.Pending);
+		cmd.AddAndSet(":generic_error", SqliteType.Integer, (int) DownloadStatus.GenericError);
+		cmd.AddAndSet(":last_custom_code", SqliteType.Integer, (int) DownloadStatus.LastCustomCode);
+		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
+		return await cmd.ExecuteNonQueryAsync(cancellationToken);
 	}
 }

app/Server/Database/Sqlite/Repositories/SqliteMessageRepository.cs

@@ -1,23 +1,28 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
+using DHT.Utils.Logging;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
 sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository {
-	private readonly SqliteConnectionPool pool;
+	private static readonly Log Log = Log.ForType<SqliteMessageRepository>();
-	private readonly SqliteAttachmentRepository attachments;
 	
-	public SqliteMessageRepository(SqliteConnectionPool pool, SqliteAttachmentRepository attachments) {
+	private readonly SqliteConnectionPool pool;
+	private readonly SqliteDownloadRepository downloads;
+
+	public SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : base(Log) {
 		this.pool = pool;
-		this.attachments = attachments;
+		this.downloads = downloads;
 	}
 
 	public async Task Add(IReadOnlyList<Message> messages) {
@@ -34,10 +39,8 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			await cmd.ExecuteNonQueryAsync();
 		}
 
-		bool addedAttachments = false;
-
 		await using (var conn = await pool.Take()) {
-			await using var tx = await conn.BeginTransactionAsync();
+			await conn.BeginTransactionAsync();
 
 			await using var messageCmd = conn.Upsert("messages", [
 				("message_id", SqliteType.Integer),
@@ -89,6 +92,8 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 				("count", SqliteType.Integer)
 			]);
 			
+			await using var downloadCollector = new SqliteDownloadRepository.NewDownloadCollector(downloads, conn);
+
 			foreach (var message in messages) {
 				object messageId = message.Id;
 
@@ -119,8 +124,6 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 				}
 
 				if (!message.Attachments.IsEmpty) {
-					addedAttachments = true;
-
 					foreach (var attachment in message.Attachments) {
 						attachmentCmd.Set(":message_id", messageId);
 						attachmentCmd.Set(":attachment_id", attachment.Id);
@@ -132,6 +135,8 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 						attachmentCmd.Set(":width", attachment.Width);
 						attachmentCmd.Set(":height", attachment.Height);
 						await attachmentCmd.ExecuteNonQueryAsync();
+						
+						await downloadCollector.Add(DownloadLinkExtractor.FromAttachment(attachment));
 					}
 				}
 
@@ -140,6 +145,10 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 						embedCmd.Set(":message_id", messageId);
 						embedCmd.Set(":json", embed.Json);
 						await embedCmd.ExecuteNonQueryAsync();
+
+						if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) {
+							await downloadCollector.Add(download);
+						}
 					}
 				}
 
@@ -151,18 +160,19 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 						reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
 						reactionCmd.Set(":count", reaction.Count);
 						await reactionCmd.ExecuteNonQueryAsync();
+
+						if (reaction.EmojiId is {} emojiId) {
+							await downloadCollector.Add(DownloadLinkExtractor.FromEmoji(emojiId, reaction.EmojiFlags));
+						}
 					}
 				}
 			}
 
-			await tx.CommitAsync();
+			await conn.CommitTransactionAsync();
+			downloadCollector.OnCommitted();
 		}
 
 		UpdateTotalCount();
-
-		if (addedAttachments) {
-			attachments.UpdateTotalCount();
-		}
 	}
 
 	public override Task<long> Count(CancellationToken cancellationToken) {
@@ -171,14 +181,14 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 	
 	public async Task<long> Count(MessageFilter? filter, CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
-		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
+		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	private sealed class MesageToManyCommand<T> : IAsyncDisposable {
+	private sealed class MessageToManyCommand<T> : IAsyncDisposable {
 		private readonly SqliteCommand cmd;
 		private readonly Func<SqliteDataReader, T> readItem;
 
-		public MesageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) {
+		public MessageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) {
 			this.cmd = conn.Command(sql);
 			this.cmd.Add(":message_id", SqliteType.Integer);
 
@@ -204,7 +214,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 		}
 	}
 
-	public async IAsyncEnumerable<Message> Get(MessageFilter? filter) {
+	public async IAsyncEnumerable<Message> Get(MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		const string AttachmentSql =
@@ -214,7 +224,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			WHERE message_id = :message_id
 			""";
 
-		await using var attachmentCmd = new MesageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment {
+		await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment {
 			Id = reader.GetUint64(0),
 			Name = reader.GetString(1),
 			Type = reader.IsDBNull(2) ? null : reader.GetString(2),
@@ -232,7 +242,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			WHERE message_id = :message_id
 			""";
 
-		await using var embedCmd = new MesageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed {
+		await using var embedCmd = new MessageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed {
 			Json = reader.GetString(0)
 		});
 
@@ -243,7 +253,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			WHERE message_id = :message_id
 			""";
 
-		await using var reactionsCmd = new MesageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction {
+		await using var reactionsCmd = new MessageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction {
 			EmojiId = reader.IsDBNull(0) ? null : reader.GetUint64(0),
 			EmojiName = reader.IsDBNull(1) ? null : reader.GetString(1),
 			EmojiFlags = (EmojiFlags) reader.GetInt16(2),
@@ -256,13 +266,13 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			 FROM messages m
 			 LEFT JOIN edit_timestamps et ON m.message_id = et.message_id
 			 LEFT JOIN replied_to rt ON m.message_id = rt.message_id
-			 {filter.GenerateWhereClause("m")}
+			 {filter.GenerateConditions("m").BuildWhereClause()}
 			 """
 		);
 
-		await using var reader = await messageCmd.ExecuteReaderAsync();
+		await using var reader = await messageCmd.ExecuteReaderAsync(cancellationToken);
 
-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			ulong messageId = reader.GetUint64(0);
 
 			yield return new Message {
@@ -283,7 +293,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 	public async IAsyncEnumerable<ulong> GetIds(MessageFilter? filter) {
 		await using var conn = await pool.Take();
 		
-		await using var cmd = conn.Command("SELECT message_id FROM messages" + filter.GenerateWhereClause());
+		await using var cmd = conn.Command("SELECT message_id FROM messages" + filter.GenerateConditions().BuildWhereClause());
 		await using var reader = await cmd.ExecuteReaderAsync();
 
 		while (await reader.ReadAsync()) {
@@ -297,7 +307,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 				$"""
 				 -- noinspection SqlWithoutWhere
 				 DELETE FROM messages
-				 {filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching)}
+				 {filter.GenerateConditions(invert: mode == FilterRemovalMode.KeepMatching).BuildWhereClause()}
 				 """
 			);
 		}

app/Server/Database/Sqlite/Repositories/SqliteServerRepository.cs

@@ -1,24 +1,28 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Utils.Logging;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
 sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
+	private static readonly Log Log = Log.ForType<SqliteServerRepository>();
+	
 	private readonly SqliteConnectionPool pool;
 
-	public SqliteServerRepository(SqliteConnectionPool pool) {
+	public SqliteServerRepository(SqliteConnectionPool pool) : base(Log) {
 		this.pool = pool;
 	}
 
 	public async Task Add(IReadOnlyList<Data.Server> servers) {
-		await using var conn = await pool.Take();
+		await using (var conn = await pool.Take()) {
+			await conn.BeginTransactionAsync();
 			
-		await using (var tx = await conn.BeginTransactionAsync()) {
 			await using var cmd = conn.Upsert("servers", [
 				("id", SqliteType.Integer),
 				("name", SqliteType.Text),
@@ -32,7 +36,7 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
 				await cmd.ExecuteNonQueryAsync();
 			}
 
-			await tx.CommitAsync();
+			await conn.CommitTransactionAsync();
 		}
 
 		UpdateTotalCount();
@@ -43,13 +47,13 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM servers", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<Data.Server> Get() {
+	public async IAsyncEnumerable<Data.Server> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, name, type FROM servers");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (reader.Read()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new Data.Server {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),

app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs

@@ -1,23 +1,30 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
+using DHT.Utils.Logging;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Repositories;
 
 sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
-	private readonly SqliteConnectionPool pool;
+	private static readonly Log Log = Log.ForType<SqliteUserRepository>();
 	
-	public SqliteUserRepository(SqliteConnectionPool pool) {
+	private readonly SqliteConnectionPool pool;
+	private readonly SqliteDownloadRepository downloads;
+
+	public SqliteUserRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : base(Log) {
 		this.pool = pool;
+		this.downloads = downloads;
 	}
 
 	public async Task Add(IReadOnlyList<User> users) {
 		await using (var conn = await pool.Take()) {
-			await using var tx = await conn.BeginTransactionAsync();
+			await conn.BeginTransactionAsync();
 
 			await using var cmd = conn.Upsert("users", [
 				("id", SqliteType.Integer),
@@ -26,15 +33,22 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 				("discriminator", SqliteType.Text)
 			]);
 
+			await using var downloadCollector = new SqliteDownloadRepository.NewDownloadCollector(downloads, conn);
+			
 			foreach (var user in users) {
 				cmd.Set(":id", user.Id);
 				cmd.Set(":name", user.Name);
 				cmd.Set(":avatar_url", user.AvatarUrl);
 				cmd.Set(":discriminator", user.Discriminator);
 				await cmd.ExecuteNonQueryAsync();
+
+				if (user.AvatarUrl is {} avatarUrl) {
+					await downloadCollector.Add(DownloadLinkExtractor.FromUserAvatar(user.Id, avatarUrl));
+				}
 			}
 
-			await tx.CommitAsync();
+			await conn.CommitTransactionAsync();
+			downloadCollector.OnCommitted();
 		}
 
 		UpdateTotalCount();
@@ -45,13 +59,13 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM users", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<User> Get() {
+	public async IAsyncEnumerable<User> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, name, avatar_url, discriminator FROM users");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (reader.Read()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new User {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),

app/Server/Database/Sqlite/Schema.cs

@@ -1,358 +0,0 @@
-using System.Collections.Generic;
-using System.Data.Common;
-using System.Threading.Tasks;
-using DHT.Server.Database.Exceptions;
-using DHT.Server.Database.Sqlite.Utils;
-using DHT.Server.Download;
-using DHT.Utils.Logging;
-using Microsoft.Data.Sqlite;
-
-namespace DHT.Server.Database.Sqlite;
-
-sealed class Schema {
-	internal const int Version = 6;
-
-	private static readonly Log Log = Log.ForType<Schema>();
-
-	private readonly ISqliteConnection conn;
-
-	public Schema(ISqliteConnection conn) {
-		this.conn = conn;
-	}
-
-	public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
-		await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
-
-		var dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
-		if (dbVersionStr == null) {
-			await InitializeSchemas();
-		}
-		else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
-			throw new InvalidDatabaseVersionException(dbVersionStr);
-		}
-		else if (dbVersion > Version) {
-			throw new DatabaseTooNewException(dbVersion);
-		}
-		else if (dbVersion < Version) {
-			var proceed = await callbacks.CanUpgrade();
-			if (!proceed) {
-				return false;
-			}
-
-			await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter));
-		}
-
-		return true;
-	}
-
-	private async Task InitializeSchemas() {
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE users (
-		                        	id            INTEGER PRIMARY KEY NOT NULL,
-		                        	name          TEXT NOT NULL,
-		                        	avatar_url    TEXT,
-		                        	discriminator TEXT
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE servers (
-		                        	id   INTEGER PRIMARY KEY NOT NULL,
-		                        	name TEXT NOT NULL,
-		                        	type TEXT NOT NULL
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE channels (
-		                        	id        INTEGER PRIMARY KEY NOT NULL,
-		                        	server    INTEGER NOT NULL,
-		                        	name      TEXT NOT NULL,
-		                        	parent_id INTEGER,
-		                        	position  INTEGER,
-		                        	topic     TEXT,
-		                        	nsfw      INTEGER
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE messages (
-		                        	message_id INTEGER PRIMARY KEY NOT NULL,
-		                        	sender_id  INTEGER NOT NULL,
-		                        	channel_id INTEGER NOT NULL,
-		                        	text       TEXT NOT NULL,
-		                        	timestamp  INTEGER NOT NULL
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE attachments (
-		                        	message_id     INTEGER NOT NULL,
-		                        	attachment_id  INTEGER NOT NULL PRIMARY KEY NOT NULL,
-		                        	name           TEXT NOT NULL,
-		                        	type           TEXT,
-		                        	normalized_url TEXT NOT NULL,
-		                        	download_url   TEXT,
-		                        	size           INTEGER NOT NULL,
-		                        	width          INTEGER,
-		                        	height         INTEGER
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE embeds (
-		                        	message_id INTEGER NOT NULL,
-		                        	json       TEXT NOT NULL
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE downloads (
-		                        	normalized_url TEXT NOT NULL PRIMARY KEY,
-		                        	download_url   TEXT,
-		                        	status         INTEGER NOT NULL,
-		                        	size           INTEGER NOT NULL,
-		                        	blob           BLOB
-		                        )
-		                        """);
-
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE reactions (
-		                        	message_id  INTEGER NOT NULL,
-		                        	emoji_id    INTEGER,
-		                        	emoji_name  TEXT,
-		                        	emoji_flags INTEGER NOT NULL,
-		                        	count       INTEGER NOT NULL
-		                        )
-		                        """);
-
-		await CreateMessageEditTimestampTable();
-		await CreateMessageRepliedToTable();
-
-		await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
-		await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
-		await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
-
-		await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
-	}
-
-	private async Task CreateMessageEditTimestampTable() {
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE edit_timestamps (
-		                        	message_id     INTEGER PRIMARY KEY NOT NULL,
-		                        	edit_timestamp INTEGER NOT NULL
-		                        )
-		                        """);
-	}
-
-	private async Task CreateMessageRepliedToTable() {
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE replied_to (
-		                        	message_id    INTEGER PRIMARY KEY NOT NULL,
-		                        	replied_to_id INTEGER NOT NULL
-		                        )
-		                        """);
-	}
-
-	private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
-		await reporter.SubWork("Preparing attachments...", 0, 0);
-
-		var normalizedUrls = new Dictionary<long, string>();
-
-		await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
-			await using var reader = await selectCmd.ExecuteReaderAsync();
-
-			while (reader.Read()) {
-				var attachmentId = reader.GetInt64(0);
-				var originalUrl = reader.GetString(1);
-				normalizedUrls[attachmentId] = DiscordCdn.NormalizeUrl(originalUrl);
-			}
-		}
-
-		await using var tx = await conn.BeginTransactionAsync();
-
-		int totalUrls = normalizedUrls.Count;
-		int processedUrls = -1;
-
-		await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
-			updateCmd.Add(":attachment_id", SqliteType.Integer);
-			updateCmd.Add(":normalized_url", SqliteType.Text);
-
-			foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
-				if (++processedUrls % 1000 == 0) {
-					await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
-				}
-
-				updateCmd.Set(":attachment_id", attachmentId);
-				updateCmd.Set(":normalized_url", normalizedUrl);
-				updateCmd.ExecuteNonQuery();
-			}
-		}
-
-		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
-
-		await tx.CommitAsync();
-	}
-
-	private async Task NormalizeDownloadUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
-		await reporter.SubWork("Preparing downloads...", 0, 0);
-
-		var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
-		var duplicateUrlsToDelete = new HashSet<string>();
-
-		await using (var selectCmd = conn.Command("SELECT url FROM downloads ORDER BY CASE WHEN status = 200 THEN 0 ELSE 1 END")) {
-			await using var reader = await selectCmd.ExecuteReaderAsync();
-
-			while (reader.Read()) {
-				var originalUrl = reader.GetString(0);
-				var normalizedUrl = DiscordCdn.NormalizeUrl(originalUrl);
-
-				if (!normalizedUrlsToOriginalUrls.TryAdd(normalizedUrl, originalUrl)) {
-					duplicateUrlsToDelete.Add(originalUrl);
-				}
-			}
-		}
-
-		await conn.ExecuteAsync("PRAGMA cache_size = -20000");
-
-		DbTransaction tx;
-
-		await using (tx = await conn.BeginTransactionAsync()) {
-			await reporter.SubWork("Deleting duplicates...", 0, 0);
-
-			await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) {
-				foreach (var duplicateUrl in duplicateUrlsToDelete) {
-					deleteCmd.Set(":url", duplicateUrl);
-					deleteCmd.ExecuteNonQuery();
-				}
-			}
-
-			await tx.CommitAsync();
-		}
-
-		int totalUrls = normalizedUrlsToOriginalUrls.Count;
-		int processedUrls = -1;
-
-		tx = await conn.BeginTransactionAsync();
-
-		await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
-			updateCmd.Add(":normalized_url", SqliteType.Text);
-			updateCmd.Add(":download_url", SqliteType.Text);
-
-			foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
-				if (++processedUrls % 100 == 0) {
-					await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
-
-					// Not proper way of dealing with transactions, but it avoids a long commit at the end.
-					// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
-					await tx.CommitAsync();
-					await tx.DisposeAsync();
-
-					tx = await conn.BeginTransactionAsync();
-					updateCmd.Transaction = (SqliteTransaction) tx;
-				}
-
-				updateCmd.Set(":normalized_url", normalizedUrl);
-				updateCmd.Set(":download_url", downloadUrl);
-				updateCmd.ExecuteNonQuery();
-			}
-		}
-
-		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
-
-		await tx.CommitAsync();
-		await tx.DisposeAsync();
-
-		await conn.ExecuteAsync("PRAGMA cache_size = -2000");
-	}
-
-	private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
-		var perf = Log.Start("from version " + dbVersion);
-
-		await conn.ExecuteAsync("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
-
-		if (dbVersion <= 1) {
-			await reporter.MainWork("Applying schema changes...", 0, 1);
-			await conn.ExecuteAsync("ALTER TABLE channels ADD parent_id INTEGER");
-
-			perf.Step("Upgrade to version 2");
-			await reporter.NextVersion();
-		}
-
-		if (dbVersion <= 2) {
-			await reporter.MainWork("Applying schema changes...", 0, 1);
-
-			await CreateMessageEditTimestampTable();
-			await CreateMessageRepliedToTable();
-
-			await conn.ExecuteAsync("""
-			                        INSERT INTO edit_timestamps (message_id, edit_timestamp)
-			                        SELECT message_id, edit_timestamp
-			                        FROM messages
-			                        WHERE edit_timestamp IS NOT NULL
-			                        """);
-
-			await conn.ExecuteAsync("""
-			                        INSERT INTO replied_to (message_id, replied_to_id)
-			                        SELECT message_id, replied_to_id
-			                        FROM messages
-			                        WHERE replied_to_id IS NOT NULL
-			                        """);
-
-			await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN replied_to_id");
-			await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN edit_timestamp");
-
-			perf.Step("Upgrade to version 3");
-
-			await reporter.MainWork("Vacuuming the database...", 1, 1);
-			await conn.ExecuteAsync("VACUUM");
-			perf.Step("Vacuum");
-
-			await reporter.NextVersion();
-		}
-
-		if (dbVersion <= 3) {
-			await conn.ExecuteAsync("""
-			                        CREATE TABLE downloads (
-			                        	url    TEXT NOT NULL PRIMARY KEY,
-			                        	status INTEGER NOT NULL,
-			                        	size   INTEGER NOT NULL,
-			                        	blob   BLOB
-			                        )
-			                        """);
-
-			perf.Step("Upgrade to version 4");
-			await reporter.NextVersion();
-		}
-
-		if (dbVersion <= 4) {
-			await reporter.MainWork("Applying schema changes...", 0, 1);
-			await conn.ExecuteAsync("ALTER TABLE attachments ADD width INTEGER");
-			await conn.ExecuteAsync("ALTER TABLE attachments ADD height INTEGER");
-
-			perf.Step("Upgrade to version 5");
-			await reporter.NextVersion();
-		}
-
-		if (dbVersion <= 5) {
-			await reporter.MainWork("Applying schema changes...", 0, 3);
-			await conn.ExecuteAsync("ALTER TABLE attachments ADD download_url TEXT");
-			await conn.ExecuteAsync("ALTER TABLE downloads ADD download_url TEXT");
-
-			await reporter.MainWork("Updating attachments...", 1, 3);
-			await NormalizeAttachmentUrls(reporter);
-
-			await reporter.MainWork("Updating downloads...", 2, 3);
-			await NormalizeDownloadUrls(reporter);
-
-			await reporter.MainWork("Applying schema changes...", 3, 3);
-			await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
-			await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
-
-			perf.Step("Upgrade to version 6");
-			await reporter.NextVersion();
-		}
-
-		perf.End();
-	}
-}

app/Server/Database/Sqlite/Schema/ISchemaUpgrade.cs

@@ -0,0 +1,8 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+interface ISchemaUpgrade {
+	Task Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter);
+}

app/Server/Database/Sqlite/Schema/ISchemaUpgradeCallbacks.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Threading.Tasks;
 
-namespace DHT.Server.Database.Sqlite.Utils;
+namespace DHT.Server.Database.Sqlite.Schema;
 
 public interface ISchemaUpgradeCallbacks {
 	Task<bool> CanUpgrade();

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo2.cs

@@ -0,0 +1,11 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo2 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+		await conn.ExecuteAsync("ALTER TABLE channels ADD parent_id INTEGER");
+	}
+}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo3.cs

@@ -0,0 +1,33 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo3 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+
+		await SqliteSchema.CreateMessageEditTimestampTable(conn);
+		await SqliteSchema.CreateMessageRepliedToTable(conn);
+
+		await conn.ExecuteAsync("""
+		                        INSERT INTO edit_timestamps (message_id, edit_timestamp)
+		                        SELECT message_id, edit_timestamp
+		                        FROM messages
+		                        WHERE edit_timestamp IS NOT NULL
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        INSERT INTO replied_to (message_id, replied_to_id)
+		                        SELECT message_id, replied_to_id
+		                        FROM messages
+		                        WHERE replied_to_id IS NOT NULL
+		                        """);
+
+		await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN replied_to_id");
+		await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN edit_timestamp");
+
+		await reporter.MainWork("Vacuuming the database...", 1, 1);
+		await conn.ExecuteAsync("VACUUM");
+	}
+}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo4.cs

@@ -0,0 +1,19 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo4 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE downloads (
+		                        	url    TEXT NOT NULL PRIMARY KEY,
+		                        	status INTEGER NOT NULL,
+		                        	size   INTEGER NOT NULL,
+		                        	blob   BLOB
+		                        )
+		                        """);
+	}
+}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo5.cs

@@ -0,0 +1,12 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo5 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+		await conn.ExecuteAsync("ALTER TABLE attachments ADD width INTEGER");
+		await conn.ExecuteAsync("ALTER TABLE attachments ADD height INTEGER");
+	}
+}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo6.cs

@@ -0,0 +1,132 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
+using Microsoft.Data.Sqlite;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 3);
+		await conn.ExecuteAsync("ALTER TABLE attachments ADD download_url TEXT");
+		await conn.ExecuteAsync("ALTER TABLE downloads ADD download_url TEXT");
+
+		await reporter.MainWork("Updating attachments...", 1, 3);
+		await NormalizeAttachmentUrls(conn, reporter);
+
+		await reporter.MainWork("Updating downloads...", 2, 3);
+		await NormalizeDownloadUrls(conn, reporter);
+
+		await reporter.MainWork("Applying schema changes...", 3, 3);
+		await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
+		await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
+	}
+
+	private async Task NormalizeAttachmentUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.SubWork("Preparing attachments...", 0, 0);
+
+		var normalizedUrls = new Dictionary<long, string>();
+
+		await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
+			await using var reader = await selectCmd.ExecuteReaderAsync();
+
+			while (await reader.ReadAsync()) {
+				var attachmentId = reader.GetInt64(0);
+				var originalUrl = reader.GetString(1);
+				normalizedUrls[attachmentId] = DiscordCdn.NormalizeUrl(originalUrl);
+			}
+		}
+
+		await conn.BeginTransactionAsync();
+
+		int totalUrls = normalizedUrls.Count;
+		int processedUrls = -1;
+
+		await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
+			updateCmd.Add(":attachment_id", SqliteType.Integer);
+			updateCmd.Add(":normalized_url", SqliteType.Text);
+
+			foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
+				if (++processedUrls % 1000 == 0) {
+					await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
+				}
+
+				updateCmd.Set(":attachment_id", attachmentId);
+				updateCmd.Set(":normalized_url", normalizedUrl);
+				await updateCmd.ExecuteNonQueryAsync();
+			}
+		}
+
+		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
+
+		await conn.CommitTransactionAsync();
+	}
+
+	private async Task NormalizeDownloadUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.SubWork("Preparing downloads...", 0, 0);
+
+		var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
+		var duplicateUrlsToDelete = new HashSet<string>();
+
+		await using (var selectCmd = conn.Command("SELECT url FROM downloads ORDER BY CASE WHEN status = 200 THEN 0 ELSE 1 END")) {
+			await using var reader = await selectCmd.ExecuteReaderAsync();
+
+			while (await reader.ReadAsync()) {
+				var originalUrl = reader.GetString(0);
+				var normalizedUrl = DiscordCdn.NormalizeUrl(originalUrl);
+
+				if (!normalizedUrlsToOriginalUrls.TryAdd(normalizedUrl, originalUrl)) {
+					duplicateUrlsToDelete.Add(originalUrl);
+				}
+			}
+		}
+
+		await conn.ExecuteAsync("PRAGMA cache_size = -20000");
+		await conn.BeginTransactionAsync();
+		
+		await reporter.SubWork("Deleting duplicates...", 0, 0);
+
+		await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) {
+			foreach (var duplicateUrl in duplicateUrlsToDelete) {
+				deleteCmd.Set(":url", duplicateUrl);
+				await deleteCmd.ExecuteNonQueryAsync();
+			}
+		}
+
+		await conn.CommitTransactionAsync();
+
+		int totalUrls = normalizedUrlsToOriginalUrls.Count;
+		int processedUrls = -1;
+
+		await conn.BeginTransactionAsync();
+
+		await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
+			updateCmd.Add(":normalized_url", SqliteType.Text);
+			updateCmd.Add(":download_url", SqliteType.Text);
+
+			foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
+				if (++processedUrls % 100 == 0) {
+					await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
+
+					// Not proper way of dealing with transactions, but it avoids a long commit at the end.
+					// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
+					await conn.CommitTransactionAsync();
+
+					await conn.BeginTransactionAsync();
+					conn.AssignActiveTransaction(updateCmd);
+				}
+
+				updateCmd.Set(":normalized_url", normalizedUrl);
+				updateCmd.Set(":download_url", downloadUrl);
+				await updateCmd.ExecuteNonQueryAsync();
+			}
+		}
+
+		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
+
+		await conn.CommitTransactionAsync();
+
+		await conn.ExecuteAsync("PRAGMA cache_size = -2000");
+	}
+}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo7.cs

@@ -0,0 +1,153 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using DHT.Server.Data;
+using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
+using Microsoft.Data.Sqlite;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 6);
+		await SqliteSchema.CreateDownloadTables(conn);
+
+		await reporter.MainWork("Migrating download metadata...", 1, 6);
+		await conn.ExecuteAsync("INSERT INTO download_metadata (normalized_url, download_url, status, size) SELECT normalized_url, download_url, status, size FROM downloads");
+
+		await reporter.MainWork("Merging attachment metadata...", 2, 6);
+		await conn.ExecuteAsync("UPDATE download_metadata SET type = (SELECT type FROM attachments WHERE download_metadata.normalized_url = attachments.normalized_url)");
+
+		await reporter.MainWork("Migrating downloaded files...", 3, 6);
+		await MigrateDownloadBlobsToNewTable(conn, reporter);
+
+		await reporter.MainWork("Applying schema changes...", 4, 6);
+		await conn.ExecuteAsync("DROP TABLE downloads");
+
+		await reporter.MainWork("Discovering downloadable links...", 5, 6);
+		await DiscoverDownloadableLinks(conn, reporter);
+	}
+
+	private async Task MigrateDownloadBlobsToNewTable(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.SubWork("Listing downloaded files...", 0, 0);
+
+		var urlsToMigrate = await GetDownloadedFileUrls(conn);
+		int totalFiles = urlsToMigrate.Count;
+		int processedFiles = -1;
+
+		await reporter.SubWork("Processing downloaded files...", 0, totalFiles);
+
+		await conn.BeginTransactionAsync();
+
+		await using (var insertCmd = conn.Command("INSERT INTO download_blobs (normalized_url, blob) SELECT normalized_url, blob FROM downloads WHERE normalized_url = :normalized_url"))
+		await using (var deleteCmd = conn.Command("DELETE FROM downloads WHERE normalized_url = :normalized_url")) {
+			insertCmd.Add(":normalized_url", SqliteType.Text);
+			deleteCmd.Add(":normalized_url", SqliteType.Text);
+
+			foreach (var url in urlsToMigrate) {
+				if (++processedFiles % 10 == 0) {
+					await reporter.SubWork("Processing downloaded files...", processedFiles, totalFiles);
+
+					// Not proper way of dealing with transactions, but it avoids a long commit at the end.
+					// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
+					await conn.CommitTransactionAsync();
+					
+					await conn.BeginTransactionAsync();
+					conn.AssignActiveTransaction(insertCmd);
+					conn.AssignActiveTransaction(deleteCmd);
+				}
+
+				insertCmd.Set(":normalized_url", url);
+				await insertCmd.ExecuteNonQueryAsync();
+
+				deleteCmd.Set(":normalized_url", url);
+				await deleteCmd.ExecuteNonQueryAsync();
+			}
+		}
+
+		await reporter.SubWork("Processing downloaded files...", totalFiles, totalFiles);
+
+		await conn.CommitTransactionAsync();
+	}
+
+	private async Task<List<string>> GetDownloadedFileUrls(ISqliteConnection conn) {
+		var urls = new List<string>();
+
+		await using var selectCmd = conn.Command("SELECT normalized_url FROM downloads WHERE blob IS NOT NULL");
+		await using var reader = await selectCmd.ExecuteReaderAsync();
+
+		while (await reader.ReadAsync()) {
+			urls.Add(reader.GetString(0));
+		}
+
+		return urls;
+	}
+
+	private async Task DiscoverDownloadableLinks(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.SubWork("Processing attachments...", 0, 4);
+
+		await using (var cmd = conn.Command("""
+		                                    INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size)
+		                                    SELECT a.normalized_url, a.download_url, :pending, a.type, MAX(a.size)
+		                                    FROM attachments a
+		                                    GROUP BY a.normalized_url
+		                                    """)) {
+			cmd.AddAndSet(":pending", SqliteType.Integer, (int) DownloadStatus.Pending);
+			await cmd.ExecuteNonQueryAsync();
+		}
+
+		static async Task InsertDownload(SqliteCommand insertCmd, Data.Download? download) {
+			if (download == null) {
+				return;
+			}
+
+			insertCmd.Set(":normalized_url", download.NormalizedUrl);
+			insertCmd.Set(":download_url", download.DownloadUrl);
+			insertCmd.Set(":status", (int) download.Status);
+			insertCmd.Set(":type", download.Type);
+			insertCmd.Set(":size", download.Size);
+			await insertCmd.ExecuteNonQueryAsync();
+		}
+
+		await conn.BeginTransactionAsync();
+
+		await using var insertCmd = conn.Command("INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size) VALUES (:normalized_url, :download_url, :status, :type, :size)");
+		insertCmd.Add(":normalized_url", SqliteType.Text);
+		insertCmd.Add(":download_url", SqliteType.Text);
+		insertCmd.Add(":status", SqliteType.Integer);
+		insertCmd.Add(":type", SqliteType.Text);
+		insertCmd.Add(":size", SqliteType.Integer);
+
+		await reporter.SubWork("Processing embeds...", 1, 4);
+
+		await using (var embedCmd = conn.Command("SELECT json FROM embeds")) {
+			await using var reader = await embedCmd.ExecuteReaderAsync();
+
+			while (await reader.ReadAsync()) {
+				await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)));
+			}
+		}
+
+		await reporter.SubWork("Processing users...", 2, 4);
+
+		await using (var avatarCmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) {
+			await using var reader = await avatarCmd.ExecuteReaderAsync();
+
+			while (await reader.ReadAsync()) {
+				await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1)));
+			}
+		}
+
+		await reporter.SubWork("Processing reactions...", 3, 4);
+
+		await using (var avatarCmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM reactions WHERE emoji_id IS NOT NULL")) {
+			await using var reader = await avatarCmd.ExecuteReaderAsync();
+
+			while (await reader.ReadAsync()) {
+				await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1)));
+			}
+		}
+
+		await conn.CommitTransactionAsync();
+	}
+}

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -2,6 +2,7 @@ using System;
 using System.Threading.Tasks;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Repositories;
+using DHT.Server.Database.Sqlite.Schema;
 using DHT.Server.Database.Sqlite.Utils;
 using Microsoft.Data.Sqlite;
 
@@ -21,7 +22,7 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 
 		try {
 			await using var conn = await pool.Take();
-			wasOpened = await new Schema(conn).Setup(schemaUpgradeCallbacks);
+			wasOpened = await new SqliteSchema(conn).Setup(schemaUpgradeCallbacks);
 		} catch (Exception) {
 			await pool.DisposeAsync();
 			throw;
@@ -42,7 +43,6 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 	public IServerRepository Servers => servers;
 	public IChannelRepository Channels => channels;
 	public IMessageRepository Messages => messages;
-	public IAttachmentRepository Attachments => attachments;
 	public IDownloadRepository Downloads => downloads;
 	
 	private readonly SqliteConnectionPool pool;
@@ -51,18 +51,17 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 	private readonly SqliteServerRepository servers;
 	private readonly SqliteChannelRepository channels;
 	private readonly SqliteMessageRepository messages;
-	private readonly SqliteAttachmentRepository attachments;
 	private readonly SqliteDownloadRepository downloads;
 	
 	private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
 		this.Path = path;
 		this.pool = pool;
 
-		users = new SqliteUserRepository(pool);
+		downloads = new SqliteDownloadRepository(pool);
+		users = new SqliteUserRepository(pool, downloads);
 		servers = new SqliteServerRepository(pool);
 		channels = new SqliteChannelRepository(pool);
-		messages = new SqliteMessageRepository(pool, attachments = new SqliteAttachmentRepository(pool));
+		messages = new SqliteMessageRepository(pool, downloads);
-		downloads = new SqliteDownloadRepository(pool);
 	}
 
 	public async ValueTask DisposeAsync() {
@@ -70,7 +69,6 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 		servers.Dispose();
 		channels.Dispose();
 		messages.Dispose();
-		attachments.Dispose();
 		downloads.Dispose();
 		await pool.DisposeAsync();
 	}

app/Server/Database/Sqlite/SqliteFilters.cs

@@ -8,77 +8,53 @@ using DHT.Server.Database.Sqlite.Utils;
 namespace DHT.Server.Database.Sqlite;
 
 static class SqliteFilters {
-	private static string WhereAll(bool invert) {
+	public static SqliteConditionBuilder GenerateConditions(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
-		return invert ? "WHERE FALSE" : "";
+		var builder = new SqliteConditionBuilder(tableAlias, invert);
-	}
-
-	public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
-		if (filter == null || filter.IsEmpty) {
-			return WhereAll(invert);
-		}
-
-		var where = new SqliteWhereGenerator(tableAlias, invert);
 
+		if (filter != null) {
 			if (filter.StartDate != null) {
-			where.AddCondition("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
+				builder.AddCondition("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
 			}
 
 			if (filter.EndDate != null) {
-			where.AddCondition("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
+				builder.AddCondition("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
 			}
 
 			if (filter.ChannelIds != null) {
-			where.AddCondition("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
+				builder.AddCondition("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
 			}
 
 			if (filter.UserIds != null) {
-			where.AddCondition("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
+				builder.AddCondition("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
 			}
 
 			if (filter.MessageIds != null) {
-			where.AddCondition("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
+				builder.AddCondition("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
+			}
 		}
 
-		return where.Generate();
+		return builder;
 	}
 
-	public static string GenerateWhereClause(this AttachmentFilter? filter, string? tableAlias = null, bool invert = false) {
+	public static SqliteConditionBuilder GenerateConditions(this DownloadItemFilter? filter, string? tableAlias = null, bool invert = false) {
-		if (filter == null || filter.IsEmpty) {
+		var builder = new SqliteConditionBuilder(tableAlias, invert);
-			return WhereAll(invert);
-		}
-
-		var where = new SqliteWhereGenerator(tableAlias, invert);
-
-		if (filter.MaxBytes != null) {
-			where.AddCondition("size <= " + filter.MaxBytes);
-		}
-
-		if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyNotPresent) {
-			where.AddCondition("normalized_url NOT IN (SELECT normalized_url FROM downloads)");
-		}
-		else if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyPresent) {
-			where.AddCondition("normalized_url IN (SELECT normalized_url FROM downloads)");
-		}
-
-		return where.Generate();
-	}
-
-	public static string GenerateWhereClause(this DownloadItemFilter? filter, string? tableAlias = null, bool invert = false) {
-		if (filter == null || filter.IsEmpty) {
-			return WhereAll(invert);
-		}
-
-		var where = new SqliteWhereGenerator(tableAlias, invert);
 
+		if (filter != null) {
 			if (filter.IncludeStatuses != null) {
-			where.AddCondition("status IN (" + filter.IncludeStatuses.In() + ")");
+				builder.AddCondition("status IN (" + filter.IncludeStatuses.In() + ")");
 			}
 
 			if (filter.ExcludeStatuses != null) {
-			where.AddCondition("status NOT IN (" + filter.ExcludeStatuses.In() + ")");
+				builder.AddCondition("status NOT IN (" + filter.ExcludeStatuses.In() + ")");
 			}
 
-		return where.Generate();
+			if (filter.MaxBytes != null) {
+				builder.AddCondition("size IS NOT NULL");
+				builder.AddCondition("size <= " + filter.MaxBytes);
+			}
+		}
+
+		return builder;
 	}
 
 	private static string In(this ISet<DownloadStatus> statuses) {

app/Server/Database/Sqlite/SqliteSchema.cs

@@ -0,0 +1,193 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using DHT.Server.Database.Exceptions;
+using DHT.Server.Database.Sqlite.Schema;
+using DHT.Server.Database.Sqlite.Utils;
+using DHT.Utils.Logging;
+
+namespace DHT.Server.Database.Sqlite;
+
+sealed class SqliteSchema {
+	internal const int Version = 7;
+
+	private static readonly Log Log = Log.ForType<SqliteSchema>();
+
+	private readonly ISqliteConnection conn;
+
+	public SqliteSchema(ISqliteConnection conn) {
+		this.conn = conn;
+	}
+
+	public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
+		await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
+
+		var dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
+		if (dbVersionStr == null) {
+			await InitializeSchemas();
+		}
+		else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
+			throw new InvalidDatabaseVersionException(dbVersionStr);
+		}
+		else if (dbVersion > Version) {
+			throw new DatabaseTooNewException(dbVersion);
+		}
+		else if (dbVersion < Version) {
+			var proceed = await callbacks.CanUpgrade();
+			if (!proceed) {
+				return false;
+			}
+
+			await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter));
+		}
+
+		return true;
+	}
+
+	private async Task InitializeSchemas() {
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE users (
+		                        	id            INTEGER PRIMARY KEY NOT NULL,
+		                        	name          TEXT NOT NULL,
+		                        	avatar_url    TEXT,
+		                        	discriminator TEXT
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE servers (
+		                        	id   INTEGER PRIMARY KEY NOT NULL,
+		                        	name TEXT NOT NULL,
+		                        	type TEXT NOT NULL
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE channels (
+		                        	id        INTEGER PRIMARY KEY NOT NULL,
+		                        	server    INTEGER NOT NULL,
+		                        	name      TEXT NOT NULL,
+		                        	parent_id INTEGER,
+		                        	position  INTEGER,
+		                        	topic     TEXT,
+		                        	nsfw      INTEGER
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE messages (
+		                        	message_id INTEGER PRIMARY KEY NOT NULL,
+		                        	sender_id  INTEGER NOT NULL,
+		                        	channel_id INTEGER NOT NULL,
+		                        	text       TEXT NOT NULL,
+		                        	timestamp  INTEGER NOT NULL
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE attachments (
+		                        	message_id     INTEGER NOT NULL,
+		                        	attachment_id  INTEGER NOT NULL PRIMARY KEY NOT NULL,
+		                        	name           TEXT NOT NULL,
+		                        	type           TEXT,
+		                        	normalized_url TEXT NOT NULL,
+		                        	download_url   TEXT,
+		                        	size           INTEGER NOT NULL,
+		                        	width          INTEGER,
+		                        	height         INTEGER
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE embeds (
+		                        	message_id INTEGER NOT NULL,
+		                        	json       TEXT NOT NULL
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE reactions (
+		                        	message_id  INTEGER NOT NULL,
+		                        	emoji_id    INTEGER,
+		                        	emoji_name  TEXT,
+		                        	emoji_flags INTEGER NOT NULL,
+		                        	count       INTEGER NOT NULL
+		                        )
+		                        """);
+
+		await CreateMessageEditTimestampTable(conn);
+		await CreateMessageRepliedToTable(conn);
+		await CreateDownloadTables(conn);
+
+		await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
+		await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
+		await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
+
+		await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
+	}
+
+	internal static async Task CreateMessageEditTimestampTable(ISqliteConnection conn) {
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE edit_timestamps (
+		                        	message_id     INTEGER PRIMARY KEY NOT NULL,
+		                        	edit_timestamp INTEGER NOT NULL
+		                        )
+		                        """);
+	}
+
+	internal static async Task CreateMessageRepliedToTable(ISqliteConnection conn) {
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE replied_to (
+		                        	message_id    INTEGER PRIMARY KEY NOT NULL,
+		                        	replied_to_id INTEGER NOT NULL
+		                        )
+		                        """);
+	}
+
+	internal static async Task CreateDownloadTables(ISqliteConnection conn) {
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE download_metadata (
+		                        	normalized_url TEXT NOT NULL PRIMARY KEY,
+		                        	download_url   TEXT NOT NULL,
+		                        	status         INTEGER NOT NULL,
+		                        	type           TEXT,
+		                        	size           INTEGER
+		                        )
+		                        """);
+
+		await conn.ExecuteAsync("""
+		                        CREATE TABLE download_blobs (
+		                        	normalized_url TEXT NOT NULL PRIMARY KEY,
+		                        	blob           BLOB NOT NULL,
+		                        	FOREIGN KEY (normalized_url) REFERENCES download_metadata (normalized_url) ON UPDATE CASCADE ON DELETE CASCADE
+		                        )
+		                        """);
+	}
+
+	private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		var upgrades = new Dictionary<int, ISchemaUpgrade> {
+			{ 1, new SqliteSchemaUpgradeTo2() },
+			{ 2, new SqliteSchemaUpgradeTo3() },
+			{ 3, new SqliteSchemaUpgradeTo4() },
+			{ 4, new SqliteSchemaUpgradeTo5() },
+			{ 5, new SqliteSchemaUpgradeTo6() },
+			{ 6, new SqliteSchemaUpgradeTo7() },
+		};
+
+		var perf = Log.Start("from version " + dbVersion);
+
+		for (int fromVersion = dbVersion; fromVersion < Version; fromVersion++) {
+			var toVersion = fromVersion + 1;
+			
+			if (upgrades.TryGetValue(fromVersion, out var upgrade)) {
+				await upgrade.Run(conn, reporter);
+			}
+
+			await conn.ExecuteAsync("UPDATE metadata SET value = " + toVersion + " WHERE key = 'version'");
+
+			perf.Step("Upgrade to version " + toVersion);
+			await reporter.NextVersion();
+		}
+
+		perf.End();
+	}
+}

app/Server/Database/Sqlite/Utils/ISqliteConnection.cs

@@ -1,8 +1,15 @@
 using System;
+using System.Threading.Tasks;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite.Utils;
 
 interface ISqliteConnection : IAsyncDisposable {
 	SqliteConnection InnerConnection { get; }
+	
+	Task BeginTransactionAsync();
+	Task CommitTransactionAsync();
+	Task RollbackTransactionAsync();
+	
+	void AssignActiveTransaction(SqliteCommand command);
 }

app/Server/Database/Sqlite/Utils/SqliteConditionBuilder.cs

@@ -2,12 +2,12 @@ using System.Collections.Generic;
 
 namespace DHT.Server.Database.Sqlite.Utils;
 
-sealed class SqliteWhereGenerator {
+sealed class SqliteConditionBuilder {
 	private readonly string? tableAlias;
 	private readonly bool invert;
 	private readonly List<string> conditions = [];
 
-	public SqliteWhereGenerator(string? tableAlias, bool invert) {
+	public SqliteConditionBuilder(string? tableAlias, bool invert) {
 		this.tableAlias = tableAlias;
 		this.invert = invert;
 	}
@@ -16,16 +16,20 @@ sealed class SqliteWhereGenerator {
 		conditions.Add(tableAlias == null ? condition : tableAlias + '.' + condition);
 	}
 
-	public string Generate() {
+	public string Build() {
 		if (conditions.Count == 0) {
-			return "";
+			return invert ? "FALSE" : "TRUE";
 		}
 
 		if (invert) {
-			return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
+			return "NOT (" + string.Join(" AND ", conditions) + ")";
 		}
 		else {
-			return " WHERE " + string.Join(" AND ", conditions);
+			return string.Join(" AND ", conditions);
 		}
 	}
+	
+	public string BuildWhereClause() {
+		return " WHERE " + Build();
+	}
 }

app/Server/Database/Sqlite/Utils/SqliteConnectionPool.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Data.Common;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Utils.Collections;
@@ -42,9 +43,8 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
 
 			var pooledConnection = new PooledConnection(this, conn);
 
-			await using (var cmd = pooledConnection.Command("PRAGMA journal_mode=WAL")) {
+			await pooledConnection.ExecuteAsync("PRAGMA journal_mode=WAL", disposalToken);
-				await cmd.ExecuteNonQueryAsync(disposalToken);
+			await pooledConnection.ExecuteAsync("PRAGMA foreign_keys=ON", disposalToken);
-			}
 
 			all.Add(pooledConnection);
 			await free.Push(pooledConnection, disposalToken);
@@ -74,17 +74,48 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
 		disposalTokenSource.Dispose();
 	}
 
-	private sealed class PooledConnection : ISqliteConnection {
+	private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) : ISqliteConnection {
-		public SqliteConnection InnerConnection { get; }
+		public SqliteConnection InnerConnection { get; } = conn;
 
-		private readonly SqliteConnectionPool pool;
+		private DbTransaction? activeTransaction;
 
-		public PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) {
+		public async Task BeginTransactionAsync() {
-			this.pool = pool;
+			if (activeTransaction != null) {
-			this.InnerConnection = conn;
+				throw new InvalidOperationException("A transaction is already active.");
+			}
+			
+			activeTransaction = await InnerConnection.BeginTransactionAsync();
+		}
+
+		public async Task CommitTransactionAsync() {
+			if (activeTransaction == null) {
+				throw new InvalidOperationException("No active transaction to commit.");
+			}
+
+			await activeTransaction.CommitAsync();
+			await activeTransaction.DisposeAsync();
+			activeTransaction = null;
+		}
+
+		public async Task RollbackTransactionAsync() {
+			if (activeTransaction == null) {
+				throw new InvalidOperationException("No active transaction to rollback.");
+			}
+			
+			await activeTransaction.RollbackAsync();
+			await activeTransaction.DisposeAsync();
+			activeTransaction = null;
+		}
+
+		public void AssignActiveTransaction(SqliteCommand command) {
+			command.Transaction = (SqliteTransaction?) activeTransaction;
 		}
 
 		public async ValueTask DisposeAsync() {
+			if (activeTransaction != null) {
+				await RollbackTransactionAsync();
+			}
+			
 			await pool.Return(this);
 		}
 	}

app/Server/Database/Sqlite/Utils/SqliteExtensions.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Data.Common;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -9,11 +8,7 @@ using Microsoft.Data.Sqlite;
 namespace DHT.Server.Database.Sqlite.Utils;
 
 static class SqliteExtensions {
-	public static ValueTask<DbTransaction> BeginTransactionAsync(this ISqliteConnection conn) {
+	public static SqliteCommand Command(this ISqliteConnection conn, [LanguageInjection("sql")] string sql) {
-		return conn.InnerConnection.BeginTransactionAsync();
-	}
-
-	public static SqliteCommand Command(this ISqliteConnection conn, string sql) {
 		var cmd = conn.InnerConnection.CreateCommand();
 		cmd.CommandText = sql;
 		return cmd;
@@ -28,7 +23,11 @@ static class SqliteExtensions {
 		await using var cmd = conn.Command(sql);
 		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		return reader.Read() ? readFunction(reader) : readFunction(null);
+		return await reader.ReadAsync(cancellationToken) ? readFunction(reader) : readFunction(null);
+	}
+	
+	public static async Task<long> ExecuteLongScalarAsync(this SqliteCommand command) {
+		return (long) (await command.ExecuteScalarAsync())!;
 	}
 
 	public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {

app/Server/Download/DiscordCdn.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Frozen;
+using System.Diagnostics.CodeAnalysis;
+using System.Web;
 
 namespace DHT.Server.Download;
 
@@ -7,9 +9,35 @@ static class DiscordCdn {
 	private static FrozenSet<string> CdnHosts { get; } = new[] {
 		"cdn.discordapp.com",
 		"cdn.discord.com",
+		"media.discordapp.net"
 	}.ToFrozenSet();
 
+	private static bool IsCdnUrl(string originalUrl, [NotNullWhen(true)] out Uri? uri) {
+		return Uri.TryCreate(originalUrl, UriKind.Absolute, out uri) && CdnHosts.Contains(uri.Host);
+	}
+
 	public static string NormalizeUrl(string originalUrl) {
-		return Uri.TryCreate(originalUrl, UriKind.Absolute, out var uri) && CdnHosts.Contains(uri.Host) ? uri.GetLeftPart(UriPartial.Path) : originalUrl;
+		return IsCdnUrl(originalUrl, out var uri) ? DoNormalize(uri) : originalUrl;
+	}
+
+	public static bool NormalizeUrlAndReturnIfCdn(string originalUrl, out string normalizedUrl) {
+		if (IsCdnUrl(originalUrl, out var uri)) {
+			normalizedUrl = DoNormalize(uri);
+			return true;
+		}
+		else {
+			normalizedUrl = originalUrl;
+			return false;
+		}
+	}
+
+	private static string DoNormalize(Uri uri) {
+		var query = HttpUtility.ParseQueryString(uri.Query);
+		
+		query.Remove("ex");
+		query.Remove("is");
+		query.Remove("hm");
+
+		return new UriBuilder(uri) { Query = query.ToString() }.Uri.ToString();
 	}
 }

app/Server/Download/DownloadItem.cs

@@ -1,7 +1,21 @@
+using System;
+using System.Net;
+using DHT.Server.Data;
+
 namespace DHT.Server.Download;
 
 public readonly struct DownloadItem {
 	public string NormalizedUrl { get; init; }
 	public string DownloadUrl { get; init; }
-	public ulong Size { get; init; }
+	public string? Type { get; init; }
+	public ulong? Size { get; init; }
+	
+	internal Data.Download ToSuccess(long size) {
+		return new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, (ulong) Math.Max(size, 0));
+	}
+	
+	internal Data.Download ToFailure(HttpStatusCode? statusCode = null) {
+		var status = statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError;
+		return new Data.Download(NormalizedUrl, DownloadUrl, status, Type, Size);
+	}
 }

app/Server/Download/DownloadLinkExtractor.cs

@@ -0,0 +1,122 @@
+using System;
+using System.IO;
+using System.Net.Mime;
+using System.Text.Json;
+using System.Threading.Tasks;
+using DHT.Server.Data;
+using DHT.Server.Data.Embeds;
+using DHT.Utils.Logging;
+
+namespace DHT.Server.Download;
+
+static class DownloadLinkExtractor {
+	private static readonly Log Log = Log.ForType(typeof(DownloadLinkExtractor));
+
+	public static Data.Download FromUserAvatar(ulong userId, string avatarPath) {
+		string url = $"https://cdn.discordapp.com/avatars/{userId}/{avatarPath}.webp";
+		return new Data.Download(url, url, DownloadStatus.Pending, MediaTypeNames.Image.Webp, size: null);
+	}
+
+	public static Data.Download FromEmoji(ulong emojiId, EmojiFlags flags) {
+		var isAnimated = flags.HasFlag(EmojiFlags.Animated);
+		
+		string ext = isAnimated ? "gif" : "webp";
+		string type = isAnimated ? MediaTypeNames.Image.Gif : MediaTypeNames.Image.Webp;
+		
+		string url = $"https://cdn.discordapp.com/emojis/{emojiId}.{ext}";
+		return new Data.Download(url, url, DownloadStatus.Pending, type, size: null);
+	}
+
+	public static Data.Download FromAttachment(Attachment attachment) {
+		return new Data.Download(attachment.NormalizedUrl, attachment.DownloadUrl, DownloadStatus.Pending, attachment.Type, attachment.Size);
+	}
+
+	public static async Task<Data.Download?> TryFromEmbedJson(Stream jsonStream) {
+		try {
+			return FromEmbed(await JsonSerializer.DeserializeAsync(jsonStream, DiscordEmbedJsonContext.Default.DiscordEmbedJson));
+		} catch (Exception e) {
+			Log.Error("Could not parse embed json: " + e);
+			return null;
+		}
+	}
+
+	public static Data.Download? TryFromEmbedJson(string json) {
+		try {
+			return FromEmbed(JsonSerializer.Deserialize(json, DiscordEmbedJsonContext.Default.DiscordEmbedJson));
+		} catch (Exception e) {
+			Log.Error("Could not parse embed json: " + e);
+			return null;
+		}
+	}
+	
+	private static Data.Download? FromEmbed(DiscordEmbedJson? embed) {
+		if (embed is { Type: "image", Image.Url: {} imageUrl }) {
+			return FromEmbedImage(imageUrl);
+		}
+		else if (embed is { Type: "video", Video.Url: {} videoUrl }) {
+			return FromEmbedVideo(videoUrl);
+		}
+		else {
+			return null;
+		}
+	}
+
+	private static Data.Download? FromEmbedImage(string url) {
+		if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out var normalizedUrl)) {
+			return new Data.Download(normalizedUrl, url, DownloadStatus.Pending, GuessImageType(normalizedUrl), size: null);
+		}
+		else {
+			Log.Debug("Skipping non-CDN image url: " + url);
+			return null;
+		}
+	}
+
+	private static Data.Download? FromEmbedVideo(string url) {
+		if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out var normalizedUrl)) {
+			return new Data.Download(normalizedUrl, url, DownloadStatus.Pending, GuessVideoType(normalizedUrl), size: null);
+		}
+		else {
+			Log.Debug("Skipping non-CDN video url: " + url);
+			return null;
+		}
+	}
+
+	private static string? GuessImageType(string url) {
+		if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) {
+			return null;
+		}
+
+		ReadOnlySpan<char> extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
+		
+		// Remove Twitter quality suffix.
+		int colonIndex = extension.IndexOf(':');
+		if (colonIndex != -1) {
+			extension = extension[..colonIndex];
+		}
+
+		return extension switch {
+			".jpg"  => MediaTypeNames.Image.Jpeg,
+			".jpeg" => MediaTypeNames.Image.Jpeg,
+			".png"  => MediaTypeNames.Image.Png,
+			".gif"  => MediaTypeNames.Image.Gif,
+			".webp" => MediaTypeNames.Image.Webp,
+			".bmp"  => MediaTypeNames.Image.Bmp,
+			_       => null
+		};
+	}
+
+	private static string? GuessVideoType(string url) {
+		if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) {
+			return null;
+		}
+
+		string extension = Path.GetExtension(uri.AbsolutePath).ToLowerInvariant();
+		return extension switch {
+			".mp4"  => "video/mp4",
+			".mpeg" => "video/mpeg",
+			".webm" => "video/webm",
+			".mov"  => "video/quicktime",
+			_       => null
+		};
+	}
+}

app/Server/Download/Downloader.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using DHT.Server.Data.Filters;
 using DHT.Server.Database;
 
 namespace DHT.Server.Download;
@@ -10,16 +11,18 @@ public sealed class Downloader {
 	public bool IsDownloading => current != null;
 	
 	private readonly IDatabaseFile db;
+	private readonly int? concurrentDownloads;
 	private readonly SemaphoreSlim semaphore = new (1, 1);
 	
-	internal Downloader(IDatabaseFile db) {
+	internal Downloader(IDatabaseFile db, int? concurrentDownloads) {
 		this.db = db;
+		this.concurrentDownloads = concurrentDownloads;
 	}
 
-	public async Task<IObservable<DownloadItem>> Start() {
+	public async Task<IObservable<DownloadItem>> Start(DownloadItemFilter filter) {
 		await semaphore.WaitAsync();
 		try {
-			current ??= new DownloaderTask(db);
+			current ??= new DownloaderTask(db, filter, concurrentDownloads);
 			return current.FinishedItems;
 		} finally {
 			semaphore.Release();

app/Server/Download/DownloaderTask.cs

@@ -5,6 +5,7 @@ using System.Reactive.Subjects;
 using System.Threading;
 using System.Threading.Channels;
 using System.Threading.Tasks;
+using DHT.Server.Data.Filters;
 using DHT.Server.Database;
 using DHT.Utils.Logging;
 using DHT.Utils.Tasks;
@@ -14,10 +15,14 @@ namespace DHT.Server.Download;
 sealed class DownloaderTask : IAsyncDisposable {
 	private static readonly Log Log = Log.ForType<DownloaderTask>();
 
-	private const int DownloadTasks = 4;
+	private const int DefaultConcurrentDownloads = 4;
 	private const int QueueSize = 25;
 	private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
 
+	private static int GetDownloadTaskCount(int? concurrentDownloads) {
+		return Math.Max(1, concurrentDownloads ?? DefaultConcurrentDownloads);
+	}
+
 	private readonly Channel<DownloadItem> downloadQueue = Channel.CreateBounded<DownloadItem>(new BoundedChannelOptions(QueueSize) {
 		SingleReader = false,
 		SingleWriter = true,
@@ -29,6 +34,7 @@ sealed class DownloaderTask : IAsyncDisposable {
 	private readonly CancellationToken cancellationToken;
 
 	private readonly IDatabaseFile db;
+	private readonly DownloadItemFilter filter;
 	private readonly ISubject<DownloadItem> finishedItemPublisher = Subject.Synchronize(new Subject<DownloadItem>());
 
 	private readonly Task queueWriterTask;
@@ -36,16 +42,17 @@ sealed class DownloaderTask : IAsyncDisposable {
 
 	public IObservable<DownloadItem> FinishedItems => finishedItemPublisher;
 
-	internal DownloaderTask(IDatabaseFile db) {
+	internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter, int? concurrentDownloads) {
 		this.db = db;
+		this.filter = filter;
 		this.cancellationToken = cancellationTokenSource.Token;
 		this.queueWriterTask = Task.Run(RunQueueWriterTask);
-		this.downloadTasks = Enumerable.Range(1, DownloadTasks).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray();
+		this.downloadTasks = Enumerable.Range(1, GetDownloadTaskCount(concurrentDownloads)).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray();
 	}
 
 	private async Task RunQueueWriterTask() {
 		while (await downloadQueue.Writer.WaitToWriteAsync(cancellationToken)) {
-			var newItems = await db.Downloads.PullEnqueuedDownloadItems(QueueSize, cancellationToken).ToListAsync(cancellationToken);
+			var newItems = await db.Downloads.PullPendingDownloadItems(QueueSize, filter, cancellationToken).ToListAsync(cancellationToken);
 			if (newItems.Count == 0) {
 				await Task.Delay(TimeSpan.FromMilliseconds(50), cancellationToken);
 				continue;
@@ -60,24 +67,39 @@ sealed class DownloaderTask : IAsyncDisposable {
 	private async Task RunDownloadTask(int taskIndex) {
 		var log = Log.ForType<DownloaderTask>("Task " + taskIndex);
 
-		var client = new HttpClient();
+		var client = new HttpClient(new SocketsHttpHandler {
+			ConnectTimeout = TimeSpan.FromSeconds(30)
+		});
+		
+		client.Timeout = Timeout.InfiniteTimeSpan;
 		client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
-		client.Timeout = TimeSpan.FromSeconds(30);
 
 		while (!cancellationToken.IsCancellationRequested) {
 			var item = await downloadQueue.Reader.ReadAsync(cancellationToken);
 			log.Debug("Downloading " + item.DownloadUrl + "...");
 
 			try {
-				var downloadedBytes = await client.GetByteArrayAsync(item.DownloadUrl, cancellationToken);
+				var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, item.DownloadUrl), HttpCompletionOption.ResponseHeadersRead, cancellationToken);
-				await db.Downloads.AddDownload(Data.Download.NewSuccess(item, downloadedBytes));
+				response.EnsureSuccessStatusCode();
-			} catch (OperationCanceledException) {
+
+				if (response.Content.Headers.ContentLength is {} contentLength) {
+					await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+					await db.Downloads.AddDownload(item.ToSuccess(contentLength), stream);
+				}
+				else {
+					await db.Downloads.AddDownload(item.ToFailure(), stream: null);
+					log.Error("Download response has no content length: " + item.DownloadUrl);
+				}
+			} catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) {
 				// Ignore.
+			} catch (TaskCanceledException e) when (e.InnerException is TimeoutException) {
+				await db.Downloads.AddDownload(item.ToFailure(), stream: null);
+				log.Error("Download timed out: " + item.DownloadUrl);
 			} catch (HttpRequestException e) {
-				await db.Downloads.AddDownload(Data.Download.NewFailure(item, e.StatusCode, item.Size));
+				await db.Downloads.AddDownload(item.ToFailure(e.StatusCode), stream: null);
 				log.Error(e);
 			} catch (Exception e) {
-				await db.Downloads.AddDownload(Data.Download.NewFailure(item, null, item.Size));
+				await db.Downloads.AddDownload(item.ToFailure(), stream: null);
 				log.Error(e);
 			} finally {
 				try {

app/Server/Endpoints/BaseEndpoint.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Database;
 using DHT.Utils.Http;
@@ -9,39 +10,50 @@ using Microsoft.AspNetCore.Http;
 
 namespace DHT.Server.Endpoints;
 
-abstract class BaseEndpoint {
+abstract class BaseEndpoint(IDatabaseFile db) {
 	private static readonly Log Log = Log.ForType<BaseEndpoint>();
 
-	protected IDatabaseFile Db { get; }
+	protected IDatabaseFile Db { get; } = db;
-
-	protected BaseEndpoint(IDatabaseFile db) {
-		this.Db = db;
-	}
 
 	public async Task Handle(HttpContext ctx) {
 		var response = ctx.Response;
 
 		try {
 			response.StatusCode = (int) HttpStatusCode.OK;
-			var output = await Respond(ctx);
+			await Respond(ctx.Request, response, ctx.RequestAborted);
-			await output.WriteTo(response);
+		} catch (OperationCanceledException) {
+			throw;
 		} catch (HttpException e) {
 			Log.Error(e);
 			response.StatusCode = (int) e.StatusCode;
+			if (response.HasStarted) {
+				Log.Warn("Response has already started, cannot write status message: " + e.Message);
+			}
+			else {
 				await response.WriteAsync(e.Message);
+			}
 		} catch (Exception e) {
 			Log.Error(e);
 			response.StatusCode = (int) HttpStatusCode.InternalServerError;
 		}
 	}
 
-	protected abstract Task<IHttpOutput> Respond(HttpContext ctx);
+	protected abstract Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken);
 
-	protected static async Task<JsonElement> ReadJson(HttpContext ctx) {
+	protected static async Task<JsonElement> ReadJson(HttpRequest request) {
 		try {
-			return await ctx.Request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement);
+			return await request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement);
 		} catch (JsonException) {
 			throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON.");
 		}
 	}
+	
+	protected static Guid GetSessionId(HttpRequest request) {
+		if (request.Query.TryGetValue("session", out var sessionIdValue) && sessionIdValue.Count == 1 && Guid.TryParse(sessionIdValue[0], out Guid sessionId)) {
+			return sessionId;
+		}
+		else {
+			throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID.");
+		}
+	}
 }

app/Server/Endpoints/GetAttachmentEndpoint.cs

@@ -1,24 +0,0 @@
-using System.Net;
-using System.Threading.Tasks;
-using DHT.Server.Data;
-using DHT.Server.Database;
-using DHT.Utils.Http;
-using Microsoft.AspNetCore.Http;
-
-namespace DHT.Server.Endpoints;
-
-sealed class GetAttachmentEndpoint : BaseEndpoint {
-	public GetAttachmentEndpoint(IDatabaseFile db) : base(db) {}
-
-	protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
-		string attachmentUrl = WebUtility.UrlDecode((string) ctx.Request.RouteValues["url"]!);
-		DownloadedAttachment? maybeDownloadedAttachment = await Db.Downloads.GetDownloadedAttachment(attachmentUrl);
-
-		if (maybeDownloadedAttachment is {} downloadedAttachment) {
-			return new HttpOutput.File(downloadedAttachment.Type, downloadedAttachment.Data);
-		}
-		else {
-			return new HttpOutput.Redirect(attachmentUrl, permanent: false);
-		}
-	}
-}

app/Server/Endpoints/GetDownloadedFileEndpoint.cs

@@ -0,0 +1,26 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Download;
+using DHT.Utils.Http;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints;
+
+sealed class GetDownloadedFileEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		string url = WebUtility.UrlDecode((string) request.RouteValues["url"]!);
+		string normalizedUrl = DiscordCdn.NormalizeUrl(url);
+		
+		if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, WriteDataTo(response), cancellationToken)) {
+			response.Redirect(url, permanent: false);
+		}
+	}
+	
+	private static Func<Data.Download, Stream, CancellationToken, Task> WriteDataTo(HttpResponse response) {
+		return (download, stream, cancellationToken) => response.WriteStreamAsync(download.Type, download.Size, stream, cancellationToken);
+	}
+}

app/Server/Endpoints/GetTrackingScriptEndpoint.cs

@@ -1,5 +1,5 @@
-using System.Reflection;
+using System.Net.Mime;
-using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
 using DHT.Server.Database;
@@ -10,25 +10,17 @@ using Microsoft.AspNetCore.Http;
 
 namespace DHT.Server.Endpoints;
 
-sealed class GetTrackingScriptEndpoint : BaseEndpoint {
+sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters, ResourceLoader resources) : BaseEndpoint(db) {
-	private static ResourceLoader Resources { get; } = new (Assembly.GetExecutingAssembly());
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		string bootstrap = await resources.ReadTextAsync("Tracker/bootstrap.js");
+		string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + parameters.Port + ";")
+		                         .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(parameters.Token))
+		                         .Replace("/*[IMPORTS]*/", await resources.ReadJoinedAsync("Tracker/scripts/", '\n'))
+		                         .Replace("/*[CSS-CONTROLLER]*/", await resources.ReadTextAsync("Tracker/styles/controller.css"))
+		                         .Replace("/*[CSS-SETTINGS]*/", await resources.ReadTextAsync("Tracker/styles/settings.css"))
+		                         .Replace("/*[DEBUGGER]*/", request.Query.ContainsKey("debug") ? "debugger;" : "");
 		
-	private readonly ServerParameters serverParameters;
+		response.Headers.Append("X-DHT", "1");
-	
+		await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script, cancellationToken);
-	public GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : base(db) {
-		serverParameters = parameters;
-	}
-
-	protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
-		string bootstrap = await Resources.ReadTextAsync("Tracker/bootstrap.js");
-		string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + serverParameters.Port + ";")
-		                         .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(serverParameters.Token))
-		                         .Replace("/*[IMPORTS]*/", await Resources.ReadJoinedAsync("Tracker/scripts/", '\n'))
-		                         .Replace("/*[CSS-CONTROLLER]*/", await Resources.ReadTextAsync("Tracker/styles/controller.css"))
-		                         .Replace("/*[CSS-SETTINGS]*/", await Resources.ReadTextAsync("Tracker/styles/settings.css"))
-		                         .Replace("/*[DEBUGGER]*/", ctx.Request.Query.ContainsKey("debug") ? "debugger;" : "");
-		
-		ctx.Response.Headers.Append("X-DHT", "1");
-		return new HttpOutput.File("text/javascript", Encoding.UTF8.GetBytes(script));
 	}
 }

app/Server/Endpoints/GetViewerMessagesEndpoint.cs

@@ -0,0 +1,18 @@
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMessagesEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = "application/x-ndjson";
+		return ViewerJsonExport.GetMessages(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}

app/Server/Endpoints/GetViewerMetadataEndpoint.cs

@@ -0,0 +1,19 @@
+using System.Net.Mime;
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMetadataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = MediaTypeNames.Application.Json;
+		return ViewerJsonExport.GetMetadata(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}

app/Server/Endpoints/TrackChannelEndpoint.cs

@@ -1,5 +1,6 @@
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database;
@@ -8,18 +9,14 @@ using Microsoft.AspNetCore.Http;
 
 namespace DHT.Server.Endpoints;
 
-sealed class TrackChannelEndpoint : BaseEndpoint {
+sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
-	public TrackChannelEndpoint(IDatabaseFile db) : base(db) {}
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
-
+		var root = await ReadJson(request);
-	protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
-		var root = await ReadJson(ctx);
 		var server = ReadServer(root.RequireObject("server"), "server");
 		var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id);
 
 		await Db.Servers.Add([server]);
 		await Db.Channels.Add([channel]);
-
-		return HttpOutput.None;
 	}
 
 	private static Data.Server ReadServer(JsonElement json, string path) => new () {