Release v37.0

Closes #189

.idea/Discord-History-Tracker.iml

@@ -8,6 +8,7 @@
       <excludeFolder url="file://$MODULE_DIR$/.vscode" />
       <excludeFolder url="file://$MODULE_DIR$/bld" />
       <excludeFolder url="file://$MODULE_DIR$/lib" />
+      <excludeFolder url="file://$MODULE_DIR$/app" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.8" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

.vscode/tasks.json

@@ -1,52 +0,0 @@
-{
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "Build Raw",
-            "type": "shell",
-            "command": "python ./build.py --nominify --copytracker",
-            "group": {
-                "kind": "build",
-                "isDefault": true
-            },
-            "presentation": {
-                "reveal": "always",
-                "panel": "shared",
-                "echo": true,
-                "focus": false,
-                "showReuseMessage": false
-            },
-            "problemMatcher": []
-        },
-        {
-            "label": "Build Minified",
-            "type": "shell",
-            "command": "python ./build.py --copytracker",
-            "group": "build",
-            "presentation": {
-                "reveal": "always",
-                "panel": "shared",
-                "echo": true,
-                "focus": false,
-                "showReuseMessage": false
-            },
-            "problemMatcher": []
-        },
-        {
-            "label": "Build Website",
-            "type": "shell",
-            "command": "python ./build.py --website",
-            "group": "build",
-            "presentation": {
-                "reveal": "always",
-                "panel": "shared",
-                "echo": true,
-                "focus": false,
-                "showReuseMessage": false
-            },
-            "problemMatcher": []
-        }
-    ]
-}

README.md

@@ -1,48 +1,55 @@
 # Welcome
 
-All you need to **use Discord History Tracker** is either an up-to-date browser, or the [Discord desktop client](https://discord.com/download). Visit the [official website](https://dht.chylex.com) for instructions.
+For instructions on how to **use Discord History Tracker**, visit the [official website](https://dht.chylex.com).
 
 To **report an issue or suggestion**, first please see the [issues](https://github.com/chylex/Discord-History-Tracker/issues) page and make sure someone else hasn't already created a similar issue report. If you do find an existing issue, comment on it or add a reaction. Otherwise, either click [New Issue](https://github.com/chylex/Discord-History-Tracker/issues/new), or contact me via email [contact@chylex.com](mailto:contact@chylex.com) or Twitter [@chylexmc](https://twitter.com/chylexmc).
 
-If you are interested in **creating your own version** from the source code, continue reading the [build instructions](#Build-Instructions) below.
+If you are interested in **building from source code**, continue reading the [build instructions](#Build-Instructions) below.
+
+This branch is dedicated to the Discord History Tracker desktop app. If you are looking for the older browser-only version, visit the [master-browser-only](https://github.com/chylex/Discord-History-Tracker/tree/master-browser-only) branch.
 
 # Build Instructions
 
-Follow the steps below to create your own version of Discord History Tracker.
-
 ### Setup
 
 Fork the repository and clone it to your computer (if you've never used git, you can download the [GitHub Desktop](https://desktop.github.com) client to get started quickly).
 
-Now you can modify the source code:
-* `src/tracker/` contains JS files that are automatically combined into the **tracker bookmark/script**
-* `src/viewer/` contains HTML, CSS, JS files that are then combined into the **offline viewer page**
+Folder organization:
+* `app/` contains a Visual Studio solution for the desktop app
 * `lib/` contains utilities required to build the project
 * `web/` contains source code of the [official website](https://dht.chylex.com), which can be used as a template when making your own website
 
+To start editing source code for the desktop app, install the [.NET 5 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/5.0), and then open `app/DiscordHistoryTracker.sln` in [Visual Studio](https://visualstudio.microsoft.com/downloads/) or [Rider](https://www.jetbrains.com/rider/).
+
 ### Building
 
-After you've done changes to the source code, you will need to build it. Before that, download and install:
-* (**required**) [Python 3](https://www.python.org/downloads)
-  * Use to run the build script
-* (optional) [Node + npm](https://nodejs.org/en) & command line [uglify-js](https://www.npmjs.com/package/uglify-js)
-  * Not required on Windows
-  * Only required for optional [JS minification](#Minification) on Linux/Mac
+To build a `Debug` version of the desktop app, there are no additional requirements.
 
-Now open the folder that contains `build.py` in a command line, and run `python build.py` to create a build with default settings. The following files will be created:
-* `bld/track.js` is the raw tracker script that can be pasted into a browser console
-* `bld/track.html` is the tracker script but sanitized for inclusion in HTML (see `web/index.php` for examples)
-* `bld/viewer.html` is the complete offline viewer
+To build a `Release` version of the desktop app, follow the instructions for your operating system.
 
-You can tweak the build process using the following flags:
-* `python build.py --nominify` to disable [minification](#Minification)
+#### Release – Windows (64-bit)
 
-### Minification
+1. Install [Python 3](https://www.python.org/downloads), and ensure the `python` executable is in your `PATH`
+2. Install [Powershell 5](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows) or newer (on Windows 10, the included version of Powershell should be enough)
 
-The build process automatically minifies JS using `UglifyJS@3`, and CSS using a custom minifier.
+The `lib/` folder contains an installation of [Node](https://nodejs.org/en) and [uglify-js](https://www.npmjs.com/package/uglify-js), which are used to minify the tracking script. This installation will only work on 64-bit Windows; building on 32-bit Windows is not supported, but you can try.
 
-* If the `--nominify` flag is used, minification will be completely disabled
-* If `uglify-js` is not available from the command line, JS minification will be skipped
-  * When building on Windows 64-bit, the build script will use the included Node runner and packages
-  * When building on Windows 32-bit, you will need to download [Node 32-bit](https://nodejs.org/en/download) and replace the included one in `lib/`
-  * When building on Linux/Mac, the build script will attempt to find `uglifyjs` in the command line
+Run the `app/build.bat` script, and read the [Distribution](#distribution) section below.
+
+#### Release – Other Operating Systems
+
+1. Install [Python 3](https://www.python.org/downloads), and ensure the `python` executable exists and launches Python 3
+   - On Debian and derivatives, you can install `python-is-python3`
+   - On other distributions, you can create a link manually, for ex. `ln -s /usr/bin/python3 /usr/bin/python`
+   - If you don't want `python` to mean Python 3, then edit `Desktop.csproj` and change `python` to `python3`
+2. Install [Node + npm](https://nodejs.org/en)
+3. Install [uglify-js](https://www.npmjs.com/package/uglify-js) globally (`npm install -g uglify-js`)
+4. Install the `zip` package from your repository
+
+Run the `app/build.sh` script, and read the [Distribution](#distribution) section below.
+
+#### Distribution
+
+The mentioned build scripts will prepare `Release` builds ready for distribution. Once the script finishes, the `app/bin` folder will contain self-contained executables for each major operating system, and a portable version that works on all other systems but requires .NET 5 to be installed.
+
+Note that when building on Windows, the generated `.zip` files for Linux and Mac will not have correct file permissions, so it will not be possible to run them by double-clicking `DiscordHistoryTracker`. I tried using Python to re-create the archives with correct file permissions, but found that Linux `zip` tools could not see them. The only working solution is building the Windows + portable version on Windows, and Linux + Mac version on Linux.

app/.idea/.idea.DiscordHistoryTracker/.idea/avalonia.xml

@@ -7,13 +7,17 @@
         <entry key="Desktop/Dialogs/CheckBox/CheckBoxDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Dialogs/Message/MessageDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Dialogs/TextBox/TextBoxDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
-        <entry key="Desktop/Main/Controls/FilterPanel.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Controls/AttachmentFilterPanel.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Pages/AttachmentsPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Pages/DebugPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/TrackingPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/ViewerPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Screens/MainContentScreen.axaml" value="Desktop/Desktop.csproj" />

app/Desktop/App.axaml

@@ -7,6 +7,7 @@
     <Application.Styles>
 
         <FluentTheme Mode="Light" />
+        <StyleInclude Source="avares://Avalonia.Controls.DataGrid/Themes/Default.xaml" />
 
         <Style Selector="Button, CheckBox, RadioButton, Expander /template/ ToggleButton#ExpanderHeader">
             <Setter Property="Cursor" Value="Hand" />
@@ -35,6 +36,20 @@
         <Style Selector="TextBox:disabled"><!-- TODO bug in Avalonia (https://github.com/AvaloniaUI/Avalonia/pull/7792) -->
             <Setter Property="Foreground" Value="{DynamicResource TextControlForegroundDisabled}" />
         </Style>
+        <Style Selector="TextBox:error DataValidationErrors">
+            <Style.Resources>
+                <ControlTemplate x:Key="InlineDataValidationContentTemplate" TargetType="DataValidationErrors">
+                    <ContentPresenter Name="PART_ContentPresenter"
+                                      Padding="{TemplateBinding Padding}"
+                                      Background="{TemplateBinding Background}"
+                                      BorderThickness="{TemplateBinding BorderThickness}"
+                                      CornerRadius="{TemplateBinding CornerRadius}"
+                                      Content="{TemplateBinding Content}"
+                                      ContentTemplate="{TemplateBinding ContentTemplate}" />
+                </ControlTemplate>
+            </Style.Resources>
+            <Setter Property="Template" Value="{StaticResource InlineDataValidationContentTemplate}" />
+        </Style>
 
         <Style Selector="Expander /template/ ToggleButton#ExpanderHeader">
             <Setter Property="HorizontalContentAlignment" Value="Left" />
@@ -105,6 +120,7 @@
     <Application.Resources>
 
         <common:NumberValueConverter x:Key="NumberValueConverter" />
+        <common:BytesValueConverter x:Key="BytesValueConverter" />
 
         <system:Double x:Key="ControlContentThemeFontSize">14</system:Double>
         <CornerRadius x:Key="ControlCornerRadius">0</CornerRadius>
@@ -154,7 +170,7 @@
         <SolidColorBrush x:Key="TextControlPlaceholderForegroundDisabled" Color="#AAAAAA" />
 
         <Thickness x:Key="ExpanderHeaderPadding">15,0</Thickness>
-        <Thickness x:Key="ExpanderContentPadding">15</Thickness>
+        <Thickness x:Key="ExpanderContentPadding">12</Thickness>
         <SolidColorBrush x:Key="ExpanderBorderBrush" Color="#697DAB" />
         <SolidColorBrush x:Key="ExpanderBackground" Color="#697DAB" />
         <SolidColorBrush x:Key="ExpanderForeground" Color="#FFFFFF" />

app/Desktop/Common/BytesValueConverter.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Globalization;
+using Avalonia.Data.Converters;
+
+namespace DHT.Desktop.Common {
+	sealed class BytesValueConverter : IValueConverter {
+		private static readonly string[] Units = {
+			"B",
+			"kB",
+			"MB",
+			"GB",
+			"TB"
+		};
+
+		private const int Scale = 1000;
+
+		private static string Convert(ulong size) {
+			int power = size == 0L ? 0 : (int) Math.Log(size, Scale);
+			int unit = power >= Units.Length ? Units.Length - 1 : power;
+			if (unit == 0) {
+				return string.Format(Program.Culture, "{0:n0}", size) + " " + Units[unit];
+			}
+			else {
+				double humanReadableSize = size / Math.Pow(Scale, unit);
+				return string.Format(Program.Culture, "{0:n0}", humanReadableSize) + " " + Units[unit];
+			}
+		}
+
+		public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) {
+			if (value is long size and >= 0L) {
+				return Convert((ulong) size);
+			}
+			else if (value is ulong usize) {
+				return Convert(usize);
+			}
+			else {
+				return "-";
+			}
+		}
+
+		public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) {
+			throw new NotSupportedException();
+		}
+	}
+}

app/Desktop/Common/NumberValueConverter.cs

@@ -5,7 +5,7 @@ using Avalonia.Data.Converters;
 namespace DHT.Desktop.Common {
 	sealed class NumberValueConverter : IValueConverter {
 		public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) {
-			return string.Format(Program.Culture, "{0:n0}", value);
+			return value == null ? "-" : string.Format(Program.Culture, "{0:n0}", value);
 		}
 
 		public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) {

app/Desktop/Desktop.csproj

@@ -21,15 +21,20 @@
     <DebugType>none</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Avalonia" Version="0.10.13" />
-    <PackageReference Include="Avalonia.Desktop" Version="0.10.13" />
-    <PackageReference Include="Avalonia.Diagnostics" Version="0.10.13" Condition=" '$(Configuration)' == 'Debug' " />
+    <PackageReference Include="Avalonia" Version="0.10.14" />
+    <PackageReference Include="Avalonia.Controls.DataGrid" Version="0.10.14" />
+    <PackageReference Include="Avalonia.Desktop" Version="0.10.14" />
+    <PackageReference Include="Avalonia.Diagnostics" Version="0.10.14" Condition=" '$(Configuration)' == 'Debug' " />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Server\Server.csproj" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Version.cs" Link="Version.cs" />
+    <Compile Update="Dialogs\TextBox\TextBoxDialog.axaml.cs">
+      <DependentUpon>CheckBoxDialog.axaml</DependentUpon>
+      <SubType>Code</SubType>
+    </Compile>
   </ItemGroup>
   <ItemGroup>
     <AvaloniaResource Include="Resources/icon.ico" />
@@ -43,11 +48,6 @@
       <Link>Resources/Tracker/scripts/%(RecursiveDir)%(Filename)%(Extension)</Link>
       <Visible>false</Visible>
     </EmbeddedResource>
-    <EmbeddedResource Include="$(ProjectDir)bin/.res/scripts/**" Condition=" '$(Configuration)' == 'Release' ">
-      <LogicalName>Tracker\scripts\%(RecursiveDir)%(Filename)%(Extension)</LogicalName>
-      <Link>Resources/Tracker/scripts/%(RecursiveDir)%(Filename)%(Extension)</Link>
-      <Visible>false</Visible>
-    </EmbeddedResource>
     <EmbeddedResource Include="../Resources/Tracker/styles/**">
       <LogicalName>Tracker\styles\%(RecursiveDir)%(Filename)%(Extension)</LogicalName>
       <Link>Resources/Tracker/styles/%(RecursiveDir)%(Filename)%(Extension)</Link>
@@ -60,10 +60,18 @@
     </EmbeddedResource>
   </ItemGroup>
   <Target Name="MinifyResources" BeforeTargets="PrepareForBuild" Condition=" '$(Configuration)' == 'Release' ">
+    <PropertyGroup>
+      <MinifiedResourceDir>$(ProjectDir)bin/.res/scripts</MinifiedResourceDir>
+    </PropertyGroup>
     <ItemGroup>
       <UpToDateCheckInput Include="$(ProjectDir)../Resources/Tracker/scripts/**" Visible="false" />
+      <EmbeddedResource Include="$(MinifiedResourceDir)/discord.js" LogicalName="Tracker\scripts\discord.js" Visible="false" />
+      <EmbeddedResource Include="$(MinifiedResourceDir)/dom.js" LogicalName="Tracker\scripts\dom.js" Visible="false" />
+      <EmbeddedResource Include="$(MinifiedResourceDir)/gui.js" LogicalName="Tracker\scripts\gui.js" Visible="false" />
+      <EmbeddedResource Include="$(MinifiedResourceDir)/settings.js" LogicalName="Tracker\scripts\settings.js" Visible="false" />
+      <EmbeddedResource Include="$(MinifiedResourceDir)/state.js" LogicalName="Tracker\scripts\state.js" Visible="false" />
     </ItemGroup>
     <RemoveDir Directories="$(ProjectDir)bin/.res/scripts" />
-    <Exec Command="$(ProjectDir)../Resources/minify.py" WorkingDirectory="$(ProjectDir)../Resources" IgnoreExitCode="false" />
+    <Exec Command="python $(ProjectDir)../Resources/minify.py" WorkingDirectory="$(ProjectDir)../Resources" IgnoreExitCode="false" />
   </Target>
 </Project>

app/Desktop/Dialogs/CheckBox/CheckBoxDialog.axaml

@@ -34,8 +34,8 @@
 
     <StackPanel Margin="20">
         <ScrollViewer MaxHeight="400">
-            <ItemsControl Items="{Binding Items}">
-                <ItemsControl.ItemTemplate>
+            <ItemsRepeater Items="{Binding Items}">
+                <ItemsRepeater.ItemTemplate>
                     <DataTemplate>
                         <CheckBox IsChecked="{Binding Checked}">
                             <Label>
@@ -43,8 +43,8 @@
                             </Label>
                         </CheckBox>
                     </DataTemplate>
-                </ItemsControl.ItemTemplate>
-            </ItemsControl>
+                </ItemsRepeater.ItemTemplate>
+            </ItemsRepeater>
         </ScrollViewer>
         <Panel Classes="buttons">
             <WrapPanel Classes="left">

app/Desktop/Dialogs/TextBox/TextBoxDialog.axaml

@@ -0,0 +1,56 @@
+<Window xmlns="https://github.com/avaloniaui"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:namespace="clr-namespace:DHT.Desktop.Dialogs.TextBox"
+        mc:Ignorable="d" d:DesignWidth="500"
+        x:Class="DHT.Desktop.Dialogs.TextBox.TextBoxDialog"
+        Title="{Binding Title}"
+        Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
+        Width="500" SizeToContent="Height" CanResize="False"
+        WindowStartupLocation="CenterOwner">
+
+    <Window.DataContext>
+        <namespace:TextBoxDialogModel />
+    </Window.DataContext>
+
+    <Window.Styles>
+        <Style Selector="Panel.buttons">
+            <Setter Property="Margin" Value="0 20 0 0" />
+        </Style>
+        <Style Selector="Panel.buttons > WrapPanel.right">
+            <Setter Property="HorizontalAlignment" Value="Right" />
+        </Style>
+        <Style Selector="Panel.buttons Button">
+            <Setter Property="MinWidth" Value="80" />
+            <Setter Property="Margin" Value="8 0 0 0" />
+        </Style>
+    </Window.Styles>
+
+    <StackPanel Margin="20">
+        <ScrollViewer MaxHeight="400">
+            <StackPanel Spacing="10">
+                <TextBlock Text="{Binding Description}" TextWrapping="Wrap" />
+                <ItemsRepeater Items="{Binding Items}">
+                    <ItemsRepeater.ItemTemplate>
+                        <DataTemplate>
+                            <DockPanel Margin="0 5 25 0">
+                                <TextBox Name="Input" Text="{Binding Value}" Width="180" VerticalAlignment="Top" DockPanel.Dock="Right" />
+                                <Label Target="Input" VerticalAlignment="Center" DockPanel.Dock="Left">
+                                    <TextBlock Text="{Binding Title}" TextWrapping="Wrap" />
+                                </Label>
+                            </DockPanel>
+                        </DataTemplate>
+                    </ItemsRepeater.ItemTemplate>
+                </ItemsRepeater>
+            </StackPanel>
+        </ScrollViewer>
+        <Panel Classes="buttons">
+            <WrapPanel Classes="right">
+                <Button Click="ClickOk" IsEnabled="{Binding !HasErrors}">OK</Button>
+                <Button Click="ClickCancel">Cancel</Button>
+            </WrapPanel>
+        </Panel>
+    </StackPanel>
+
+</Window>

app/Desktop/Dialogs/TextBox/TextBoxDialog.axaml.cs

@@ -0,0 +1,31 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia;
+using Avalonia.Controls;
+using Avalonia.Interactivity;
+using Avalonia.Markup.Xaml;
+using DHT.Desktop.Dialogs.Message;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+	public sealed class TextBoxDialog : Window {
+		public TextBoxDialog() {
+			InitializeComponent();
+			#if DEBUG
+			this.AttachDevTools();
+			#endif
+		}
+
+		private void InitializeComponent() {
+			AvaloniaXamlLoader.Load(this);
+		}
+
+		public void ClickOk(object? sender, RoutedEventArgs e) {
+			Close(DialogResult.OkCancel.Ok);
+		}
+
+		public void ClickCancel(object? sender, RoutedEventArgs e) {
+			Close(DialogResult.OkCancel.Cancel);
+		}
+	}
+}
+

app/Desktop/Dialogs/TextBox/TextBoxDialogModel.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	class TextBoxDialogModel : BaseModel {
+		public string Title { get; init; } = "";
+		public string Description { get; init; } = "";
+
+		private IReadOnlyList<TextBoxItem> items = Array.Empty<TextBoxItem>();
+
+		public IReadOnlyList<TextBoxItem> Items {
+			get => items;
+
+			protected set {
+				foreach (var item in items) {
+					item.ErrorsChanged -= OnItemErrorsChanged;
+				}
+
+				items = value;
+
+				foreach (var item in items) {
+					item.ErrorsChanged += OnItemErrorsChanged;
+				}
+			}
+		}
+		
+		public bool HasErrors => Items.Any(static item => !item.IsValid);
+
+		private void OnItemErrorsChanged(object? sender, DataErrorsChangedEventArgs e) {
+			OnPropertyChanged(nameof(HasErrors));
+		}
+	}
+
+	sealed class TextBoxDialogModel<T> : TextBoxDialogModel {
+		public new IReadOnlyList<TextBoxItem<T>> Items { get; }
+
+		public IEnumerable<TextBoxItem<T>> ValidItems => Items.Where(static item => item.IsValid);
+
+		public TextBoxDialogModel(IEnumerable<TextBoxItem<T>> items) {
+			this.Items = new List<TextBoxItem<T>>(items);
+			base.Items = this.Items;
+		}
+	}
+}

app/Desktop/Dialogs/TextBox/TextBoxItem.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections;
+using System.ComponentModel;
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	class TextBoxItem : BaseModel, INotifyDataErrorInfo {
+		public string Title { get; init; } = "";
+		public object? Item { get; init; } = null;
+		
+		public Func<string, bool> ValidityCheck { get; init; } = static _ => true;
+		public bool IsValid => ValidityCheck(Value);
+
+		private string value = string.Empty;
+
+		public string Value {
+			get => this.value;
+			set {
+				Change(ref this.value, value);
+				ErrorsChanged?.Invoke(this, new DataErrorsChangedEventArgs(nameof(Value)));
+			}
+		}
+
+		public IEnumerable GetErrors(string? propertyName) {
+			if (propertyName == nameof(Value) && !IsValid) {
+				yield return string.Empty;
+			}
+		}
+
+		public bool HasErrors => !IsValid;
+		public event EventHandler<DataErrorsChangedEventArgs>? ErrorsChanged;
+	}
+
+	sealed class TextBoxItem<T> : TextBoxItem {
+		public new T Item { get; }
+
+		public TextBoxItem(T item) {
+			this.Item = item;
+			base.Item = item;
+		}
+	}
+}

app/Desktop/Main/Controls/AttachmentFilterPanel.axaml

@@ -0,0 +1,51 @@
+<UserControl xmlns="https://github.com/avaloniaui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+             xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
+             mc:Ignorable="d"
+             x:Class="DHT.Desktop.Main.Controls.AttachmentFilterPanel">
+
+    <Design.DataContext>
+        <controls:AttachmentFilterPanelModel />
+    </Design.DataContext>
+
+    <UserControl.Styles>
+        <Style Selector="WrapPanel > StackPanel">
+            <Setter Property="Margin" Value="0 20 40 0" />
+            <Setter Property="Spacing" Value="4" />
+        </Style>
+        <Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
+            <Setter Property="Margin" Value="0 20 0 0" />
+        </Style>
+        <Style Selector="Grid > Label">
+            <Setter Property="VerticalAlignment" Value="Center" />
+        </Style>
+        <Style Selector="ComboBox">
+            <Setter Property="Margin" Value="8 0 0 0" />
+        </Style>
+        <Style Selector="Button">
+            <Setter Property="Margin" Value="0 0 0 8" />
+        </Style>
+    </UserControl.Styles>
+
+    <StackPanel>
+        <TextBlock Text="{Binding FilterStatisticsText}" />
+        <WrapPanel>
+            <StackPanel>
+                <CheckBox IsChecked="{Binding LimitSize}">Limit Size</CheckBox>
+                <StackPanel Orientation="Horizontal">
+                    <TextBox Text="{Binding MaximumSize}" IsEnabled="{Binding LimitSize}" HorizontalContentAlignment="Right" />
+                    <ComboBox IsEnabled="{Binding LimitSize}" Items="{Binding Units}" SelectedItem="{Binding MaximumSizeUnit}">
+                        <ComboBox.ItemTemplate>
+                            <DataTemplate>
+                                <TextBlock Text="{Binding Name}" />
+                            </DataTemplate>
+                        </ComboBox.ItemTemplate>
+                    </ComboBox>
+                </StackPanel>
+            </StackPanel>
+        </WrapPanel>
+    </StackPanel>
+
+</UserControl>

app/Desktop/Main/Controls/AttachmentFilterPanel.axaml.cs

@@ -0,0 +1,17 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia.Controls;
+using Avalonia.Markup.Xaml;
+
+namespace DHT.Desktop.Main.Controls {
+	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+	public sealed class AttachmentFilterPanel : UserControl {
+		public AttachmentFilterPanel() {
+			InitializeComponent();
+		}
+
+		private void InitializeComponent() {
+			AvaloniaXamlLoader.Load(this);
+		}
+	}
+}
+

app/Desktop/Main/Controls/AttachmentFilterPanelModel.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using DHT.Desktop.Common;
+using DHT.Server.Data.Filters;
+using DHT.Server.Database;
+using DHT.Utils.Models;
+using DHT.Utils.Tasks;
+
+namespace DHT.Desktop.Main.Controls {
+	sealed class AttachmentFilterPanelModel : BaseModel, IDisposable {
+		public sealed record Unit(string Name, uint Scale);
+
+		private static readonly Unit[] AllUnits = {
+			new ("B", 1),
+			new ("kB", 1024),
+			new ("MB", 1024 * 1024)
+		};
+
+		private static readonly HashSet<string> FilterProperties = new () {
+			nameof(LimitSize),
+			nameof(MaximumSize),
+			nameof(MaximumSizeUnit)
+		};
+
+		public string FilterStatisticsText { get; private set; } = "";
+
+		private bool limitSize = false;
+		private ulong maximumSize = 0L;
+		private Unit maximumSizeUnit = AllUnits[0];
+
+		public bool LimitSize {
+			get => limitSize;
+			set => Change(ref limitSize, value);
+		}
+
+		public ulong MaximumSize {
+			get => maximumSize;
+			set => Change(ref maximumSize, value);
+		}
+
+		public Unit MaximumSizeUnit {
+			get => maximumSizeUnit;
+			set => Change(ref maximumSizeUnit, value);
+		}
+
+		public IEnumerable<Unit> Units => AllUnits;
+
+		private readonly IDatabaseFile db;
+		private readonly string verb;
+
+		private readonly AsyncValueComputer<long> matchingAttachmentCountComputer;
+		private long? matchingAttachmentCount;
+		private long? totalAttachmentCount;
+
+		[Obsolete("Designer")]
+		public AttachmentFilterPanelModel() : this(DummyDatabaseFile.Instance) {}
+
+		public AttachmentFilterPanelModel(IDatabaseFile db, string verb = "Matches") {
+			this.db = db;
+			this.verb = verb;
+
+			this.matchingAttachmentCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetAttachmentCounts).Build();
+
+			UpdateFilterStatistics();
+			
+			PropertyChanged += OnPropertyChanged;
+			db.Statistics.PropertyChanged += OnDbStatisticsChanged;
+		}
+
+		public void Dispose() {
+			db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
+		}
+
+		private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
+			if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
+				UpdateFilterStatistics();
+			}
+		}
+
+		private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
+			if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
+				totalAttachmentCount = db.Statistics.TotalAttachments;
+				UpdateFilterStatistics();
+			}
+		}
+
+		private void UpdateFilterStatistics() {
+			var filter = CreateFilter();
+			if (filter.IsEmpty) {
+				matchingAttachmentCountComputer.Cancel();
+				matchingAttachmentCount = totalAttachmentCount;
+				UpdateFilterStatisticsText();
+			}
+			else {
+				matchingAttachmentCount = null;
+				UpdateFilterStatisticsText();
+				matchingAttachmentCountComputer.Compute(() => db.CountAttachments(filter));
+			}
+		}
+
+		private void SetAttachmentCounts(long matchingAttachmentCount) {
+			this.matchingAttachmentCount = matchingAttachmentCount;
+			UpdateFilterStatisticsText();
+		}
+
+		private void UpdateFilterStatisticsText() {
+			var matchingAttachmentCountStr = matchingAttachmentCount?.Format() ?? "(...)";
+			var totalAttachmentCountStr = totalAttachmentCount?.Format() ?? "(...)";
+
+			FilterStatisticsText = verb + " " + matchingAttachmentCountStr + " out of " + totalAttachmentCountStr + " attachment" + (totalAttachmentCount is null or 1 ? "." : "s.");
+			OnPropertyChanged(nameof(FilterStatisticsText));
+		}
+
+		public AttachmentFilter CreateFilter() {
+			AttachmentFilter filter = new();
+
+			if (LimitSize) {
+				try {
+					filter.MaxBytes = maximumSize * maximumSizeUnit.Scale;
+				} catch (ArithmeticException) {
+					// set no size limit, because the overflown size is larger than any file could possibly be
+				}
+			}
+
+			return filter;
+		}
+	}
+}

app/Desktop/Main/Controls/FilterPanel.axaml

@@ -1,60 +0,0 @@
-<UserControl xmlns="https://github.com/avaloniaui"
-             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
-             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
-             xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
-             mc:Ignorable="d"
-             x:Class="DHT.Desktop.Main.Controls.FilterPanel">
-
-    <Design.DataContext>
-        <controls:FilterPanelModel />
-    </Design.DataContext>
-
-    <UserControl.Styles>
-        <Style Selector="WrapPanel > StackPanel">
-            <Setter Property="Margin" Value="0 20 40 0" />
-            <Setter Property="Spacing" Value="4" />
-        </Style>
-        <Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
-            <Setter Property="Margin" Value="0 20 0 0" />
-        </Style>
-        <Style Selector="Grid > Label">
-            <Setter Property="VerticalAlignment" Value="Center" />
-        </Style>
-        <Style Selector="Grid > CalendarDatePicker">
-            <Setter Property="CornerRadius" Value="0" />
-            <Setter Property="HorizontalAlignment" Value="Stretch" />
-            <Setter Property="HorizontalContentAlignment" Value="Center" />
-            <Setter Property="VerticalAlignment" Value="Center" />
-            <Setter Property="VerticalContentAlignment" Value="Center" />
-            <Setter Property="IsTodayHighlighted" Value="True" />
-            <Setter Property="SelectedDateFormat" Value="Short" />
-        </Style>
-        <Style Selector="Button">
-            <Setter Property="Margin" Value="0 0 0 8" />
-        </Style>
-    </UserControl.Styles>
-
-    <WrapPanel>
-        <StackPanel>
-            <CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
-            <Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
-                <Label Grid.Row="0" Grid.Column="0">From:</Label>
-                <CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
-                <Label Grid.Row="2" Grid.Column="0">To:</Label>
-                <CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
-            </Grid>
-        </StackPanel>
-        <StackPanel>
-            <CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
-            <Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
-            <TextBlock Text="{Binding ChannelFilterLabel}" />
-        </StackPanel>
-        <StackPanel>
-            <CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
-            <Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
-            <TextBlock Text="{Binding UserFilterLabel}" />
-        </StackPanel>
-    </WrapPanel>
-
-</UserControl>

app/Desktop/Main/Controls/MessageFilterPanel.axaml

@@ -0,0 +1,63 @@
+<UserControl xmlns="https://github.com/avaloniaui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+             xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
+             mc:Ignorable="d"
+             x:Class="DHT.Desktop.Main.Controls.MessageFilterPanel">
+
+    <Design.DataContext>
+        <controls:MessageFilterPanelModel />
+    </Design.DataContext>
+
+    <UserControl.Styles>
+        <Style Selector="WrapPanel > StackPanel">
+            <Setter Property="Margin" Value="0 20 40 0" />
+            <Setter Property="Spacing" Value="4" />
+        </Style>
+        <Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
+            <Setter Property="Margin" Value="0 20 0 0" />
+        </Style>
+        <Style Selector="Grid > Label">
+            <Setter Property="VerticalAlignment" Value="Center" />
+        </Style>
+        <Style Selector="Grid > CalendarDatePicker">
+            <Setter Property="CornerRadius" Value="0" />
+            <Setter Property="HorizontalAlignment" Value="Stretch" />
+            <Setter Property="HorizontalContentAlignment" Value="Center" />
+            <Setter Property="VerticalAlignment" Value="Center" />
+            <Setter Property="VerticalContentAlignment" Value="Center" />
+            <Setter Property="IsTodayHighlighted" Value="True" />
+            <Setter Property="SelectedDateFormat" Value="Short" />
+        </Style>
+        <Style Selector="Button">
+            <Setter Property="Margin" Value="0 0 0 8" />
+        </Style>
+    </UserControl.Styles>
+
+    <StackPanel>
+        <TextBlock Text="{Binding FilterStatisticsText}" />
+        <WrapPanel>
+            <StackPanel>
+                <CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
+                <Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
+                    <Label Grid.Row="0" Grid.Column="0">From:</Label>
+                    <CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
+                    <Label Grid.Row="2" Grid.Column="0">To:</Label>
+                    <CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
+                </Grid>
+            </StackPanel>
+            <StackPanel>
+                <CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
+                <Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
+                <TextBlock Text="{Binding ChannelFilterLabel}" />
+            </StackPanel>
+            <StackPanel>
+                <CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
+                <Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
+                <TextBlock Text="{Binding UserFilterLabel}" />
+            </StackPanel>
+        </WrapPanel>
+    </StackPanel>
+
+</UserControl>

app/Desktop/Main/Controls/MessageFilterPanel.axaml.cs

@@ -4,11 +4,11 @@ using Avalonia.Markup.Xaml;
 
 namespace DHT.Desktop.Main.Controls {
 	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
-	public sealed class FilterPanel : UserControl {
+	public sealed class MessageFilterPanel : UserControl {
 		private CalendarDatePicker StartDatePicker => this.FindControl<CalendarDatePicker>("StartDatePicker");
 		private CalendarDatePicker EndDatePicker => this.FindControl<CalendarDatePicker>("EndDatePicker");
 
-		public FilterPanel() {
+		public MessageFilterPanel() {
 			InitializeComponent();
 		}
 
@@ -25,7 +25,7 @@ namespace DHT.Desktop.Main.Controls {
 		}
 
 		public void CalendarDatePicker_OnSelectedDateChanged(object? sender, SelectionChangedEventArgs e) {
-			if (DataContext is FilterPanelModel model) {
+			if (DataContext is MessageFilterPanelModel model) {
 				model.StartDate = StartDatePicker.SelectedDate;
 				model.EndDate = EndDatePicker.SelectedDate;
 			}

app/Desktop/Main/Controls/MessageFilterPanelModel.cs

@@ -12,9 +12,10 @@ using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Server.Database;
 using DHT.Utils.Models;
+using DHT.Utils.Tasks;
 
 namespace DHT.Desktop.Main.Controls {
-	sealed class FilterPanelModel : BaseModel, IDisposable {
+	sealed class MessageFilterPanelModel : BaseModel, IDisposable {
 		private static readonly HashSet<string> FilterProperties = new () {
 			nameof(FilterByDate),
 			nameof(StartDate),
@@ -25,6 +26,8 @@ namespace DHT.Desktop.Main.Controls {
 			nameof(IncludedUsers)
 		};
 
+		public string FilterStatisticsText { get; private set; } = "";
+
 		public event PropertyChangedEventHandler? FilterPropertyChanged;
 
 		public bool HasAnyFilters => FilterByDate || FilterByChannel || FilterByUser;
@@ -88,14 +91,23 @@ namespace DHT.Desktop.Main.Controls {
 
 		private readonly Window window;
 		private readonly IDatabaseFile db;
+		private readonly string verb;
+
+		private readonly AsyncValueComputer<long> exportedMessageCountComputer;
+		private long? exportedMessageCount;
+		private long? totalMessageCount;
 
 		[Obsolete("Designer")]
-		public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
+		public MessageFilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
 
-		public FilterPanelModel(Window window, IDatabaseFile db) {
+		public MessageFilterPanelModel(Window window, IDatabaseFile db, string verb = "Matches") {
 			this.window = window;
 			this.db = db;
+			this.verb = verb;
+			
+			this.exportedMessageCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetExportedMessageCount).Build();
 
+			UpdateFilterStatistics();
 			UpdateChannelFilterLabel();
 			UpdateUserFilterLabel();
 
@@ -109,6 +121,7 @@ namespace DHT.Desktop.Main.Controls {
 
 		private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
 			if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
+				UpdateFilterStatistics();
 				FilterPropertyChanged?.Invoke(sender, e);
 			}
 
@@ -121,7 +134,11 @@ namespace DHT.Desktop.Main.Controls {
 		}
 
 		private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
-			if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
+			if (e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
+				totalMessageCount = db.Statistics.TotalMessages;
+				UpdateFilterStatistics();
+			}
+			else if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
 				UpdateChannelFilterLabel();
 			}
 			else if (e.PropertyName == nameof(DatabaseStatistics.TotalUsers)) {
@@ -129,6 +146,33 @@ namespace DHT.Desktop.Main.Controls {
 			}
 		}
 
+		private void UpdateFilterStatistics() {
+			var filter = CreateFilter();
+			if (filter.IsEmpty) {
+				exportedMessageCountComputer.Cancel();
+				exportedMessageCount = totalMessageCount;
+				UpdateFilterStatisticsText();
+			}
+			else {
+				exportedMessageCount = null;
+				UpdateFilterStatisticsText();
+				exportedMessageCountComputer.Compute(() => db.CountMessages(filter));
+			}
+		}
+
+		private void SetExportedMessageCount(long exportedMessageCount) {
+			this.exportedMessageCount = exportedMessageCount;
+			UpdateFilterStatisticsText();
+		}
+		
+		private void UpdateFilterStatisticsText() {
+			var exportedMessageCountStr = exportedMessageCount?.Format() ?? "(...)";
+			var totalMessageCountStr = totalMessageCount?.Format() ?? "(...)";
+			
+			FilterStatisticsText = verb + " " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or 1 ? "." : "s.");
+			OnPropertyChanged(nameof(FilterStatisticsText));
+		}
+
 		public async void OpenChannelFilterDialog() {
 			var servers = db.GetAllServers().ToDictionary(static server => server.Id);
 			var items = new List<CheckBoxItem<ulong>>();
@@ -211,7 +255,7 @@ namespace DHT.Desktop.Main.Controls {
 
 			if (FilterByDate) {
 				filter.StartDate = StartDate;
-				filter.EndDate = EndDate;
+				filter.EndDate = EndDate?.AddDays(1).AddMilliseconds(-1);
 			}
 
 			if (FilterByChannel) {

app/Desktop/Main/Controls/ServerConfigurationPanel.axaml

@@ -24,7 +24,7 @@
     <StackPanel>
         <Button Command="{Binding OnClickToggleServerButton}" Content="{Binding ToggleServerButtonText}" IsEnabled="{Binding IsToggleServerButtonEnabled}" />
         <TextBlock TextWrapping="Wrap" Margin="0 15">
-            The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy and apply the tracking script again.
+            The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy/paste the tracking script again.
         </TextBlock>
         <WrapPanel>
             <StackPanel>
@@ -33,9 +33,7 @@
             </StackPanel>
             <StackPanel>
                 <Label Target="Token">Token</Label>
-                <StackPanel Orientation="Horizontal">
-                    <TextBox x:Name="Token" Width="200" Text="{Binding InputToken}" />
-                </StackPanel>
+                <TextBox x:Name="Token" Width="200" Text="{Binding InputToken}" />
             </StackPanel>
             <StackPanel VerticalAlignment="Bottom">
                 <Button Command="{Binding OnClickRandomizeToken}">Randomize Token</Button>

app/Desktop/Main/MainWindow.axaml

@@ -8,7 +8,7 @@
         Title="{Binding Title}"
         Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
         Width="800" Height="500"
-        MinWidth="500" MinHeight="275"
+        MinWidth="520" MinHeight="300"
         WindowStartupLocation="CenterScreen"
         Closed="OnClosed">
 

app/Desktop/Main/MainWindow.axaml.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using Avalonia;
 using Avalonia.Controls;
 using Avalonia.Markup.Xaml;
+using DHT.Desktop.Main.Pages;
 using JetBrains.Annotations;
 
 namespace DHT.Desktop.Main {
@@ -30,6 +32,14 @@ namespace DHT.Desktop.Main {
 			if (DataContext is IDisposable disposable) {
 				disposable.Dispose();
 			}
+
+			foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {
+				try {
+					File.Delete(temporaryFile);
+				} catch (Exception) {
+					// ignored
+				}
+			}
 		}
 	}
 }

app/Desktop/Main/Pages/AdvancedPage.axaml

@@ -15,5 +15,11 @@
         <Expander Header="Internal Server Configuration" IsExpanded="True">
             <controls:ServerConfigurationPanel DataContext="{Binding ServerConfigurationModel}" />
         </Expander>
+        <Expander Header="Database Tools" IsExpanded="True">
+            <StackPanel Orientation="Vertical" Spacing="10">
+                <TextBlock TextWrapping="Wrap">Recreates the database to remove any traces of deleted data, which frees up space and prevents forensic recovery.</TextBlock>
+                <Button Command="{Binding VacuumDatabase}">Vacuum Database</Button>
+            </StackPanel>
+        </Expander>
     </StackPanel>
 </UserControl>

app/Desktop/Main/Pages/AdvancedPageModel.cs

@@ -1,5 +1,6 @@
 using System;
 using Avalonia.Controls;
+using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Main.Controls;
 using DHT.Desktop.Server;
 using DHT.Server.Database;
@@ -9,19 +10,30 @@ namespace DHT.Desktop.Main.Pages {
 	sealed class AdvancedPageModel : BaseModel, IDisposable {
 		public ServerConfigurationPanelModel ServerConfigurationModel { get; }
 
-		[Obsolete("Designer")]
-		public AdvancedPageModel() : this(null!, new ServerManager(DummyDatabaseFile.Instance)) {}
+		private readonly Window window;
+		private readonly IDatabaseFile db;
+
+		[Obsolete("Designer")]
+		public AdvancedPageModel() : this(null!, DummyDatabaseFile.Instance, new ServerManager(DummyDatabaseFile.Instance)) {}
+
+		public AdvancedPageModel(Window window, IDatabaseFile db, ServerManager serverManager) {
+			this.window = window;
+			this.db = db;
 
-		public AdvancedPageModel(Window window, ServerManager serverManager) {
 			ServerConfigurationModel = new ServerConfigurationPanelModel(window, serverManager);
 		}
 
 		public void Initialize() {
 			ServerConfigurationModel.Initialize();
 		}
-		
+
 		public void Dispose() {
 			ServerConfigurationModel.Dispose();
 		}
+
+		public async void VacuumDatabase() {
+			db.Vacuum();
+			await Dialog.ShowOk(window, "Vacuum Database", "Done.");
+		}
 	}
 }

app/Desktop/Main/Pages/AttachmentsPage.axaml

@@ -0,0 +1,54 @@
+<UserControl xmlns="https://github.com/avaloniaui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+             xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
+             xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
+             mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
+             x:Class="DHT.Desktop.Main.Pages.AttachmentsPage">
+
+    <Design.DataContext>
+        <pages:AttachmentsPageModel />
+    </Design.DataContext>
+
+    <UserControl.Styles>
+        <Style Selector="Expander">
+            <Setter Property="Margin" Value="0 5 0 0" />
+        </Style>
+        <Style Selector="DataGridColumnHeader">
+            <Setter Property="FontWeight" Value="Medium" />
+        </Style>
+        <Style Selector="DataGridColumnHeader:nth-child(2)">
+            <Setter Property="HorizontalContentAlignment" Value="Right" />
+        </Style>
+        <Style Selector="DataGridColumnHeader:nth-child(3)">
+            <Setter Property="HorizontalContentAlignment" Value="Right" />
+        </Style>
+        <Style Selector="DataGridCell.right">
+            <Setter Property="HorizontalContentAlignment" Value="Right" />
+        </Style>
+    </UserControl.Styles>
+
+    <StackPanel Orientation="Vertical" Spacing="20">
+        <DockPanel>
+            <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" />
+            <TextBlock Text="{Binding DownloadMessage}" Margin="10 0 0 0" VerticalAlignment="Center" DockPanel.Dock="Left" />
+            <ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" />
+        </DockPanel>
+        <controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !DataContext.IsDownloading, RelativeSource={RelativeSource AncestorType=UserControl}}" />
+        <StackPanel Orientation="Vertical" Spacing="12">
+            <Expander Header="Download Status" IsExpanded="True">
+                <DataGrid Items="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
+                    <DataGrid.Columns>
+                        <DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" />
+                        <DataGridTextColumn Header="Attachments" Binding="{Binding Items, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
+                        <DataGridTextColumn Header="Size" Binding="{Binding Size, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" />
+                    </DataGrid.Columns>
+                </DataGrid>
+            </Expander>
+            <StackPanel Orientation="Horizontal" Spacing="10">
+                <Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding HasFailedDownloads}">Retry Failed Downloads</Button>
+            </StackPanel>
+        </StackPanel>
+    </StackPanel>
+</UserControl>

app/Desktop/Main/Pages/AttachmentsPage.axaml.cs

@@ -0,0 +1,16 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia.Controls;
+using Avalonia.Markup.Xaml;
+
+namespace DHT.Desktop.Main.Pages {
+	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+	public sealed class AttachmentsPage : UserControl {
+		public AttachmentsPage() {
+			InitializeComponent();
+		}
+
+		private void InitializeComponent() {
+			AvaloniaXamlLoader.Load(this);
+		}
+	}
+}

app/Desktop/Main/Pages/AttachmentsPageModel.cs

@@ -0,0 +1,203 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using DHT.Desktop.Common;
+using DHT.Desktop.Main.Controls;
+using DHT.Server.Data;
+using DHT.Server.Data.Aggregations;
+using DHT.Server.Data.Filters;
+using DHT.Server.Database;
+using DHT.Server.Download;
+using DHT.Utils.Models;
+using DHT.Utils.Tasks;
+
+namespace DHT.Desktop.Main.Pages {
+	sealed class AttachmentsPageModel : BaseModel, IDisposable {
+		private static readonly DownloadItemFilter EnqueuedItemFilter = new() {
+			IncludeStatuses = new HashSet<DownloadStatus> {
+				DownloadStatus.Enqueued
+			}
+		};
+
+		private bool isThreadDownloadButtonEnabled = true;
+
+		public string ToggleDownloadButtonText => downloadThread == null ? "Start Downloading" : "Stop Downloading";
+
+		public bool IsToggleDownloadButtonEnabled {
+			get => isThreadDownloadButtonEnabled;
+			set => Change(ref isThreadDownloadButtonEnabled, value);
+		}
+
+		public string DownloadMessage { get; set; } = "";
+		public double DownloadProgress => allItemsCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / allItemsCount.Value;
+
+		public AttachmentFilterPanelModel FilterModel { get; }
+
+		private readonly StatisticsRow statisticsEnqueued = new ("Enqueued");
+		private readonly StatisticsRow statisticsDownloaded = new ("Downloaded");
+		private readonly StatisticsRow statisticsFailed = new ("Failed");
+		private readonly StatisticsRow statisticsSkipped = new ("Skipped");
+
+		public List<StatisticsRow> StatisticsRows {
+			get {
+				return new List<StatisticsRow> {
+					statisticsEnqueued,
+					statisticsDownloaded,
+					statisticsFailed,
+					statisticsSkipped
+				};
+			}
+		}
+
+		public bool IsDownloading => downloadThread != null;
+		public bool HasFailedDownloads => statisticsFailed.Items > 0;
+
+		private readonly IDatabaseFile db;
+		private readonly AsyncValueComputer<DownloadStatusStatistics>.Single downloadStatisticsComputer;
+		private BackgroundDownloadThread? downloadThread;
+
+		private int doneItemsCount;
+		private int? allItemsCount;
+
+		public AttachmentsPageModel() : this(DummyDatabaseFile.Instance) {}
+
+		public AttachmentsPageModel(IDatabaseFile db) {
+			this.db = db;
+			this.FilterModel = new AttachmentFilterPanelModel(db);
+
+			this.downloadStatisticsComputer = AsyncValueComputer<DownloadStatusStatistics>.WithResultProcessor(UpdateStatistics).WithOutdatedResults().BuildWithComputer(db.GetDownloadStatusStatistics);
+			this.downloadStatisticsComputer.Recompute();
+
+			db.Statistics.PropertyChanged += OnDbStatisticsChanged;
+		}
+
+		public void Dispose() {
+			db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
+
+			FilterModel.Dispose();
+			DisposeDownloadThread();
+		}
+
+		private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
+			if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
+				if (IsDownloading) {
+					EnqueueDownloadItems();
+				}
+				else {
+					downloadStatisticsComputer.Recompute();
+				}
+			}
+			else if (e.PropertyName == nameof(DatabaseStatistics.TotalDownloads)) {
+				downloadStatisticsComputer.Recompute();
+			}
+		}
+
+		private void EnqueueDownloadItems() {
+			var filter = FilterModel.CreateFilter();
+			filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
+			db.EnqueueDownloadItems(filter);
+
+			downloadStatisticsComputer.Recompute();
+		}
+
+		private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
+			var hadFailedDownloads = HasFailedDownloads;
+
+			statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
+			statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
+
+			statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
+			statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
+
+			statisticsFailed.Items = statusStatistics.FailedCount;
+			statisticsFailed.Size = statusStatistics.FailedSize;
+
+			statisticsSkipped.Items = statusStatistics.SkippedCount;
+			statisticsSkipped.Size = statusStatistics.SkippedSize;
+
+			OnPropertyChanged(nameof(StatisticsRows));
+
+			if (hadFailedDownloads != HasFailedDownloads) {
+				OnPropertyChanged(nameof(HasFailedDownloads));
+			}
+
+			allItemsCount = doneItemsCount + statisticsEnqueued.Items;
+			UpdateDownloadMessage();
+		}
+
+		private void UpdateDownloadMessage() {
+			DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (allItemsCount?.Format() ?? "?") : "";
+
+			OnPropertyChanged(nameof(DownloadMessage));
+			OnPropertyChanged(nameof(DownloadProgress));
+		}
+
+		private void DownloadThreadOnOnItemFinished(object? sender, DownloadItem e) {
+			++doneItemsCount;
+
+			UpdateDownloadMessage();
+			downloadStatisticsComputer.Recompute();
+		}
+
+		private void DownloadThreadOnOnServerStopped(object? sender, EventArgs e) {
+			downloadStatisticsComputer.Recompute();
+			IsToggleDownloadButtonEnabled = true;
+		}
+
+		public void OnClickToggleDownload() {
+			if (downloadThread == null) {
+				EnqueueDownloadItems();
+				downloadThread = new BackgroundDownloadThread(db);
+				downloadThread.OnItemFinished += DownloadThreadOnOnItemFinished;
+				downloadThread.OnServerStopped += DownloadThreadOnOnServerStopped;
+			}
+			else {
+				IsToggleDownloadButtonEnabled = false;
+				DisposeDownloadThread();
+
+				db.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
+				
+				doneItemsCount = 0;
+				allItemsCount = null;
+				UpdateDownloadMessage();
+			}
+
+			OnPropertyChanged(nameof(ToggleDownloadButtonText));
+			OnPropertyChanged(nameof(IsDownloading));
+		}
+
+		public void OnClickRetryFailedDownloads() {
+			var allExceptFailedFilter = new DownloadItemFilter {
+				IncludeStatuses = new HashSet<DownloadStatus> {
+					DownloadStatus.Enqueued,
+					DownloadStatus.Success
+				}
+			};
+
+			db.RemoveDownloadItems(allExceptFailedFilter, FilterRemovalMode.KeepMatching);
+
+			if (IsDownloading) {
+				EnqueueDownloadItems();
+			}
+		}
+
+		private void DisposeDownloadThread() {
+			if (downloadThread != null) {
+				downloadThread.OnItemFinished -= DownloadThreadOnOnItemFinished;
+				downloadThread.StopThread();
+			}
+
+			downloadThread = null;
+		}
+
+		public sealed class StatisticsRow {
+			public string State { get; }
+			public int Items { get; set; }
+			public ulong? Size { get; set; }
+
+			public StatisticsRow(string state) {
+				State = state;
+			}
+		}
+	}
+}

app/Desktop/Main/Pages/DatabasePage.axaml

@@ -24,6 +24,7 @@
         <WrapPanel>
             <Button Command="{Binding OpenDatabaseFolder}">Open Database Folder</Button>
             <Button Command="{Binding MergeWithDatabase}">Merge with Database(s)...</Button>
+            <Button Command="{Binding ImportLegacyArchive}">Import Legacy Archive(s)...</Button>
         </WrapPanel>
     </StackPanel>
 

app/Desktop/Main/Pages/DatabasePageModel.cs

@@ -1,13 +1,20 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Avalonia.Controls;
+using Avalonia.Threading;
 using DHT.Desktop.Common;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
+using DHT.Desktop.Dialogs.TextBox;
+using DHT.Server.Data;
 using DHT.Server.Database;
+using DHT.Server.Database.Import;
 using DHT.Utils.Logging;
 using DHT.Utils.Models;
 
@@ -56,6 +63,10 @@ namespace DHT.Desktop.Main.Pages {
 			}
 		}
 
+		public void CloseDatabase() {
+			DatabaseClosed?.Invoke(this, EventArgs.Empty);
+		}
+
 		public async void MergeWithDatabase() {
 			var fileDialog = DatabaseGui.NewOpenDatabaseFileDialog();
 			fileDialog.Directory = Path.GetDirectoryName(Db.Path);
@@ -74,10 +85,6 @@ namespace DHT.Desktop.Main.Pages {
 			await progressDialog.ShowDialog(window);
 		}
 
-		public void CloseDatabase() {
-			DatabaseClosed?.Invoke(this, EventArgs.Empty);
-		}
-
 		private static async Task MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
 			int total = paths.Length;
 
@@ -91,7 +98,96 @@ namespace DHT.Desktop.Main.Pages {
 				return DialogResult.YesNo.Yes == upgradeResult;
 			}
 
-			var oldStatistics = target.Statistics.Clone();
+			await PerformImport(target, paths, dialog, callback, "Database Merge", "Database Error", "database file", async path => {
+				SynchronizationContext? prevSyncContext = SynchronizationContext.Current;
+				SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext());
+				IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, CheckCanUpgradeDatabase);
+				SynchronizationContext.SetSynchronizationContext(prevSyncContext);
+
+				if (db == null) {
+					return false;
+				}
+
+				try {
+					target.AddFrom(db);
+					return true;
+				} finally {
+					db.Dispose();
+				}
+			});
+		}
+
+		public async void ImportLegacyArchive() {
+			var fileDialog = new OpenFileDialog {
+				Title = "Open Legacy DHT Archive",
+				Directory = Path.GetDirectoryName(Db.Path),
+				AllowMultiple = true
+			};
+
+			string[]? paths = await fileDialog.ShowAsync(window);
+			if (paths == null || paths.Length == 0) {
+				return;
+			}
+
+			ProgressDialog progressDialog = new ProgressDialog();
+			progressDialog.DataContext = new ProgressDialogModel(async callback => await ImportLegacyArchiveFromPaths(Db, paths, progressDialog, callback)) {
+				Title = "Legacy Archive Import"
+			};
+
+			await progressDialog.ShowDialog(window);
+		}
+
+		private static async Task ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
+			var fakeSnowflake = new FakeSnowflake();
+
+			await PerformImport(target, paths, dialog, callback, "Legacy Archive Import", "Legacy Archive Error", "archive file", async path => {
+				await using var jsonStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+				
+				return await LegacyArchiveImport.Read(jsonStream, target, fakeSnowflake, async servers => {
+					SynchronizationContext? prevSyncContext = SynchronizationContext.Current;
+					SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext());
+					Dictionary<DHT.Server.Data.Server, ulong>? result = await Dispatcher.UIThread.InvokeAsync(() => AskForServerIds(dialog, servers));
+					SynchronizationContext.SetSynchronizationContext(prevSyncContext);
+					return result;
+				});
+			});
+		}
+
+		private static async Task<Dictionary<DHT.Server.Data.Server, ulong>?> AskForServerIds(Window window, DHT.Server.Data.Server[] servers) {
+			static bool IsValidSnowflake(string value) {
+				return string.IsNullOrEmpty(value) || ulong.TryParse(value, out _);
+			}
+			
+			var items = new List<TextBoxItem<DHT.Server.Data.Server>>();
+
+			foreach (var server in servers.OrderBy(static server => server.Type).ThenBy(static server => server.Name)) {
+				items.Add(new TextBoxItem<DHT.Server.Data.Server>(server) {
+					Title = server.Name + " (" + ServerTypes.ToNiceString(server.Type) + ")",
+					ValidityCheck = IsValidSnowflake
+				});
+			}
+
+			var model = new TextBoxDialogModel<DHT.Server.Data.Server>(items) {
+				Title = "Imported Server IDs",
+				Description = "Please fill in the IDs of servers and direct messages. First enable Developer Mode in Discord, then right-click each server or direct message, click 'Copy ID', and paste it into the input field. If a server no longer exists, leave its input field empty to use a random ID."
+			};
+
+			var dialog = new TextBoxDialog { DataContext = model };
+			var result = await dialog.ShowDialog<DialogResult.OkCancel>(window);
+
+			if (result != DialogResult.OkCancel.Ok) {
+				return null;
+			}
+
+			return model.ValidItems
+			            .Where(static item => !string.IsNullOrEmpty(item.Value))
+			            .ToDictionary(static item => item.Item, static item => ulong.Parse(item.Value));
+		}
+
+		private static async Task PerformImport(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback, string neutralDialogTitle, string errorDialogTitle, string itemName, Func<string, Task<bool>> performImport) {
+			int total = paths.Length;
+			var oldStatistics = target.SnapshotStatistics();
+
 			int successful = 0;
 			int finished = 0;
 
@@ -100,56 +196,53 @@ namespace DHT.Desktop.Main.Pages {
 				++finished;
 
 				if (!File.Exists(path)) {
-					await Dialog.ShowOk(dialog, "Database Error", "Database '" + Path.GetFileName(path) + "' no longer exists.");
-					continue;
-				}
-
-				IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, CheckCanUpgradeDatabase);
-				if (db == null) {
+					await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' no longer exists.");
 					continue;
 				}
 
 				try {
-					target.AddFrom(db);
+					if (await performImport(path)) {
+						++successful;
+					}
 				} catch (Exception ex) {
 					Log.Error(ex);
-					await Dialog.ShowOk(dialog, "Database Error", "Database '" + Path.GetFileName(path) + "' could not be merged: " + ex.Message);
-					continue;
-				} finally {
-					db.Dispose();
+					await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' could not be imported: " + ex.Message);
 				}
-
-				++successful;
 			}
 
 			await callback.Update("Done", finished, total);
 
 			if (successful == 0) {
-				await Dialog.ShowOk(dialog, "Database Merge", "Nothing was merged.");
+				await Dialog.ShowOk(dialog, neutralDialogTitle, "Nothing was imported.");
 				return;
 			}
 
-			var newStatistics = target.Statistics;
+			await Dialog.ShowOk(dialog, neutralDialogTitle, GetImportDialogMessage(oldStatistics, target.SnapshotStatistics(), successful, total, itemName));
+		}
+
+		private static string GetImportDialogMessage(DatabaseStatisticsSnapshot oldStatistics, DatabaseStatisticsSnapshot newStatistics, int successfulItems, int totalItems, string itemName) {
 			long newServers = newStatistics.TotalServers - oldStatistics.TotalServers;
 			long newChannels = newStatistics.TotalChannels - oldStatistics.TotalChannels;
+			long newUsers = newStatistics.TotalUsers - oldStatistics.TotalUsers;
 			long newMessages = newStatistics.TotalMessages - oldStatistics.TotalMessages;
 
 			StringBuilder message = new StringBuilder();
 			message.Append("Processed ");
 
-			if (successful == total) {
-				message.Append(successful.Pluralize("database file"));
+			if (successfulItems == totalItems) {
+				message.Append(successfulItems.Pluralize(itemName));
 			}
 			else {
-				message.Append(successful.Format()).Append(" out of ").Append(total.Pluralize("database file"));
+				message.Append(successfulItems.Format()).Append(" out of ").Append(totalItems.Pluralize(itemName));
 			}
 
 			message.Append(" and added:\n\n  \u2022 ");
 			message.Append(newServers.Pluralize("server")).Append("\n  \u2022 ");
 			message.Append(newChannels.Pluralize("channel")).Append("\n  \u2022 ");
+			message.Append(newUsers.Pluralize("user")).Append("\n  \u2022 ");
 			message.Append(newMessages.Pluralize("message"));
 
-			await Dialog.ShowOk(dialog, "Database Merge", message.ToString());
+			return message.ToString();
 		}
 	}
 }

app/Desktop/Main/Pages/DebugPage.axaml

@@ -0,0 +1,45 @@
+<UserControl xmlns="https://github.com/avaloniaui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+             xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
+             mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
+             x:Class="DHT.Desktop.Main.Pages.DebugPage">
+    
+    <Design.DataContext>
+        <pages:DebugPageModel />
+    </Design.DataContext>
+
+    <UserControl.Styles>
+        <Style Selector="TextBox">
+            <Setter Property="FontFamily" Value="Consolas,Courier" />
+            <Setter Property="FontSize" Value="15" />
+        </Style>
+        <Style Selector="WrapPanel > StackPanel">
+            <Setter Property="Orientation" Value="Vertical" />
+            <Setter Property="Margin" Value="0 0 10 10" />
+        </Style>
+    </UserControl.Styles>
+
+    <StackPanel Orientation="Vertical" Spacing="10">
+        <Expander Header="Generate Random Data" IsExpanded="True">
+            <WrapPanel>
+                <StackPanel>
+                    <Label Target="Channels">Channels</Label>
+                    <TextBox x:Name="Channels" Width="100" Text="{Binding GenerateChannels}" />
+                </StackPanel>
+                <StackPanel>
+                    <Label Target="Users">Users</Label>
+                    <TextBox x:Name="Users" Width="100" Text="{Binding GenerateUsers}" />
+                </StackPanel>
+                <StackPanel>
+                    <Label Target="Messages">Messages</Label>
+                    <TextBox x:Name="Messages" Width="100" Text="{Binding GenerateMessages}" />
+                </StackPanel>
+                <StackPanel VerticalAlignment="Bottom">
+                    <Button Command="{Binding OnClickAddRandomDataToDatabase}">Add to Database</Button>
+                </StackPanel>
+            </WrapPanel>
+        </Expander>
+    </StackPanel>
+</UserControl>

app/Desktop/Main/Pages/DebugPage.axaml.cs

@@ -0,0 +1,16 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia.Controls;
+using Avalonia.Markup.Xaml;
+
+namespace DHT.Desktop.Main.Pages {
+	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+	public sealed class DebugPage : UserControl {
+		public DebugPage() {
+			InitializeComponent();
+		}
+
+		private void InitializeComponent() {
+			AvaloniaXamlLoader.Load(this);
+		}
+	}
+}

app/Desktop/Main/Pages/DebugPageModel.cs

@@ -0,0 +1,185 @@
+#if DEBUG
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Threading.Tasks;
+using Avalonia.Controls;
+using DHT.Desktop.Dialogs.Message;
+using DHT.Desktop.Dialogs.Progress;
+using DHT.Server.Data;
+using DHT.Server.Database;
+using DHT.Server.Service;
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Main.Pages {
+	sealed class DebugPageModel : BaseModel {
+		public string GenerateChannels { get; set; } = "0";
+		public string GenerateUsers { get; set; } = "0";
+		public string GenerateMessages { get; set; } = "0";
+
+		private readonly Window window;
+		private readonly IDatabaseFile db;
+
+		[Obsolete("Designer")]
+		public DebugPageModel() : this(null!, DummyDatabaseFile.Instance) {}
+
+		public DebugPageModel(Window window, IDatabaseFile db) {
+			this.window = window;
+			this.db = db;
+		}
+
+		public async void OnClickAddRandomDataToDatabase() {
+			if (!int.TryParse(GenerateChannels, out int channels) || channels < 1) {
+				await Dialog.ShowOk(window, "Generate Random Data", "Amount of channels must be at least 1!");
+				return;
+			}
+
+			if (!int.TryParse(GenerateUsers, out int users) || users < 1) {
+				await Dialog.ShowOk(window, "Generate Random Data", "Amount of users must be at least 1!");
+				return;
+			}
+
+			if (!int.TryParse(GenerateMessages, out int messages) || messages < 1) {
+				await Dialog.ShowOk(window, "Generate Random Data", "Amount of messages must be at least 1!");
+				return;
+			}
+
+			ProgressDialog progressDialog = new ProgressDialog {
+				DataContext = new ProgressDialogModel(async callback => await GenerateRandomData(channels, users, messages, callback)) {
+					Title = "Generating Random Data"
+				}
+			};
+
+			await progressDialog.ShowDialog(window);
+		}
+		
+		private const int BatchSize = 500;
+		
+		private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) {
+			int batchCount = (messageCount + BatchSize - 1) / BatchSize;
+			await callback.Update("Adding messages in batches of " + BatchSize, 0, batchCount);
+
+			var rand = new Random();
+			var server = new DHT.Server.Data.Server {
+				Id = RandomId(rand),
+				Name = RandomName("s"),
+				Type = ServerType.Server
+			};
+
+			var channels = Enumerable.Range(0, channelCount).Select(i => new Channel {
+				Id = RandomId(rand),
+				Server = server.Id,
+				Name = RandomName("c"),
+				ParentId = null,
+				Position = i,
+				Topic = RandomText(rand, 10),
+				Nsfw = rand.Next(4) == 0
+			}).ToArray();
+			
+			var users = Enumerable.Range(0, userCount).Select(_ => new User {
+				Id = RandomId(rand),
+				Name = RandomName("u"),
+				AvatarUrl = null,
+				Discriminator = rand.Next(0, 9999).ToString()
+			}).ToArray();
+			
+			db.AddServer(server);
+			db.AddUsers(users);
+			
+			foreach (var channel in channels) {
+				db.AddChannel(channel);
+			}
+			
+			var now = DateTimeOffset.Now;
+			int batchIndex = 0;
+			
+			while (messageCount > 0) {
+				int hourOffset = batchIndex;
+				
+				var messages = Enumerable.Range(0, Math.Min(messageCount, BatchSize)).Select(i => {
+					DateTimeOffset time = now.AddHours(hourOffset).AddMinutes((i * 60.0) / BatchSize);
+					DateTimeOffset? edit = rand.Next(100) == 0 ? time.AddSeconds(rand.Next(1, 60)) : null;
+
+					var timeMillis = time.ToUnixTimeMilliseconds();
+					var editMillis = edit?.ToUnixTimeMilliseconds();
+					
+					return new Message {
+						Id = (ulong) timeMillis,
+						Sender = RandomBiasedIndex(rand, users).Id,
+						Channel = RandomBiasedIndex(rand, channels).Id,
+						Text = RandomText(rand, 100),
+						Timestamp = timeMillis,
+						EditTimestamp = editMillis,
+						RepliedToId = null,
+						Attachments = ImmutableArray<Attachment>.Empty,
+						Embeds = ImmutableArray<Embed>.Empty,
+						Reactions = ImmutableArray<Reaction>.Empty
+					};
+				}).ToArray();
+				
+				db.AddMessages(messages);
+				
+				messageCount -= BatchSize;
+				await callback.Update("Adding messages in batches of " + BatchSize, ++batchIndex, batchCount);
+			}
+		}
+
+		private static ulong RandomId(Random rand) {
+			ulong h = unchecked((ulong) rand.Next());
+			ulong l = unchecked((ulong) rand.Next());
+			return (h << 32) | l;
+		}
+
+		private static string RandomName(string prefix) {
+			return prefix + "-" + ServerUtils.GenerateRandomToken(5);
+		}
+
+		private static T RandomBiasedIndex<T>(Random rand, T[] options) {
+			return options[(int) Math.Floor(options.Length * rand.NextDouble() * rand.NextDouble())];
+		}
+
+		private static readonly string[] RandomWords = {
+			"apple", "apricot", "artichoke", "arugula", "asparagus", "avocado",
+			"banana", "bean", "beechnut", "beet", "blackberry", "blackcurrant", "blueberry", "boysenberry", "bramble", "broccoli",
+			"cabbage", "cacao", "cantaloupe", "caper", "carambola", "carrot", "cauliflower", "celery", "chard", "cherry", "chokeberry", "citron", "clementine", "coconut", "corn", "crabapple", "cranberry", "cucumber", "currant",
+			"daikon", "date", "dewberry", "durian",
+			"edamame", "eggplant", "elderberry", "endive",
+			"fig",
+			"garlic", "ginger", "gooseberry", "grape", "grapefruit", "guava",
+			"honeysuckle", "horseradish", "huckleberry",
+			"jackfruit", "jicama",
+			"kale", "kiwi", "kohlrabi", "kumquat",
+			"leek", "lemon", "lentil", "lettuce", "lime",
+			"mandarin", "mango", "mushroom", "myrtle",
+			"nectarine", "nut",
+			"olive", "okra", "onion", "orange",
+			"papaya", "parsnip", "pawpaw", "peach", "pear", "pea", "pepper", "persimmon", "pineapple", "plum", "plantain", "pomegranate", "pomelo", "potato", "prune", "pumpkin",
+			"quandong", "quinoa",
+			"radicchio", "radish", "raisin", "raspberry", "redcurrant", "rhubarb", "rutabaga",
+			"spinach", "strawberry", "squash",
+			"tamarind", "tangerine", "tomatillo", "tomato", "turnip",
+			"vanilla",
+			"watercress", "watermelon",
+			"yam",
+			"zucchini"
+		};
+		
+		private static string RandomText(Random rand, int maxWords) {
+			int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3));
+			return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)]));
+		}
+	}
+}
+#else
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Main.Pages {
+	sealed class DebugPageModel : BaseModel {
+		public string GenerateChannels { get; set; } = "0";
+		public string GenerateUsers { get; set; } = "0";
+		public string GenerateMessages { get; set; } = "0";
+
+		public void OnClickAddRandomDataToDatabase() {}
+	}
+}
+#endif

app/Desktop/Main/Pages/ViewerPage.axaml

@@ -13,17 +13,16 @@
 
     <UserControl.Styles>
         <Style Selector="Expander">
-            <Setter Property="Margin" Value="0 25 0 0" />
+            <Setter Property="Margin" Value="0 5 0 0" />
         </Style>
     </UserControl.Styles>
 
-    <StackPanel Orientation="Vertical">
+    <StackPanel Orientation="Vertical" Spacing="20">
         <StackPanel Orientation="Horizontal" VerticalAlignment="Top">
             <Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
             <Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
         </StackPanel>
-        <TextBlock Text="{Binding ExportedMessageText}" Margin="0 20 0 0" />
-        <controls:FilterPanel DataContext="{Binding FilterModel}" />
+        <controls:MessageFilterPanel DataContext="{Binding FilterModel}" />
         <Expander Header="Database Tools">
             <StackPanel Orientation="Vertical" Spacing="10">
                 <StackPanel Orientation="Vertical" Spacing="4">

app/Desktop/Main/Pages/ViewerPageModel.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
 using System.IO;
+using System.Text;
 using System.Threading.Tasks;
 using System.Web;
 using Avalonia.Controls;
@@ -17,8 +19,8 @@ using static DHT.Desktop.Program;
 
 namespace DHT.Desktop.Main.Pages {
 	sealed class ViewerPageModel : BaseModel, IDisposable {
-		public string ExportedMessageText { get; private set; } = "";
-
+		public static readonly ConcurrentBag<string> TemporaryFiles = new ();
+		
 		public bool DatabaseToolFilterModeKeep { get; set; } = true;
 		public bool DatabaseToolFilterModeRemove { get; set; } = false;
 
@@ -29,7 +31,7 @@ namespace DHT.Desktop.Main.Pages {
 			set => Change(ref hasFilters, value);
 		}
 
-		private FilterPanelModel FilterModel { get; }
+		private MessageFilterPanelModel FilterModel { get; }
 
 		private readonly Window window;
 		private readonly IDatabaseFile db;
@@ -41,41 +43,53 @@ namespace DHT.Desktop.Main.Pages {
 			this.window = window;
 			this.db = db;
 
-			FilterModel = new FilterPanelModel(window, db);
+			FilterModel = new MessageFilterPanelModel(window, db, "Will export");
 			FilterModel.FilterPropertyChanged += OnFilterPropertyChanged;
-			db.Statistics.PropertyChanged += OnDbStatisticsChanged;
-			UpdateStatistics();
 		}
 
 		public void Dispose() {
-			db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
 			FilterModel.Dispose();
 		}
 
 		private void OnFilterPropertyChanged(object? sender, PropertyChangedEventArgs e) {
-			UpdateStatistics();
 			HasFilters = FilterModel.HasAnyFilters;
 		}
 
-		private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
-			if (e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
-				UpdateStatistics();
-			}
-		}
+		private async Task WriteViewerFile(string path) {
+			const string ArchiveTag = "/*[ARCHIVE]*/";
 
-		private void UpdateStatistics() {
-			ExportedMessageText = "Will export " + db.CountMessages(FilterModel.CreateFilter()).Format() + " out of " + db.Statistics.TotalMessages.Format() + " message(s).";
-			OnPropertyChanged(nameof(ExportedMessageText));
-		}
-
-		private async Task<string> GenerateViewerContents() {
-			string json = ViewerJsonExport.Generate(db, FilterModel.CreateFilter());
+			string indexFile = await Resources.ReadTextAsync("Viewer/index.html");
+			string viewerTemplate = indexFile.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
+			                                 .Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'));
 			
-			string index = await Resources.ReadTextAsync("Viewer/index.html");
-			string viewer = index.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
-			                     .Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'))
-			                     .Replace("/*[ARCHIVE]*/", HttpUtility.JavaScriptStringEncode(json));
-			return viewer;
+			int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag);
+			int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length;
+
+			string jsonTempFile = path + ".tmp";
+
+			await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) {
+				await ViewerJsonExport.Generate(jsonStream, db, FilterModel.CreateFilter());
+
+				char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)];
+				jsonStream.Position = 0;
+
+				await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
+				await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) {
+					await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]);
+
+					using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) {
+						int readBytes;
+						while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) {
+							string jsonChunk = new string(jsonBuffer, 0, readBytes);
+							await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk));
+						}
+					}
+
+					await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]);
+				}
+			}
+
+			File.Delete(jsonTempFile);
 		}
 
 		public async void OnClickOpenViewer() {
@@ -89,8 +103,10 @@ namespace DHT.Desktop.Main.Pages {
 				fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html");
 			}
 
+			TemporaryFiles.Add(fullPath);
+			
 			Directory.CreateDirectory(rootPath);
-			await File.WriteAllTextAsync(fullPath, await GenerateViewerContents());
+			await WriteViewerFile(fullPath);
 
 			Process.Start(new ProcessStartInfo(fullPath) { UseShellExecute = true });
 		}
@@ -98,7 +114,7 @@ namespace DHT.Desktop.Main.Pages {
 		public async void OnClickSaveViewer() {
 			var dialog = new SaveFileDialog {
 				Title = "Save Viewer",
-				InitialFileName = "archive.html",
+				InitialFileName = Path.GetFileNameWithoutExtension(db.Path) + ".html",
 				Directory = Path.GetDirectoryName(db.Path),
 				Filters = new List<FileDialogFilter> {
 					new() {
@@ -110,7 +126,7 @@ namespace DHT.Desktop.Main.Pages {
 
 			string? path = await dialog;
 			if (!string.IsNullOrEmpty(path)) {
-				await File.WriteAllTextAsync(path, await GenerateViewerContents());
+				await WriteViewerFile(path);
 			}
 		}
 
@@ -119,12 +135,12 @@ namespace DHT.Desktop.Main.Pages {
 
 			if (DatabaseToolFilterModeKeep) {
 				if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
-					db.RemoveMessages(filter, MessageFilterRemovalMode.KeepMatching);
+					db.RemoveMessages(filter, FilterRemovalMode.KeepMatching);
 				}
 			}
 			else if (DatabaseToolFilterModeRemove) {
 				if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
-					db.RemoveMessages(filter, MessageFilterRemovalMode.RemoveMatching);
+					db.RemoveMessages(filter, FilterRemovalMode.RemoveMatching);
 				}
 			}
 		}

app/Desktop/Main/Screens/MainContentScreen.axaml

@@ -73,14 +73,14 @@
     <DockPanel>
         <Border Classes="statusBar" DockPanel.Dock="Bottom">
             <DockPanel>
-                <TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Advanced</TextBlock>
+                <TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Attachments</TextBlock>
                 <controls:StatusBar DataContext="{Binding StatusBarModel}" DockPanel.Dock="Right" />
             </DockPanel>
         </Border>
         <TabControl x:Name="TabControl" TabStripPlacement="Left" DockPanel.Dock="Top">
             <TabControl.ItemsPanel>
                 <ItemsPanelTemplate>
-                    <Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,*,Auto" />
+                    <Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,Auto,*,Auto,Auto" />
                 </ItemsPanelTemplate>
             </TabControl.ItemsPanel>
             <TabItem x:Name="TabDatabase" Header="Database" Classes="first" Grid.Row="0">
@@ -88,21 +88,31 @@
                     <ContentPresenter Content="{Binding DatabasePage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
-            <TabItem x:Name="TabTracking" Header="Tracking" DockPanel.Dock="Top" Grid.Row="1">
+            <TabItem x:Name="TabTracking" Header="Tracking" Grid.Row="1">
                 <ScrollViewer>
                     <ContentPresenter Content="{Binding TrackingPage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
-            <TabItem x:Name="TabViewer" Header="Viewer" DockPanel.Dock="Top" Grid.Row="2">
+            <TabItem x:Name="TabAttachments" Header="Attachments" Grid.Row="2">
+                <ScrollViewer>
+                    <ContentPresenter Content="{Binding AttachmentsPage}" Classes="page" />
+                </ScrollViewer>
+            </TabItem>
+            <TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="3">
                 <ScrollViewer>
                     <ContentPresenter Content="{Binding ViewerPage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
-            <TabItem x:Name="TabAdvanced" Header="Advanced" DockPanel.Dock="Bottom" Grid.Row="4">
+            <TabItem x:Name="TabAdvanced" Header="Advanced" Grid.Row="5">
                 <ScrollViewer>
                     <ContentPresenter Content="{Binding AdvancedPage}" Classes="page" />
                 </ScrollViewer>
             </TabItem>
+            <TabItem x:Name="TabDebug" Header="Debug" Grid.Row="6" IsVisible="{Binding HasDebugPage}">
+                <ScrollViewer>
+                    <ContentPresenter Content="{Binding DebugPage}" Classes="page" />
+                </ScrollViewer>
+            </TabItem>
         </TabControl>
     </DockPanel>
 

app/Desktop/Main/Screens/MainContentScreenModel.cs

@@ -19,12 +19,24 @@ namespace DHT.Desktop.Main.Screens {
 		public TrackingPage TrackingPage { get; }
 		private TrackingPageModel TrackingPageModel { get; }
 
+		public AttachmentsPage AttachmentsPage { get; }
+		private AttachmentsPageModel AttachmentsPageModel { get; }
+
 		public ViewerPage ViewerPage { get; }
 		private ViewerPageModel ViewerPageModel { get; }
 
 		public AdvancedPage AdvancedPage { get; }
 		private AdvancedPageModel AdvancedPageModel { get; }
 
+		public DebugPage? DebugPage { get; }
+
+		#if DEBUG
+		public bool HasDebugPage => true;
+		private DebugPageModel DebugPageModel { get; }
+		#else
+		public bool HasDebugPage => false;
+		#endif
+
 		public StatusBarModel StatusBarModel { get; }
 
 		public event EventHandler? DatabaseClosed {
@@ -35,7 +47,7 @@ namespace DHT.Desktop.Main.Screens {
 				DatabasePageModel.DatabaseClosed -= value;
 			}
 		}
-		
+
 		private readonly Window window;
 		private readonly ServerManager serverManager;
 
@@ -45,26 +57,36 @@ namespace DHT.Desktop.Main.Screens {
 		public MainContentScreenModel(Window window, IDatabaseFile db) {
 			this.window = window;
 			this.serverManager = new ServerManager(db);
-			
+
 			ServerLauncher.ServerManagementExceptionCaught += ServerLauncherOnServerManagementExceptionCaught;
-			
+
 			DatabasePageModel = new DatabasePageModel(window, db);
 			DatabasePage = new DatabasePage { DataContext = DatabasePageModel };
 
 			TrackingPageModel = new TrackingPageModel(window);
 			TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
 
+			AttachmentsPageModel = new AttachmentsPageModel(db);
+			AttachmentsPage = new AttachmentsPage { DataContext = AttachmentsPageModel };
+
 			ViewerPageModel = new ViewerPageModel(window, db);
 			ViewerPage = new ViewerPage { DataContext = ViewerPageModel };
-			
-			AdvancedPageModel = new AdvancedPageModel(window, serverManager);
+
+			AdvancedPageModel = new AdvancedPageModel(window, db, serverManager);
 			AdvancedPage = new AdvancedPage { DataContext = AdvancedPageModel };
 
+			#if DEBUG
+			DebugPageModel = new DebugPageModel(window, db);
+			DebugPage = new DebugPage { DataContext = DebugPageModel };
+			#else
+			DebugPage = null;
+			#endif
+
 			StatusBarModel = new StatusBarModel(db.Statistics);
-			
+
 			AdvancedPageModel.ServerConfigurationModel.ServerStatusChanged += OnServerStatusChanged;
 			DatabaseClosed += OnDatabaseClosed;
-			
+
 			StatusBarModel.CurrentStatus = serverManager.IsRunning ? StatusBarModel.Status.Ready : StatusBarModel.Status.Stopped;
 		}
 
@@ -76,6 +98,7 @@ namespace DHT.Desktop.Main.Screens {
 
 		public void Dispose() {
 			ServerLauncher.ServerManagementExceptionCaught -= ServerLauncherOnServerManagementExceptionCaught;
+			AttachmentsPageModel.Dispose();
 			ViewerPageModel.Dispose();
 			serverManager.Dispose();
 		}

app/Resources/Tracker/scripts/discord.js

@@ -91,11 +91,13 @@ class DISCORD {
 	static getMessageElementProps(ele) {
 		const props = DOM.getReactProps(ele);
 		
-		if (props.children && props.children.length >= 4) {
-			const childProps = props.children[3].props;
-			
-			if ("message" in childProps && "channel" in childProps) {
-				return childProps;
+		if (props.children && props.children.length) {
+			for (let i = 3; i < props.children.length; i++) {
+				const childProps = props.children[i].props;
+				
+				if (childProps && "message" in childProps && "channel" in childProps) {
+					return childProps;
+				}
 			}
 		}
 		
@@ -110,16 +112,20 @@ class DISCORD {
 			const messages = [];
 			
 			for (const ele of this.getMessageElements()) {
-				const props = this.getMessageElementProps(ele);
-				
-				if (props != null) {
-					messages.push(props.message);
+				try {
+					const props = this.getMessageElementProps(ele);
+					
+					if (props != null) {
+						messages.push(props.message);
+					}
+				} catch (e) {
+					console.error("[DHT] Error extracing message data, skipping it.", e, ele, DOM.tryGetReactProps(ele));
 				}
 			}
 			
 			return messages;
 		} catch (e) {
-			console.error(e);
+			console.error("[DHT] Error retrieving messages.", e);
 			return [];
 		}
 	}
@@ -193,7 +199,7 @@ class DISCORD {
 			else if (obj.guild_id) {
 				const server = {
 					"id": obj.guild_id,
-					"name": document.querySelector("nav header > h1").innerText,
+					"name": document.querySelector("nav header h1[class*='name-']").innerText,
 					"type": "SERVER"
 				};
 				

app/Resources/Tracker/scripts/dom.js

@@ -71,4 +71,15 @@ class DOM {
 		key = keys.find(key => key.startsWith("__reactProps$"));
 		return key ? ele[key] : null;
 	}
+	
+	/**
+	 * Returns internal React state object of an element, or null if the retrieval throws.
+	 */
+	static tryGetReactProps(ele) {
+		try {
+			return this.getReactProps(ele);
+		} catch (e) {
+			return null;
+		}
+	}
 }

app/Resources/Tracker/styles/controller.css

@@ -1,13 +1,18 @@
-#app-mount > div[class*="app-"] {
+#app-mount div[class*="app-"] {
   margin-bottom: 48px !important;
 }
 
+#app-mount div[class*="app-"] > div[class*="app-"] {
+  margin-bottom: 0 !important;
+}
+
 #dht-ctrl {
   position: absolute;
   bottom: 0;
   width: 100%;
   height: 48px;
   background-color: #fff;
+  z-index: 1000000;
 }
 
 #dht-ctrl button {

app/Resources/Tracker/styles/settings.css

@@ -7,7 +7,7 @@
   background-color: #000;
   opacity: 0.5;
   display: block;
-  z-index: 1000;
+  z-index: 1000001;
 }
 
 #dht-cfg {
@@ -20,7 +20,7 @@
   margin-top: -131px;
   padding: 8px;
   background-color: #fff;
-  z-index: 1001;
+  z-index: 1000002;
 }
 
 #dht-cfg-note {

app/Resources/Viewer/scripts/discord.js

@@ -1,7 +1,7 @@
 const DISCORD = (function() {
 	const regex = {
 		formatBold: /\*\*([\s\S]+?)\*\*(?!\*)/g,
-		formatItalic: /(.)?\*([\s\S]+?)\*(?!\*)/g,
+		formatItalic: /(.)?([_*])([\s\S]+?)\2(?!\2)/g,
 		formatUnderline: /__([\s\S]+?)__(?!_)/g,
 		formatStrike: /~~([\s\S]+?)~~(?!~)/g,
 		formatCodeInline: /(`+)\s*([\s\S]*?[^`])\s*\1(?!`)/g,
@@ -9,7 +9,7 @@ const DISCORD = (function() {
 		formatUrl: /(\b(?:https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig,
 		formatUrlNoEmbed: /<(\b(?:https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])>/ig,
 		specialEscapedBacktick: /\\`/g,
-		specialEscapedSingle: /\\([*\\])/g,
+		specialEscapedSingle: /\\([*_\\])/g,
 		specialEscapedDouble: /\\__|_\\_|\\_\\_|\\~~|~\\~|\\~\\~/g,
 		specialUnescaped: /([*_~\\])/g,
 		mentionRole: /&lt;@&(\d+?)&gt;/g,
@@ -26,6 +26,7 @@ const DISCORD = (function() {
 	let templateUserAvatar;
 	let templateAttachmentDownload;
 	let templateEmbedImage;
+	let templateEmbedImageWithSize;
 	let templateEmbedRich;
 	let templateEmbedRichNoDescription;
 	let templateEmbedUrl;
@@ -46,8 +47,8 @@ const DISCORD = (function() {
 				.replace(regex.specialEscapedSingle, escapeHtmlMatch)
 				.replace(regex.specialEscapedDouble, full => full.replace(/\\/g, "").replace(/(.)/g, escapeHtmlMatch))
 				.replace(regex.formatBold, "<b>$1</b>")
-				.replace(regex.formatItalic, (full, pre, match) => pre === "\\" ? full : (pre || "") + "<i>" + match + "</i>")
 				.replace(regex.formatUnderline, "<u>$1</u>")
+				.replace(regex.formatItalic, (full, pre, char, match) => pre === "\\" ? full : (pre || "") + "<i>" + match + "</i>")
 				.replace(regex.formatStrike, "<s>$1</s>");
 		}
 		
@@ -64,6 +65,19 @@ const DISCORD = (function() {
 		return "<p>" + processed + "</p>";
 	};
 	
+	const getImageEmbed = function(url, image) {
+		if (!SETTINGS.enableImagePreviews) {
+			return "";
+		}
+		
+		if (image.width && image.height) {
+			return templateEmbedImageWithSize.apply({ url, src: image.url, width: image.width, height: image.height });
+		}
+		else {
+			return templateEmbedImage.apply({ url, src: image.url });
+		}
+	};
+	
 	return {
 		setup() {
 			templateChannelServer = new TEMPLATE([
@@ -114,7 +128,12 @@ const DISCORD = (function() {
 			
 			// noinspection HtmlUnknownTarget
 			templateEmbedImage = new TEMPLATE([
-				"<a href='{url}' class='embed thumbnail'><img src='{src}' alt='(image attachment not found)'></a><br>"
+				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
+			].join(""));
+			
+			// noinspection HtmlUnknownTarget
+			templateEmbedImageWithSize = new TEMPLATE([
+				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' width='{width}' height='{height}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
@@ -145,6 +164,17 @@ const DISCORD = (function() {
 			].join(""));
 		},
 		
+		handleImageLoad(ele) {
+			ele.parentElement.classList.remove("loading");
+		},
+		
+		handleImageLoadError(ele) {
+			// noinspection JSUnusedGlobalSymbols
+			ele.onerror = null;
+			ele.parentElement.classList.remove("loading");
+			ele.setAttribute("alt", "(image attachment not found)");
+		},
+		
 		isImageAttachment(attachment) {
 			const dot = attachment.url.lastIndexOf(".");
 			const ext = dot === -1 ? "" : attachment.url.substring(dot).toLowerCase();
@@ -183,10 +213,10 @@ const DISCORD = (function() {
 							return templateEmbedUnsupported.apply(embed);
 						}
 						else if ("image" in embed && embed.image.url) {
-							return SETTINGS.enableImagePreviews ? templateEmbedImage.apply({ url: embed.url, src: embed.image.url }) : "";
+							return getImageEmbed(embed.url, embed.image);
 						}
 						else if ("thumbnail" in embed && embed.thumbnail.url) {
-							return SETTINGS.enableImagePreviews ? templateEmbedImage.apply({ url: embed.url, src: embed.thumbnail.url }) : "";
+							return getImageEmbed(embed.url, embed.thumbnail);
 						}
 						else if ("title" in embed && "description" in embed) {
 							return templateEmbedRich.apply(embed);

app/Resources/Viewer/styles/messages.css

@@ -107,11 +107,25 @@
 }
 
 .message .thumbnail {
+  position: relative;
   max-width: calc(100% - 20px);
   max-height: 320px;
 }
 
+.message .thumbnail.loading::after {
+  content: "";
+  background: rgba(0, 0, 0, 0.75)
+              url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 300' preserveAspectRatio='xMidYMid'%3E %3Ccircle cx='150' cy='150' fill='none' stroke='%237983f5' stroke-width='8' r='42' stroke-dasharray='198 68'%3E %3CanimateTransform attributeName='transform' type='rotate' repeatCount='indefinite' dur='1.25s' values='0 150 150;360 150 150' keyTimes='0;1' /%3E %3C/circle%3E %3C/svg%3E")
+              no-repeat center center;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}
+
 .message .thumbnail img {
+  width: auto;
   max-width: 100%;
   max-height: 320px;
   border-radius: 3px;

app/Server/Data/Aggregations/DownloadStatusStatistics.cs

@@ -0,0 +1,15 @@
+namespace DHT.Server.Data.Aggregations {
+	public sealed class DownloadStatusStatistics {
+		public int EnqueuedCount { get; internal set; }
+		public ulong EnqueuedSize { get; internal set; }
+
+		public int SuccessfulCount { get; internal set; }
+		public ulong SuccessfulSize { get; internal set; }
+
+		public int FailedCount { get; internal set; }
+		public ulong FailedSize { get; internal set; }
+
+		public int SkippedCount { get; internal set; }
+		public ulong SkippedSize { get; internal set; }
+	}
+}

app/Server/Data/Channel.cs

@@ -1,11 +1,11 @@
 namespace DHT.Server.Data {
 	public readonly struct Channel {
-		public ulong Id { get; internal init; }
-		public ulong Server { get; internal init; }
-		public string Name { get; internal init; }
-		public ulong? ParentId { get; internal init; }
-		public int? Position { get; internal init; }
-		public string? Topic { get; internal init; }
-		public bool? Nsfw { get; internal init; }
+		public ulong Id { get; init; }
+		public ulong Server { get; init; }
+		public string Name { get; init; }
+		public ulong? ParentId { get; init; }
+		public int? Position { get; init; }
+		public string? Topic { get; init; }
+		public bool? Nsfw { get; init; }
 	}
 }

app/Server/Data/Download.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Net;
+
+namespace DHT.Server.Data {
+	public readonly struct Download {
+		internal static Download NewSuccess(string url, byte[] data) {
+			return new Download(url, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
+		}
+
+		internal static Download NewFailure(string url, HttpStatusCode? statusCode, ulong size) {
+			return new Download(url, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
+		}
+
+		public string Url { get; }
+		public DownloadStatus Status { get; }
+		public ulong Size { get; }
+		public byte[]? Data { get; }
+
+		internal Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
+			Url = url;
+			Status = status;
+			Size = size;
+			Data = data;
+		}
+
+		internal Download WithData(byte[] data) {
+			return new Download(Url, Status, Size, data);
+		}
+	}
+}

app/Server/Data/DownloadStatus.cs

@@ -0,0 +1,12 @@
+using System.Net;
+
+namespace DHT.Server.Data {
+	/// <summary>
+	/// Extends <see cref="HttpStatusCode"/> with custom status codes in the range 0-99.
+	/// </summary>
+	public enum DownloadStatus {
+		Enqueued = 0,
+		GenericError = 1,
+		Success = HttpStatusCode.OK
+	}
+}

app/Server/Data/Filters/AttachmentFilter.cs

@@ -0,0 +1,15 @@
+namespace DHT.Server.Data.Filters {
+	public sealed class AttachmentFilter {
+		public ulong? MaxBytes { get; set; } = null;
+
+		public DownloadItemRules? DownloadItemRule { get; set; } = null;
+
+		public bool IsEmpty => MaxBytes == null &&
+		                       DownloadItemRule == null;
+
+		public enum DownloadItemRules {
+			OnlyNotPresent,
+			OnlyPresent
+		}
+	}
+}

app/Server/Data/Filters/DownloadItemFilter.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+
+namespace DHT.Server.Data.Filters {
+	public sealed class DownloadItemFilter {
+		public HashSet<DownloadStatus>? IncludeStatuses { get; set; } = null;
+		public HashSet<DownloadStatus>? ExcludeStatuses { get; set; } = null;
+
+		public bool IsEmpty => IncludeStatuses == null && ExcludeStatuses == null;
+	}
+}

app/Server/Data/Filters/FilterRemovalMode.cs

@@ -1,5 +1,5 @@
 namespace DHT.Server.Data.Filters {
-	public enum MessageFilterRemovalMode {
+	public enum FilterRemovalMode {
 		KeepMatching,
 		RemoveMatching
 	}

app/Server/Data/Filters/MessageFilter.cs

@@ -3,11 +3,17 @@ using System.Collections.Generic;
 
 namespace DHT.Server.Data.Filters {
 	public sealed class MessageFilter {
-		public DateTime? StartDate { get; set; }
-		public DateTime? EndDate { get; set; }
+		public DateTime? StartDate { get; set; } = null;
+		public DateTime? EndDate { get; set; } = null;
 
 		public HashSet<ulong>? ChannelIds { get; set; } = null;
 		public HashSet<ulong>? UserIds { get; set; } = null;
 		public HashSet<ulong>? MessageIds { get; set; } = null;
+
+		public bool IsEmpty => StartDate == null &&
+		                       EndDate == null &&
+		                       ChannelIds == null &&
+		                       UserIds == null &&
+		                       MessageIds == null;
 	}
 }

app/Server/Data/Message.cs

@@ -2,15 +2,15 @@ using System.Collections.Immutable;
 
 namespace DHT.Server.Data {
 	public readonly struct Message {
-		public ulong Id { get; internal init; }
-		public ulong Sender { get; internal init; }
-		public ulong Channel { get; internal init; }
-		public string Text { get; internal init; }
-		public long Timestamp { get; internal init; }
-		public long? EditTimestamp { get; internal init; }
-		public ulong? RepliedToId { get; internal init; }
-		public ImmutableArray<Attachment> Attachments { get; internal init; }
-		public ImmutableArray<Embed> Embeds { get; internal init; }
-		public ImmutableArray<Reaction> Reactions { get; internal init; }
+		public ulong Id { get; init; }
+		public ulong Sender { get; init; }
+		public ulong Channel { get; init; }
+		public string Text { get; init; }
+		public long Timestamp { get; init; }
+		public long? EditTimestamp { get; init; }
+		public ulong? RepliedToId { get; init; }
+		public ImmutableArray<Attachment> Attachments { get; init; }
+		public ImmutableArray<Embed> Embeds { get; init; }
+		public ImmutableArray<Reaction> Reactions { get; init; }
 	}
 }

app/Server/Data/Server.cs

@@ -1,7 +1,7 @@
 namespace DHT.Server.Data {
 	public readonly struct Server {
-		public ulong Id { get; internal init; }
-		public string Name { get; internal init; }
-		public ServerType? Type { get; internal init; }
+		public ulong Id { get; init; }
+		public string Name { get; init; }
+		public ServerType? Type { get; init; }
 	}
 }

app/Server/Data/ServerType.cs

@@ -24,6 +24,15 @@ namespace DHT.Server.Data {
 			};
 		}
 
+		public static string ToNiceString(ServerType? type) {
+			return type switch {
+				ServerType.Server        => "Server",
+				ServerType.Group         => "Group",
+				ServerType.DirectMessage => "DM",
+				_                        => "Unknown"
+			};
+		}
+
 		internal static string ToJsonViewerString(ServerType? type) {
 			return type switch {
 				ServerType.Server        => "server",

app/Server/Data/User.cs

@@ -1,8 +1,8 @@
 namespace DHT.Server.Data {
 	public readonly struct User {
-		public ulong Id { get; internal init; }
-		public string Name { get; internal init; }
-		public string? AvatarUrl { get; internal init; }
-		public string? Discriminator { get; internal init; }
+		public ulong Id { get; init; }
+		public string Name { get; init; }
+		public string? AvatarUrl { get; init; }
+		public string? Discriminator { get; init; }
 	}
 }

app/Server/Database/DatabaseExtensions.cs

@@ -1,16 +1,29 @@
+using System.Collections.Generic;
+using DHT.Server.Data;
+
 namespace DHT.Server.Database {
 	public static class DatabaseExtensions {
 		public static void AddFrom(this IDatabaseFile target, IDatabaseFile source) {
-			foreach (var server in source.GetAllServers()) {
-				target.AddServer(server);
-			}
-
-			foreach (var channel in source.GetAllChannels()) {
-				target.AddChannel(channel);
-			}
-
+			target.AddServers(source.GetAllServers());
+			target.AddChannels(source.GetAllChannels());
 			target.AddUsers(source.GetAllUsers().ToArray());
 			target.AddMessages(source.GetMessages().ToArray());
+			
+			foreach (var download in source.GetDownloadsWithoutData()) {
+				target.AddDownload(download.Status == DownloadStatus.Success ? source.GetDownloadWithData(download) : download);
+			}
+		}
+
+		internal static void AddServers(this IDatabaseFile target, IEnumerable<Data.Server> servers) {
+			foreach (var server in servers) {
+				target.AddServer(server);
+			}
+		}
+
+		internal static void AddChannels(this IDatabaseFile target, IEnumerable<Channel> channels) {
+			foreach (var channel in channels) {
+				target.AddChannel(channel);
+			}
 		}
 	}
 }

app/Server/Database/DatabaseStatistics.cs

@@ -1,11 +1,17 @@
 using DHT.Utils.Models;
 
 namespace DHT.Server.Database {
+	/// <summary>
+	/// A live view of database statistics.
+	/// Some of the totals are computed asynchronously and may not reflect the most recent version of the database, or may not be available at all until computed for the first time.
+	/// </summary>
 	public sealed class DatabaseStatistics : BaseModel {
 		private long totalServers;
 		private long totalChannels;
 		private long totalUsers;
-		private long totalMessages;
+		private long? totalMessages;
+		private long? totalAttachments;
+		private long? totalDownloads;
 
 		public long TotalServers {
 			get => totalServers;
@@ -22,18 +28,19 @@ namespace DHT.Server.Database {
 			internal set => Change(ref totalUsers, value);
 		}
 
-		public long TotalMessages {
+		public long? TotalMessages {
 			get => totalMessages;
 			internal set => Change(ref totalMessages, value);
 		}
 
-		public DatabaseStatistics Clone() {
-			return new DatabaseStatistics {
-				totalServers = totalServers,
-				totalChannels = totalChannels,
-				totalUsers = TotalUsers,
-				totalMessages = totalMessages
-			};
+		public long? TotalAttachments {
+			get => totalAttachments;
+			internal set => Change(ref totalAttachments, value);
+		}
+
+		public long? TotalDownloads {
+			get => totalDownloads;
+			internal set => Change(ref totalDownloads, value);
 		}
 	}
 }

app/Server/Database/DatabaseStatisticsSnapshot.cs

@@ -0,0 +1,11 @@
+namespace DHT.Server.Database {
+	/// <summary>
+	/// A complete snapshot of database statistics at a particular point in time.
+	/// </summary>
+	public readonly struct DatabaseStatisticsSnapshot {
+		public long TotalServers { get; internal init; }
+		public long TotalChannels { get; internal init; }
+		public long TotalUsers { get; internal init; }
+		public long TotalMessages { get; internal init; }
+	}
+}

app/Server/Database/DummyDatabaseFile.cs

@@ -1,6 +1,8 @@
 using System.Collections.Generic;
 using DHT.Server.Data;
+using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
+using DHT.Server.Download;
 
 namespace DHT.Server.Database {
 	public sealed class DummyDatabaseFile : IDatabaseFile {
@@ -11,6 +13,10 @@ namespace DHT.Server.Database {
 
 		private DummyDatabaseFile() {}
 
+		public DatabaseStatisticsSnapshot SnapshotStatistics() {
+			return new();
+		}
+
 		public void AddServer(Data.Server server) {}
 
 		public List<Data.Server> GetAllServers() {
@@ -39,7 +45,39 @@ namespace DHT.Server.Database {
 			return new();
 		}
 
-		public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {}
+		public HashSet<ulong> GetMessageIds(MessageFilter? filter = null) {
+			return new();
+		}
+
+		public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {}
+
+		public int CountAttachments(AttachmentFilter? filter = null) {
+			return new();
+		}
+
+		public List<Data.Download> GetDownloadsWithoutData() {
+			return new();
+		}
+
+		public Data.Download GetDownloadWithData(Data.Download download) {
+			return download;
+		}
+
+		public void AddDownload(Data.Download download) {}
+
+		public void EnqueueDownloadItems(AttachmentFilter? filter = null) {}
+		
+		public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
+			return new();
+		}
+
+		public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {}
+
+		public DownloadStatusStatistics GetDownloadStatusStatistics() {
+			return new();
+		}
+
+		public void Vacuum() {}
 
 		public void Dispose() {}
 	}

app/Server/Database/Export/ViewerJsonExport.cs

@@ -1,6 +1,8 @@
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text.Json;
+using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Utils.Logging;
@@ -9,7 +11,7 @@ namespace DHT.Server.Database.Export {
 	public static class ViewerJsonExport {
 		private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
 
-		public static string Generate(IDatabaseFile db, MessageFilter? filter = null) {
+		public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) {
 			var perf = Log.Start();
 
 			var includedUserIds = new HashSet<ulong>();
@@ -37,17 +39,20 @@ namespace DHT.Server.Database.Export {
 
 			perf.Step("Collect database data");
 
+			var value = new {
+				meta = new { users, userindex, servers, channels },
+				data = GenerateMessageList(includedMessages, userIndices)
+			};
+			
+			perf.Step("Generate value object");
+			
 			var opts = new JsonSerializerOptions();
 			opts.Converters.Add(new ViewerJsonSnowflakeSerializer());
 
-			var json = JsonSerializer.Serialize(new {
-				meta = new { users, userindex, servers, channels },
-				data = GenerateMessageList(includedMessages, userIndices)
-			}, opts);
+			await JsonSerializer.SerializeAsync(stream, value, opts);
 
 			perf.Step("Serialize to JSON");
 			perf.End();
-			return json;
 		}
 
 		private static object GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds, out List<string> userindex, out Dictionary<ulong, object> userIndices) {
@@ -159,8 +164,8 @@ namespace DHT.Server.Database.Export {
 					}
 
 					if (!message.Attachments.IsEmpty) {
-						obj["a"] = message.Attachments.Select(static attachment => new {
-							url = attachment.Url
+						obj["a"] = message.Attachments.Select(static attachment => new Dictionary<string, object> {
+							{ "url", attachment.Url }
 						}).ToArray();
 					}
 

app/Server/Database/IDatabaseFile.cs

@@ -1,12 +1,15 @@
 using System;
 using System.Collections.Generic;
 using DHT.Server.Data;
+using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
+using DHT.Server.Download;
 
 namespace DHT.Server.Database {
 	public interface IDatabaseFile : IDisposable {
 		string Path { get; }
 		DatabaseStatistics Statistics { get; }
+		DatabaseStatisticsSnapshot SnapshotStatistics();
 
 		void AddServer(Data.Server server);
 		List<Data.Server> GetAllServers();
@@ -20,6 +23,20 @@ namespace DHT.Server.Database {
 		void AddMessages(Message[] messages);
 		int CountMessages(MessageFilter? filter = null);
 		List<Message> GetMessages(MessageFilter? filter = null);
-		void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode);
+		HashSet<ulong> GetMessageIds(MessageFilter? filter = null);
+		void RemoveMessages(MessageFilter filter, FilterRemovalMode mode);
+
+		int CountAttachments(AttachmentFilter? filter = null);
+
+		void AddDownload(Data.Download download);
+		List<Data.Download> GetDownloadsWithoutData();
+		Data.Download GetDownloadWithData(Data.Download download);
+		
+		void EnqueueDownloadItems(AttachmentFilter? filter = null);
+		List<DownloadItem> GetEnqueuedDownloadItems(int count);
+		void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);
+		DownloadStatusStatistics GetDownloadStatusStatistics();
+
+		void Vacuum();
 	}
 }

app/Server/Database/Import/FakeSnowflake.cs

@@ -0,0 +1,21 @@
+using System;
+
+namespace DHT.Server.Database.Import {
+	/// <summary>
+	/// https://discord.com/developers/docs/reference#snowflakes
+	/// </summary>
+	public sealed class FakeSnowflake {
+		private const ulong DiscordEpoch = 1420070400000UL;
+			
+		private ulong id;
+
+		public FakeSnowflake() {
+			var unixMillis = (ulong) (DateTime.UtcNow.Subtract(DateTime.UnixEpoch).Ticks / TimeSpan.TicksPerMillisecond);
+			this.id = (unixMillis - DiscordEpoch) << 22;
+		}
+
+		internal ulong Next() {
+			return id++;
+		}
+	}
+}

app/Server/Database/Import/LegacyArchiveImport.cs

@@ -0,0 +1,263 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using DHT.Server.Data;
+using DHT.Utils.Collections;
+using DHT.Utils.Http;
+using DHT.Utils.Logging;
+using Microsoft.AspNetCore.StaticFiles;
+
+namespace DHT.Server.Database.Import {
+	public static class LegacyArchiveImport {
+		private static readonly Log Log = Log.ForType(typeof(LegacyArchiveImport));
+
+		private static readonly FileExtensionContentTypeProvider ContentTypeProvider = new ();
+
+		public static async Task<bool> Read(Stream stream, IDatabaseFile db, FakeSnowflake fakeSnowflake, Func<Data.Server[], Task<Dictionary<Data.Server, ulong>?>> askForServerIds) {
+			var perf = Log.Start();
+			var root = await JsonSerializer.DeserializeAsync<JsonElement>(stream);
+
+			try {
+				var meta = root.RequireObject("meta");
+				var data = root.RequireObject("data");
+
+				perf.Step("Deserialize JSON");
+
+				var users = ReadUserList(meta);
+				var servers = ReadServerList(meta, fakeSnowflake);
+
+				var newServersOnly = new HashSet<Data.Server>(servers);
+				var oldServersById = db.GetAllServers().ToDictionary(static server => server.Id, static server => server);
+
+				var oldChannels = db.GetAllChannels();
+				var oldChannelsById = oldChannels.ToDictionary(static channel => channel.Id, static channel => channel);
+
+				foreach (var (channelId, serverIndex) in ReadChannelToServerIndexMapping(meta, servers)) {
+					if (oldChannelsById.TryGetValue(channelId, out var oldChannel) && oldServersById.TryGetValue(oldChannel.Server, out var oldServer) && newServersOnly.Remove(servers[serverIndex])) {
+						servers[serverIndex] = oldServer;
+					}
+				}
+
+				perf.Step("Read server and user list");
+
+				if (newServersOnly.Count > 0) {
+					var askedServerIds = await askForServerIds(newServersOnly.ToArray());
+					if (askedServerIds == null) {
+						return false;
+					}
+
+					perf.Step("Ask for server IDs");
+
+					for (var i = 0; i < servers.Length; i++) {
+						var server = servers[i];
+						if (askedServerIds.TryGetValue(server, out var serverId)) {
+							servers[i] = new Data.Server {
+								Id = serverId,
+								Name = server.Name,
+								Type = server.Type
+							};
+						}
+					}
+				}
+
+				var channels = ReadChannelList(meta, servers);
+
+				perf.Step("Read channel list");
+
+				var oldMessageIds = db.GetMessageIds();
+				var newMessages = channels.SelectMany(channel => ReadMessages(data, channel, users, fakeSnowflake))
+				                          .Where(message => !oldMessageIds.Contains(message.Id))
+				                          .ToArray();
+
+				perf.Step("Read messages");
+
+				db.AddUsers(users);
+				db.AddServers(servers);
+				db.AddChannels(channels);
+				db.AddMessages(newMessages);
+
+				perf.Step("Import into database");
+			} catch (HttpException e) {
+				throw new JsonException(e.Message);
+			}
+
+			perf.End();
+			return true;
+		}
+
+		private static User[] ReadUserList(JsonElement meta) {
+			const string UsersPath = "meta.users[]";
+
+			static ulong ParseUserIndex(JsonElement element, int index) {
+				return ulong.Parse(element.GetString() ?? throw new JsonException("Expected key 'meta.userindex[" + index + "]' to be a string."));
+			}
+
+			var userindex = meta.RequireArray("userindex", "meta")
+			                    .Select(static (item, index) => (ParseUserIndex(item, index), index))
+			                    .ToDictionary();
+
+			var users = new User[userindex.Count];
+
+			foreach (var item in meta.RequireObject("users", "meta").EnumerateObject()) {
+				var path = UsersPath + "." + item.Name;
+				var userId = ulong.Parse(item.Name);
+				var userObj = item.Value;
+
+				users[userindex[userId]] = new User {
+					Id = userId,
+					Name = userObj.RequireString("name", path),
+					AvatarUrl = userObj.HasKey("avatar") ? userObj.RequireString("avatar", path) : null,
+					Discriminator = userObj.HasKey("tag") ? userObj.RequireString("tag", path) : null
+				};
+			}
+
+			return users;
+		}
+
+		private static Data.Server[] ReadServerList(JsonElement meta, FakeSnowflake fakeSnowflake) {
+			const string ServersPath = "meta.servers[]";
+
+			return meta.RequireArray("servers", "meta").Select(serverObj => new Data.Server {
+				Id = fakeSnowflake.Next(),
+				Name = serverObj.RequireString("name", ServersPath),
+				Type = ServerTypes.FromString(serverObj.RequireString("type", ServersPath))
+			}).ToArray();
+		}
+
+		private const string ChannelsPath = "meta.channels";
+
+		private static Dictionary<ulong, int> ReadChannelToServerIndexMapping(JsonElement meta, Data.Server[] servers) {
+			return meta.RequireObject("channels", "meta").EnumerateObject().Select(item => {
+				var path = ChannelsPath + "." + item.Name;
+				var channelId = ulong.Parse(item.Name);
+				var channelObj = item.Value;
+
+				return (channelId, channelObj.RequireInt("server", path, min: 0, max: servers.Length - 1));
+			}).ToDictionary();
+		}
+
+		private static Channel[] ReadChannelList(JsonElement meta, Data.Server[] servers) {
+			return meta.RequireObject("channels", "meta").EnumerateObject().Select(item => {
+				var path = ChannelsPath + "." + item.Name;
+				var channelId = ulong.Parse(item.Name);
+				var channelObj = item.Value;
+
+				return new Channel {
+					Id = channelId,
+					Server = servers[channelObj.RequireInt("server", path, min: 0, max: servers.Length - 1)].Id,
+					Name = channelObj.RequireString("name", path),
+					Position = channelObj.HasKey("position") ? channelObj.RequireInt("position", path, min: 0) : null,
+					Topic = channelObj.HasKey("topic") ? channelObj.RequireString("topic", path) : null,
+					Nsfw = channelObj.HasKey("nsfw") ? channelObj.RequireBool("nsfw", path) : null
+				};
+			}).ToArray();
+		}
+
+		private static Message[] ReadMessages(JsonElement data, Channel channel, User[] users, FakeSnowflake fakeSnowflake) {
+			const string DataPath = "data";
+
+			var channelId = channel.Id;
+			var channelIdStr = channelId.ToString();
+
+			var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null;
+			if (messagesObj == null) {
+				return Array.Empty<Message>();
+			}
+
+			return messagesObj.Value.EnumerateObject().Select(item => {
+				var path = DataPath + "." + item.Name;
+				var messageId = ulong.Parse(item.Name);
+				var messageObj = item.Value;
+
+				return new Message {
+					Id = messageId,
+					Sender = users[messageObj.RequireInt("u", path, min: 0, max: users.Length - 1)].Id,
+					Channel = channelId,
+					Text = messageObj.HasKey("m") ? messageObj.RequireString("m", path) : string.Empty,
+					Timestamp = messageObj.RequireLong("t", path),
+					EditTimestamp = messageObj.HasKey("te") ? messageObj.RequireLong("te", path) : null,
+					RepliedToId = messageObj.HasKey("r") ? messageObj.RequireSnowflake("r", path) : null,
+					Attachments = messageObj.HasKey("a") ? ReadMessageAttachments(messageObj.RequireArray("a", path), fakeSnowflake, path + ".a[]").ToImmutableArray() : ImmutableArray<Attachment>.Empty,
+					Embeds = messageObj.HasKey("e") ? ReadMessageEmbeds(messageObj.RequireArray("e", path), path + ".e[]").ToImmutableArray() : ImmutableArray<Embed>.Empty,
+					Reactions = messageObj.HasKey("re") ? ReadMessageReactions(messageObj.RequireArray("re", path), path + ".re[]").ToImmutableArray() : ImmutableArray<Reaction>.Empty
+				};
+			}).ToArray();
+		}
+
+		[SuppressMessage("ReSharper", "ConvertToLambdaExpression")]
+		private static IEnumerable<Attachment> ReadMessageAttachments(JsonElement.ArrayEnumerator attachmentsArray, FakeSnowflake fakeSnowflake, string path) {
+			return attachmentsArray.Select(attachmentObj => {
+				string url = attachmentObj.RequireString("url", path);
+				string name = url[(url.LastIndexOf('/') + 1)..];
+				string? type = ContentTypeProvider.TryGetContentType(name, out var contentType) ? contentType : null;
+
+				return new Attachment {
+					Id = fakeSnowflake.Next(),
+					Name = name,
+					Type = type,
+					Url = url,
+					Size = 0 // unknown size
+				};
+			}).DistinctByKeyStable(static attachment => {
+				// Some Discord messages have duplicate attachments with the same id for unknown reasons.
+				return attachment.Id;
+			});
+		}
+
+		private static IEnumerable<Embed> ReadMessageEmbeds(JsonElement.ArrayEnumerator embedsArray, string path) {
+			// Some rich embeds are missing URLs which causes a missing 'url' key.
+			return embedsArray.Where(static embedObj => embedObj.HasKey("url")).Select(embedObj => {
+				string url = embedObj.RequireString("url", path);
+				string type = embedObj.RequireString("type", path);
+
+				var embedJson = new Dictionary<string, object> {
+					{ "url", url },
+					{ "type", type },
+					{ "dht_legacy", true }
+				};
+
+				if (type == "image") {
+					embedJson["image"] = new Dictionary<string, string> {
+						{ "url", url }
+					};
+				}
+				else if (type == "rich") {
+					if (embedObj.HasKey("t")) {
+						embedJson["title"] = embedObj.RequireString("t", path);
+					}
+
+					if (embedObj.HasKey("d")) {
+						embedJson["description"] = embedObj.RequireString("d", path);
+					}
+				}
+
+				return new Embed {
+					Json = JsonSerializer.Serialize(embedJson)
+				};
+			});
+		}
+
+		private static IEnumerable<Reaction> ReadMessageReactions(JsonElement.ArrayEnumerator reactionsArray, string path) {
+			return reactionsArray.Select(reactionObj => {
+				var id = reactionObj.HasKey("id") ? reactionObj.RequireSnowflake("id", path) : (ulong?) null;
+				var name = reactionObj.HasKey("n") ? reactionObj.RequireString("n", path) : null;
+
+				if (id == null && name == null) {
+					throw new JsonException("Expected key '" + path + ".id' and/or '" + path + ".n' to be present.");
+				}
+
+				return new Reaction {
+					EmojiId = id,
+					EmojiName = name,
+					EmojiFlags = reactionObj.HasKey("an") && reactionObj.RequireBool("an", path) ? EmojiFlags.Animated : EmojiFlags.None,
+					Count = reactionObj.RequireInt("c", path, min: 0)
+				};
+			});
+		}
+	}
+}

app/Server/Database/Sqlite/Schema.cs

@@ -6,7 +6,7 @@ using DHT.Utils.Logging;
 
 namespace DHT.Server.Database.Sqlite {
 	sealed class Schema {
-		internal const int Version = 3;
+		internal const int Version = 4;
 
 		private static readonly Log Log = Log.ForType<Schema>();
 
@@ -94,6 +94,7 @@ namespace DHT.Server.Database.Sqlite {
 
 			CreateMessageEditTimestampTable();
 			CreateMessageRepliedToTable();
+			CreateDownloadsTable();
 
 			Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
 			Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
@@ -114,6 +115,14 @@ namespace DHT.Server.Database.Sqlite {
 			        replied_to_id INTEGER NOT NULL)");
 		}
 
+		private void CreateDownloadsTable() {
+			Execute(@"CREATE TABLE downloads (
+                      url TEXT NOT NULL PRIMARY KEY,
+                      status INTEGER NOT NULL,
+                      size INTEGER NOT NULL,
+                      blob BLOB)");
+		}
+
 		private void UpgradeSchemas(int dbVersion) {
 			var perf = Log.Start("from version " + dbVersion);
 
@@ -145,6 +154,11 @@ namespace DHT.Server.Database.Sqlite {
 				perf.Step("Vacuum");
 			}
 
+			if (dbVersion <= 3) {
+				CreateDownloadsTable();
+				perf.Step("Upgrade to version 4");
+			}
+
 			perf.End();
 		}
 	}

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -4,10 +4,13 @@ using System.Collections.Immutable;
 using System.Text;
 using System.Threading.Tasks;
 using DHT.Server.Data;
+using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
 using DHT.Utils.Collections;
 using DHT.Utils.Logging;
+using DHT.Utils.Tasks;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite {
@@ -21,14 +24,19 @@ namespace DHT.Server.Database.Sqlite {
 			};
 
 			var pool = new SqliteConnectionPool(connectionString, DefaultPoolSize);
+			bool wasOpened;
 
 			using (var conn = pool.Take()) {
-				if (!await new Schema(conn).Setup(checkCanUpgradeSchemas)) {
-					return null;
-				}
+				wasOpened = await new Schema(conn).Setup(checkCanUpgradeSchemas);
 			}
 
-			return new SqliteDatabaseFile(path, pool);
+			if (wasOpened) {
+				return new SqliteDatabaseFile(path, pool);
+			}
+			else {
+				pool.Dispose();
+				return null;
+			}
 		}
 
 		public string Path { get; }
@@ -36,24 +44,45 @@ namespace DHT.Server.Database.Sqlite {
 
 		private readonly Log log;
 		private readonly SqliteConnectionPool pool;
+		private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
+		private readonly AsyncValueComputer<long>.Single totalAttachmentsComputer;
+		private readonly AsyncValueComputer<long>.Single totalDownloadsComputer;
 
 		private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
 			this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
 			this.pool = pool;
+			
+			this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
+			this.totalAttachmentsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateAttachmentStatistics).WithOutdatedResults().BuildWithComputer(ComputeAttachmentStatistics);
+			this.totalDownloadsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateDownloadStatistics).WithOutdatedResults().BuildWithComputer(ComputeDownloadStatistics);
+
 			this.Path = path;
 			this.Statistics = new DatabaseStatistics();
 
-			using var conn = pool.Take();
-			UpdateServerStatistics(conn);
-			UpdateChannelStatistics(conn);
-			UpdateUserStatistics(conn);
-			UpdateMessageStatistics(conn);
+			using (var conn = pool.Take()) {
+				UpdateServerStatistics(conn);
+				UpdateChannelStatistics(conn);
+				UpdateUserStatistics(conn);
+			}
+
+			totalMessagesComputer.Recompute();
+			totalAttachmentsComputer.Recompute();
+			totalDownloadsComputer.Recompute();
 		}
 
 		public void Dispose() {
 			pool.Dispose();
 		}
 
+		public DatabaseStatisticsSnapshot SnapshotStatistics() {
+			return new DatabaseStatisticsSnapshot {
+				TotalServers = Statistics.TotalServers,
+				TotalChannels = Statistics.TotalChannels,
+				TotalUsers = Statistics.TotalUsers,
+				TotalMessages = ComputeMessageStatistics()
+			};
+		}
+
 		public void AddServer(Data.Server server) {
 			using var conn = pool.Take();
 			using var cmd = conn.Upsert("servers", new[] {
@@ -186,120 +215,130 @@ namespace DHT.Server.Database.Sqlite {
 				cmd.Set(":message_id", id);
 				cmd.ExecuteNonQuery();
 			}
+			
+			bool addedAttachments = false;
 
-			using var conn = pool.Take();
-			using var tx = conn.BeginTransaction();
+			using (var conn = pool.Take()) {
+				using var tx = conn.BeginTransaction();
 
-			using var messageCmd = conn.Upsert("messages", new[] {
-				("message_id", SqliteType.Integer),
-				("sender_id", SqliteType.Integer),
-				("channel_id", SqliteType.Integer),
-				("text", SqliteType.Text),
-				("timestamp", SqliteType.Integer)
-			});
+				using var messageCmd = conn.Upsert("messages", new[] {
+					("message_id", SqliteType.Integer),
+					("sender_id", SqliteType.Integer),
+					("channel_id", SqliteType.Integer),
+					("text", SqliteType.Text),
+					("timestamp", SqliteType.Integer)
+				});
 
-			using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
-			using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
+				using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
+				using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
 
-			using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
-			using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
-			using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
+				using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
+				using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
+				using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
 
-			using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
-				("message_id", SqliteType.Integer),
-				("edit_timestamp", SqliteType.Integer)
-			});
+				using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
+					("message_id", SqliteType.Integer),
+					("edit_timestamp", SqliteType.Integer)
+				});
 
-			using var repliedToCmd = conn.Insert("replied_to", new [] {
-				("message_id", SqliteType.Integer),
-				("replied_to_id", SqliteType.Integer)
-			});
+				using var repliedToCmd = conn.Insert("replied_to", new [] {
+					("message_id", SqliteType.Integer),
+					("replied_to_id", SqliteType.Integer)
+				});
 
-			using var attachmentCmd = conn.Insert("attachments", new[] {
-				("message_id", SqliteType.Integer),
-				("attachment_id", SqliteType.Integer),
-				("name", SqliteType.Text),
-				("type", SqliteType.Text),
-				("url", SqliteType.Text),
-				("size", SqliteType.Integer)
-			});
+				using var attachmentCmd = conn.Insert("attachments", new[] {
+					("message_id", SqliteType.Integer),
+					("attachment_id", SqliteType.Integer),
+					("name", SqliteType.Text),
+					("type", SqliteType.Text),
+					("url", SqliteType.Text),
+					("size", SqliteType.Integer)
+				});
 
-			using var embedCmd = conn.Insert("embeds", new[] {
-				("message_id", SqliteType.Integer),
-				("json", SqliteType.Text)
-			});
+				using var embedCmd = conn.Insert("embeds", new[] {
+					("message_id", SqliteType.Integer),
+					("json", SqliteType.Text)
+				});
 
-			using var reactionCmd = conn.Insert("reactions", new[] {
-				("message_id", SqliteType.Integer),
-				("emoji_id", SqliteType.Integer),
-				("emoji_name", SqliteType.Text),
-				("emoji_flags", SqliteType.Integer),
-				("count", SqliteType.Integer)
-			});
+				using var reactionCmd = conn.Insert("reactions", new[] {
+					("message_id", SqliteType.Integer),
+					("emoji_id", SqliteType.Integer),
+					("emoji_name", SqliteType.Text),
+					("emoji_flags", SqliteType.Integer),
+					("count", SqliteType.Integer)
+				});
 
-			foreach (var message in messages) {
-				object messageId = message.Id;
+				foreach (var message in messages) {
+					object messageId = message.Id;
 
-				messageCmd.Set(":message_id", messageId);
-				messageCmd.Set(":sender_id", message.Sender);
-				messageCmd.Set(":channel_id", message.Channel);
-				messageCmd.Set(":text", message.Text);
-				messageCmd.Set(":timestamp", message.Timestamp);
-				messageCmd.ExecuteNonQuery();
+					messageCmd.Set(":message_id", messageId);
+					messageCmd.Set(":sender_id", message.Sender);
+					messageCmd.Set(":channel_id", message.Channel);
+					messageCmd.Set(":text", message.Text);
+					messageCmd.Set(":timestamp", message.Timestamp);
+					messageCmd.ExecuteNonQuery();
 
-				ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
-				ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
+					ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
+					ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
 
-				ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
-				ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
-				ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
+					ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
+					ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
+					ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
 
-				if (message.EditTimestamp is {} timestamp) {
-					editTimestampCmd.Set(":message_id", messageId);
-					editTimestampCmd.Set(":edit_timestamp", timestamp);
-					editTimestampCmd.ExecuteNonQuery();
-				}
+					if (message.EditTimestamp is {} timestamp) {
+						editTimestampCmd.Set(":message_id", messageId);
+						editTimestampCmd.Set(":edit_timestamp", timestamp);
+						editTimestampCmd.ExecuteNonQuery();
+					}
 
-				if (message.RepliedToId is {} repliedToId) {
-					repliedToCmd.Set(":message_id", messageId);
-					repliedToCmd.Set(":replied_to_id", repliedToId);
-					repliedToCmd.ExecuteNonQuery();
-				}
+					if (message.RepliedToId is {} repliedToId) {
+						repliedToCmd.Set(":message_id", messageId);
+						repliedToCmd.Set(":replied_to_id", repliedToId);
+						repliedToCmd.ExecuteNonQuery();
+					}
 
-				if (!message.Attachments.IsEmpty) {
-					foreach (var attachment in message.Attachments) {
-						attachmentCmd.Set(":message_id", messageId);
-						attachmentCmd.Set(":attachment_id", attachment.Id);
-						attachmentCmd.Set(":name", attachment.Name);
-						attachmentCmd.Set(":type", attachment.Type);
-						attachmentCmd.Set(":url", attachment.Url);
-						attachmentCmd.Set(":size", attachment.Size);
-						attachmentCmd.ExecuteNonQuery();
+					if (!message.Attachments.IsEmpty) {
+						addedAttachments = true;
+						
+						foreach (var attachment in message.Attachments) {
+							attachmentCmd.Set(":message_id", messageId);
+							attachmentCmd.Set(":attachment_id", attachment.Id);
+							attachmentCmd.Set(":name", attachment.Name);
+							attachmentCmd.Set(":type", attachment.Type);
+							attachmentCmd.Set(":url", attachment.Url);
+							attachmentCmd.Set(":size", attachment.Size);
+							attachmentCmd.ExecuteNonQuery();
+						}
+					}
+
+					if (!message.Embeds.IsEmpty) {
+						foreach (var embed in message.Embeds) {
+							embedCmd.Set(":message_id", messageId);
+							embedCmd.Set(":json", embed.Json);
+							embedCmd.ExecuteNonQuery();
+						}
+					}
+
+					if (!message.Reactions.IsEmpty) {
+						foreach (var reaction in message.Reactions) {
+							reactionCmd.Set(":message_id", messageId);
+							reactionCmd.Set(":emoji_id", reaction.EmojiId);
+							reactionCmd.Set(":emoji_name", reaction.EmojiName);
+							reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
+							reactionCmd.Set(":count", reaction.Count);
+							reactionCmd.ExecuteNonQuery();
+						}
 					}
 				}
 
-				if (!message.Embeds.IsEmpty) {
-					foreach (var embed in message.Embeds) {
-						embedCmd.Set(":message_id", messageId);
-						embedCmd.Set(":json", embed.Json);
-						embedCmd.ExecuteNonQuery();
-					}
-				}
-
-				if (!message.Reactions.IsEmpty) {
-					foreach (var reaction in message.Reactions) {
-						reactionCmd.Set(":message_id", messageId);
-						reactionCmd.Set(":emoji_id", reaction.EmojiId);
-						reactionCmd.Set(":emoji_name", reaction.EmojiName);
-						reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
-						reactionCmd.Set(":count", reaction.Count);
-						reactionCmd.ExecuteNonQuery();
-					}
-				}
+				tx.Commit();
 			}
 
-			tx.Commit();
-			UpdateMessageStatistics(conn);
+			totalMessagesComputer.Recompute();
+
+			if (addedAttachments) {
+				totalAttachmentsComputer.Recompute();
+			}
 		}
 
 		public int CountMessages(MessageFilter? filter = null) {
@@ -347,26 +386,163 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 			return list;
 		}
 
-		public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {
-			var whereClause = filter.GenerateWhereClause(invert: mode == MessageFilterRemovalMode.KeepMatching);
-			if (string.IsNullOrEmpty(whereClause)) {
-				return;
-			}
-
+		public HashSet<ulong> GetMessageIds(MessageFilter? filter = null) {
 			var perf = log.Start();
-
-			// Rider is being stupid...
-			StringBuilder build = new StringBuilder()
-			                      .Append("DELETE ")
-			                      .Append("FROM messages")
-			                      .Append(whereClause);
+			var ids = new HashSet<ulong>();
 
 			using var conn = pool.Take();
-			using var cmd = conn.Command(build.ToString());
-			cmd.ExecuteNonQuery();
+			using var cmd = conn.Command("SELECT message_id FROM messages" + filter.GenerateWhereClause());
+			using var reader = cmd.ExecuteReader();
+
+			while (reader.Read()) {
+				ids.Add(reader.GetUint64(0));
+			}
 
-			UpdateMessageStatistics(conn);
 			perf.End();
+			return ids;
+		}
+
+		public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {
+			var perf = log.Start();
+
+			DeleteFromTable("messages", filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching));
+			totalMessagesComputer.Recompute();
+
+			perf.End();
+		}
+
+		public int CountAttachments(AttachmentFilter? filter = null) {
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT COUNT(DISTINCT url) FROM attachments a" + filter.GenerateWhereClause("a"));
+			using var reader = cmd.ExecuteReader();
+
+			return reader.Read() ? reader.GetInt32(0) : 0;
+		}
+
+		public void AddDownload(Data.Download download) {
+			using var conn = pool.Take();
+			using var cmd = conn.Upsert("downloads", new[] {
+				("url", SqliteType.Text),
+				("status", SqliteType.Integer),
+				("size", SqliteType.Integer),
+				("blob", SqliteType.Blob)
+			});
+
+			cmd.Set(":url", download.Url);
+			cmd.Set(":status", (int) download.Status);
+			cmd.Set(":size", download.Size);
+			cmd.Set(":blob", download.Data);
+			cmd.ExecuteNonQuery();
+			
+			totalDownloadsComputer.Recompute();
+		}
+
+		public List<Data.Download> GetDownloadsWithoutData() {
+			var list = new List<Data.Download>();
+			
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT url, status, size FROM downloads");
+			using var reader = cmd.ExecuteReader();
+			
+			while (reader.Read()) {
+				string url = reader.GetString(0);
+				var status = (DownloadStatus) reader.GetInt32(1);
+				ulong size = reader.GetUint64(2);
+				
+				list.Add(new Data.Download(url, status, size));
+			}
+			
+			return list;
+		}
+
+		public Data.Download GetDownloadWithData(Data.Download download) {
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT blob FROM downloads WHERE url = :url");
+			cmd.AddAndSet(":url", SqliteType.Text, download.Url);
+			
+			using var reader = cmd.ExecuteReader();
+
+			if (reader.Read() && !reader.IsDBNull(0)) {
+				return download.WithData((byte[]) reader["blob"]);
+			}
+			else {
+				return download;
+			}
+		}
+
+		public void EnqueueDownloadItems(AttachmentFilter? filter = null) {
+			using var conn = pool.Take();
+			using var cmd = conn.Command("INSERT INTO downloads (url, status, size) SELECT a.url, :enqueued, MAX(a.size) FROM attachments a" + filter.GenerateWhereClause("a") + " GROUP BY a.url");
+			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+			cmd.ExecuteNonQuery();
+		}
+
+		public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
+			var list = new List<DownloadItem>();
+			
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
+			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+			cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
+			
+			using var reader = cmd.ExecuteReader();
+			
+			while (reader.Read()) {
+				list.Add(new DownloadItem {
+					Url = reader.GetString(0),
+					Size = reader.GetUint64(1)
+				});
+			}
+			
+			return list;
+		}
+
+		public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
+			DeleteFromTable("downloads", filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching));
+			totalDownloadsComputer.Recompute();
+		}
+
+		public DownloadStatusStatistics GetDownloadStatusStatistics() {
+			static void LoadUndownloadedStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
+				using var cmd = conn.Command("SELECT IFNULL(COUNT(size), 0), IFNULL(SUM(size), 0) FROM (SELECT MAX(a.size) size FROM attachments a WHERE a.url NOT IN (SELECT d.url FROM downloads d) GROUP BY a.url)");
+				using var reader = cmd.ExecuteReader();
+
+				if (reader.Read()) {
+					result.SkippedCount = reader.GetInt32(0);
+					result.SkippedSize = reader.GetUint64(1);
+				}
+			}
+
+			static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
+				using var cmd = conn.Command(@"SELECT
+IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
+IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
+IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
+IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
+IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
+IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
+FROM downloads");
+				cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
+				cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
+				
+				using var reader = cmd.ExecuteReader();
+
+				if (reader.Read()) {
+					result.EnqueuedCount = reader.GetInt32(0);
+					result.EnqueuedSize = reader.GetUint64(1);
+					result.SuccessfulCount = reader.GetInt32(2);
+					result.SuccessfulSize = reader.GetUint64(3);
+					result.FailedCount = reader.GetInt32(4);
+					result.FailedSize = reader.GetUint64(5);
+				}
+			}
+
+			var result = new DownloadStatusStatistics();
+
+			using var conn = pool.Take();
+			LoadUndownloadedStatistics(conn, result);
+			LoadSuccessStatistics(conn, result);
+			return result;
 		}
 
 		private MultiDictionary<ulong, Attachment> GetAllAttachments() {
@@ -430,6 +606,25 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 			return dict;
 		}
 
+		private void DeleteFromTable(string table, string whereClause) {
+			// Rider is being stupid...
+			StringBuilder build = new StringBuilder()
+			                      .Append("DELETE ")
+			                      .Append("FROM ")
+			                      .Append(table)
+			                      .Append(whereClause);
+
+			using var conn = pool.Take();
+			using var cmd = conn.Command(build.ToString());
+			cmd.ExecuteNonQuery();
+		}
+
+		public void Vacuum() {
+			using var conn = pool.Take();
+			using var cmd = conn.Command("VACUUM");
+			cmd.ExecuteNonQuery();
+		}
+
 		private void UpdateServerStatistics(ISqliteConnection conn) {
 			Statistics.TotalServers = conn.SelectScalar("SELECT COUNT(*) FROM servers") as long? ?? 0;
 		}
@@ -442,8 +637,31 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 			Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0;
 		}
 
-		private void UpdateMessageStatistics(ISqliteConnection conn) {
-			Statistics.TotalMessages = conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
+		private long ComputeMessageStatistics() {
+			using var conn = pool.Take();
+			return conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
+		}
+
+		private void UpdateMessageStatistics(long totalMessages) {
+			Statistics.TotalMessages = totalMessages;
+		}
+
+		private long ComputeAttachmentStatistics() {
+			using var conn = pool.Take();
+			return conn.SelectScalar("SELECT COUNT(DISTINCT url) FROM attachments") as long? ?? 0L;
+		}
+
+		private void UpdateAttachmentStatistics(long totalAttachments) {
+			Statistics.TotalAttachments = totalAttachments;
+		}
+
+		private long ComputeDownloadStatistics() {
+			using var conn = pool.Take();
+			return conn.SelectScalar("SELECT COUNT(*) FROM downloads") as long? ?? 0L;
+		}
+
+		private void UpdateDownloadStatistics(long totalDownloads) {
+			Statistics.TotalDownloads = totalDownloads;
 		}
 	}
 }

app/Server/Database/Sqlite/SqliteFilters.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using DHT.Server.Data;
+using DHT.Server.Data.Filters;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite {
+	static class SqliteFilters {
+		public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
+			if (filter == null) {
+				return "";
+			}
+
+			var where = new SqliteWhereGenerator(tableAlias, invert);
+
+			if (filter.StartDate != null) {
+				where.AddCondition("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
+			}
+
+			if (filter.EndDate != null) {
+				where.AddCondition("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
+			}
+
+			if (filter.ChannelIds != null) {
+				where.AddCondition("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
+			}
+
+			if (filter.UserIds != null) {
+				where.AddCondition("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
+			}
+
+			if (filter.MessageIds != null) {
+				where.AddCondition("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
+			}
+
+			return where.Generate();
+		}
+
+		public static string GenerateWhereClause(this AttachmentFilter? filter, string? tableAlias = null, bool invert = false) {
+			if (filter == null) {
+				return "";
+			}
+
+			var where = new SqliteWhereGenerator(tableAlias, invert);
+
+			if (filter.MaxBytes != null) {
+				where.AddCondition("size <= " + filter.MaxBytes);
+			}
+
+			if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyNotPresent) {
+				where.AddCondition("url NOT IN (SELECT url FROM downloads)");
+			}
+			else if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyPresent) {
+				where.AddCondition("url IN (SELECT url FROM downloads)");
+			}
+			
+			return where.Generate();
+		}
+
+		public static string GenerateWhereClause(this DownloadItemFilter? filter, string? tableAlias = null, bool invert = false) {
+			if (filter == null) {
+				return "";
+			}
+
+			var where = new SqliteWhereGenerator(tableAlias, invert);
+
+			if (filter.IncludeStatuses != null) {
+				where.AddCondition("status IN (" + filter.IncludeStatuses.In() + ")");
+			}
+
+			if (filter.ExcludeStatuses != null) {
+				where.AddCondition("status NOT IN (" + filter.ExcludeStatuses.In() + ")");
+			}
+
+			return where.Generate();
+		}
+
+		private static string In(this ISet<DownloadStatus> statuses) {
+			return string.Join(",", statuses.Select(static status => (int) status));
+		}
+	}
+}

app/Server/Database/Sqlite/SqliteMessageFilter.cs

@@ -1,50 +0,0 @@
-using System;
-using System.Collections.Generic;
-using DHT.Server.Data.Filters;
-
-namespace DHT.Server.Database.Sqlite {
-	static class SqliteMessageFilter {
-		public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
-			if (filter == null) {
-				return "";
-			}
-
-			if (tableAlias != null) {
-				tableAlias += ".";
-			}
-
-			List<string> conditions = new();
-
-			if (filter.StartDate != null) {
-				conditions.Add(tableAlias + "timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
-			}
-
-			if (filter.EndDate != null) {
-				conditions.Add(tableAlias + "timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
-			}
-
-			if (filter.ChannelIds != null) {
-				conditions.Add(tableAlias + "channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
-			}
-
-			if (filter.UserIds != null) {
-				conditions.Add(tableAlias + "sender_id IN (" + string.Join(",", filter.UserIds) + ")");
-			}
-
-			if (filter.MessageIds != null) {
-				conditions.Add(tableAlias + "message_id IN (" + string.Join(",", filter.MessageIds) + ")");
-			}
-
-			if (conditions.Count == 0) {
-				return "";
-			}
-
-			if (invert) {
-				return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
-			}
-			else {
-				return " WHERE " + string.Join(" AND ", conditions);
-			}
-		}
-	}
-}

app/Server/Database/Sqlite/Utils/SqliteConnectionPool.cs

@@ -13,6 +13,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
 		}
 
 		private readonly object monitor = new ();
+		private readonly Random rand = new ();
 		private volatile bool isDisposed;
 
 		private readonly BlockingCollection<PooledConnection> free = new (new ConcurrentStack<PooledConnection>());
@@ -24,7 +25,14 @@ namespace DHT.Server.Database.Sqlite.Utils {
 			for (int i = 0; i < poolSize; i++) {
 				var conn = new SqliteConnection(connectionString);
 				conn.Open();
-				free.Add(new PooledConnection(this, conn));
+				
+				var pooledConn = new PooledConnection(this, conn);
+				
+				using (var cmd = pooledConn.Command("PRAGMA journal_mode=WAL")) {
+					cmd.ExecuteNonQuery();
+				}
+				
+				free.Add(pooledConn);
 			}
 
 			used = new List<PooledConnection>(poolSize);
@@ -42,7 +50,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
 			while (conn == null) {
 				ThrowIfDisposed();
 				lock (monitor) {
-					if (free.TryTake(out conn, TimeSpan.FromMilliseconds(100))) {
+					if (free.TryTake(out conn, TimeSpan.FromMilliseconds(rand.Next(100, 200)))) {
 						used.Add(conn);
 						break;
 					}

app/Server/Database/Sqlite/Utils/SqliteExtensions.cs

@@ -56,6 +56,10 @@ namespace DHT.Server.Database.Sqlite.Utils {
 			}
 		}
 
+		public static void AddAndSet(this SqliteCommand cmd, string key, SqliteType type, object? value) {
+			cmd.Parameters.Add(key, type).Value = value ?? DBNull.Value;
+		}
+
 		public static void Set(this SqliteCommand cmd, string key, object? value) {
 			cmd.Parameters[key].Value = value ?? DBNull.Value;
 		}

app/Server/Database/Sqlite/Utils/SqliteWhereGenerator.cs

@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+
+namespace DHT.Server.Database.Sqlite.Utils {
+	sealed class SqliteWhereGenerator {
+		private readonly string? tableAlias;
+		private readonly bool invert;
+		private readonly List<string> conditions = new ();
+
+		public SqliteWhereGenerator(string? tableAlias, bool invert) {
+			this.tableAlias = tableAlias;
+			this.invert = invert;
+		}
+
+		public void AddCondition(string condition) {
+			conditions.Add(tableAlias == null ? condition : tableAlias + '.' + condition);
+		}
+
+		public string Generate() {
+			if (conditions.Count == 0) {
+				return "";
+			}
+
+			if (invert) {
+				return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
+			}
+			else {
+				return " WHERE " + string.Join(" AND ", conditions);
+			}
+		}
+	}
+}

app/Server/Download/BackgroundDownloadThread.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net;
+using System.Threading;
+using DHT.Server.Database;
+using DHT.Utils.Logging;
+using DHT.Utils.Models;
+
+namespace DHT.Server.Download {
+	public sealed class BackgroundDownloadThread : BaseModel {
+		private static readonly Log Log = Log.ForType<BackgroundDownloadThread>();
+
+		public event EventHandler<DownloadItem>? OnItemFinished {
+			add => parameters.OnItemFinished += value;
+			remove => parameters.OnItemFinished -= value;
+		}
+		
+		public event EventHandler? OnServerStopped {
+			add => parameters.OnServerStopped += value;
+			remove => parameters.OnServerStopped -= value;
+		}
+
+		private readonly CancellationTokenSource cancellationTokenSource;
+		private readonly ThreadInstance.Parameters parameters;
+
+		public BackgroundDownloadThread(IDatabaseFile db) {
+			this.cancellationTokenSource = new CancellationTokenSource();
+			this.parameters = new ThreadInstance.Parameters(db, cancellationTokenSource);
+
+			var thread = new Thread(new ThreadInstance().Work) {
+				Name = "DHT download thread"
+			};
+
+			thread.Start(parameters);
+		}
+
+		public void StopThread() {
+			try {
+				cancellationTokenSource.Cancel();
+			} catch (ObjectDisposedException) {
+				Log.Warn("Attempted to stop background download thread after the cancellation token has been disposed.");
+			}
+		}
+
+		private sealed class ThreadInstance {
+			private const int QueueSize = 32;
+
+			public sealed class Parameters {
+				public event EventHandler<DownloadItem>? OnItemFinished;
+				public event EventHandler? OnServerStopped;
+
+				public IDatabaseFile Db { get; }
+				public CancellationTokenSource CancellationTokenSource { get; }
+
+				public Parameters(IDatabaseFile db, CancellationTokenSource cancellationTokenSource) {
+					Db = db;
+					CancellationTokenSource = cancellationTokenSource;
+				}
+
+				public void FireOnItemFinished(DownloadItem item) {
+					OnItemFinished?.Invoke(null, item);
+				}
+
+				public void FireOnServerStopped() {
+					OnServerStopped?.Invoke(null, EventArgs.Empty);
+				}
+			}
+
+			private readonly WebClient client = new ();
+
+			public ThreadInstance() {
+				client.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36";
+			}
+
+			public void Work(object? obj) {
+				var parameters = (Parameters) obj!;
+
+				var cancellationTokenSource = parameters.CancellationTokenSource;
+				var cancellationToken = cancellationTokenSource.Token;
+
+				var db = parameters.Db;
+				var queue = new ConcurrentQueue<DownloadItem>();
+
+				cancellationToken.Register(client.CancelAsync);
+
+				try {
+					while (!cancellationToken.IsCancellationRequested) {
+						FillQueue(db, queue, cancellationToken);
+
+						while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var item)) {
+							var url = item.Url;
+							Log.Debug("Downloading " + url + "...");
+
+							try {
+								db.AddDownload(Data.Download.NewSuccess(url, client.DownloadData(url)));
+							} catch (WebException e) {
+								db.AddDownload(Data.Download.NewFailure(url, e.Response is HttpWebResponse response ? response.StatusCode : null, item.Size));
+								Log.Error(e);
+							} finally {
+								parameters.FireOnItemFinished(item);
+							}
+						}
+					}
+				} catch (OperationCanceledException) {
+					//
+				} catch (ObjectDisposedException) {
+					//
+				} finally {
+					cancellationTokenSource.Dispose();
+					parameters.FireOnServerStopped();
+				}
+			}
+
+			private static void FillQueue(IDatabaseFile db, ConcurrentQueue<DownloadItem> queue, CancellationToken cancellationToken) {
+				while (!cancellationToken.IsCancellationRequested && queue.IsEmpty) {
+					var newItems = db.GetEnqueuedDownloadItems(QueueSize);
+					if (newItems.Count == 0) {
+						Thread.Sleep(TimeSpan.FromMilliseconds(50));
+					}
+					else {
+						foreach (var item in newItems) {
+							queue.Enqueue(item);
+						}
+					}
+				}
+			}
+		}
+	}
+}

app/Server/Download/DownloadItem.cs

@@ -0,0 +1,6 @@
+namespace DHT.Server.Download {
+	public readonly struct DownloadItem {
+		public string Url { get; init; }
+		public ulong Size { get; init; }
+	}
+}

app/Server/Endpoints/TrackMessagesEndpoint.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Net;
 using System.Text.Json;
@@ -8,6 +9,7 @@ using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Server.Database;
 using DHT.Server.Service;
+using DHT.Utils.Collections;
 using DHT.Utils.Http;
 using Microsoft.AspNetCore.Http;
 
@@ -53,12 +55,16 @@ namespace DHT.Server.Endpoints {
 			Reactions = json.HasKey("reactions") ? ReadReactions(json.RequireArray("reactions", path + ".reactions"), path + ".reactions[]").ToImmutableArray() : ImmutableArray<Reaction>.Empty
 		};
 
+		[SuppressMessage("ReSharper", "ConvertToLambdaExpression")]
 		private static IEnumerable<Attachment> ReadAttachments(JsonElement.ArrayEnumerator array, string path) => array.Select(ele => new Attachment {
 			Id = ele.RequireSnowflake("id", path),
 			Name = ele.RequireString("name", path),
 			Type = ele.HasKey("type") ? ele.RequireString("type", path) : null,
 			Url = ele.RequireString("url", path),
 			Size = (ulong) ele.RequireLong("size", path)
+		}).DistinctByKeyStable(static attachment => {
+			// Some Discord messages have duplicate attachments with the same id for unknown reasons.
+			return attachment.Id;
 		});
 
 		private static IEnumerable<Embed> ReadEmbeds(JsonElement.ArrayEnumerator array, string path) => array.Select(ele => new Embed {

app/Server/Server.csproj

@@ -20,7 +20,7 @@
     <FrameworkReference Include="Microsoft.AspNetCore.App" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.2" />
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.5" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Utils\Utils.csproj" />

app/Utils/Collections/LinqExtensions.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace DHT.Utils.Collections {
+	public static class LinqExtensions {
+		public static IEnumerable<TItem> DistinctByKeyStable<TItem, TKey>(this IEnumerable<TItem> collection, Func<TItem, TKey> getKeyFromItem) where TKey : IEquatable<TKey> {
+			HashSet<TKey>? seenKeys = null;
+			
+			foreach (var item in collection) {
+				seenKeys ??= new HashSet<TKey>();
+				
+				if (seenKeys.Add(getKeyFromItem(item))) {
+					yield return item;
+				}
+			}
+		}
+
+		public static Dictionary<TKey, TValue> ToDictionary<TKey, TValue>(this IEnumerable<(TKey, TValue)> collection) where TKey : notnull {
+			return collection.ToDictionary(static item => item.Item1, static item => item.Item2);
+		}
+	}
+}

app/Utils/Tasks/AsyncValueComputer.cs

@@ -0,0 +1,126 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace DHT.Utils.Tasks {
+	public sealed class AsyncValueComputer<TValue> {
+		private readonly Action<TValue> resultProcessor;
+		private readonly TaskScheduler resultTaskScheduler;
+		private readonly bool processOutdatedResults;
+
+		private readonly object stateLock = new ();
+
+		private SoftHardCancellationToken? currentCancellationTokenSource;
+		private bool wasHardCancelled = false;
+		
+		private Func<TValue>? currentComputeFunction;
+		private bool hasComputeFunctionChanged = false;
+
+		private AsyncValueComputer(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler, bool processOutdatedResults) {
+			this.resultProcessor = resultProcessor;
+			this.resultTaskScheduler = resultTaskScheduler;
+			this.processOutdatedResults = processOutdatedResults;
+		}
+
+		public void Cancel() {
+			lock (stateLock) {
+				wasHardCancelled = true;
+				currentCancellationTokenSource?.RequestHardCancellation();
+			}
+		}
+
+		public void Compute(Func<TValue> func) {
+			lock (stateLock) {
+				wasHardCancelled = false;
+				
+				if (currentComputeFunction != null) {
+					currentComputeFunction = func;
+					hasComputeFunctionChanged = true;
+					currentCancellationTokenSource?.RequestSoftCancellation();
+				}
+				else {
+					EnqueueComputation(func);
+				}
+			}
+		}
+
+		[SuppressMessage("ReSharper", "MethodSupportsCancellation")]
+		private void EnqueueComputation(Func<TValue> func) {
+			var cancellationTokenSource = new SoftHardCancellationToken();
+
+			currentCancellationTokenSource?.RequestSoftCancellation();
+			currentCancellationTokenSource = cancellationTokenSource;
+			currentComputeFunction = func;
+			hasComputeFunctionChanged = false;
+
+			var task = Task.Run(func);
+
+			task.ContinueWith(t => {
+				if (!cancellationTokenSource.IsCancelled(processOutdatedResults)) {
+					resultProcessor(t.Result);
+				}
+			}, CancellationToken.None, TaskContinuationOptions.NotOnFaulted, resultTaskScheduler);
+
+			task.ContinueWith(_ => {
+				lock (stateLock) {
+					cancellationTokenSource.Dispose();
+
+					if (currentCancellationTokenSource == cancellationTokenSource) {
+						currentCancellationTokenSource = null;
+					}
+
+					if (hasComputeFunctionChanged && !wasHardCancelled) {
+						EnqueueComputation(currentComputeFunction);
+					}
+					else {
+						currentComputeFunction = null;
+						hasComputeFunctionChanged = false;
+					}
+				}
+			});
+		}
+
+		public sealed class Single {
+			private readonly AsyncValueComputer<TValue> baseComputer;
+			private readonly Func<TValue> resultComputer;
+
+			internal Single(AsyncValueComputer<TValue> baseComputer, Func<TValue> resultComputer) {
+				this.baseComputer = baseComputer;
+				this.resultComputer = resultComputer;
+			}
+
+			public void Recompute() {
+				baseComputer.Compute(resultComputer);
+			}
+		}
+
+		public static Builder WithResultProcessor(Action<TValue> resultProcessor, TaskScheduler? scheduler = null) {
+			return new Builder(resultProcessor, scheduler ?? TaskScheduler.FromCurrentSynchronizationContext());
+		}
+
+		public sealed class Builder {
+			private readonly Action<TValue> resultProcessor;
+			private readonly TaskScheduler resultTaskScheduler;
+			private bool processOutdatedResults;
+
+			internal Builder(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler) {
+				this.resultProcessor = resultProcessor;
+				this.resultTaskScheduler = resultTaskScheduler;
+			}
+
+			public Builder WithOutdatedResults() {
+				this.processOutdatedResults = true;
+				return this;
+			}
+
+			public AsyncValueComputer<TValue> Build() {
+				return new AsyncValueComputer<TValue>(resultProcessor, resultTaskScheduler, processOutdatedResults);
+			}
+
+			public Single BuildWithComputer(Func<TValue> resultComputer) {
+				return new Single(Build(), resultComputer);
+			}
+		}
+	}
+}

app/Utils/Tasks/SoftHardCancellationToken.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Threading;
+
+namespace DHT.Utils.Tasks {
+	/// <summary>
+	/// Manages a pair of cancellation tokens that follow these rules:
+	/// <list type="number">
+	/// <item><description>If the soft token is cancelled, the hard token remains uncancelled.</description></item>
+	/// <item><description>If the hard token is cancelled, the soft token is also cancelled.</description></item>
+	/// </list>
+	/// </summary>
+	sealed class SoftHardCancellationToken : IDisposable {
+		private readonly CancellationTokenSource soft;
+		private readonly CancellationTokenSource hard;
+		
+		public SoftHardCancellationToken() {
+			this.soft = new CancellationTokenSource();
+			this.hard = new CancellationTokenSource();
+		}
+
+		public bool IsCancelled(bool onlyHardCancellation) {
+			return (onlyHardCancellation ? hard : soft).IsCancellationRequested;
+		}
+
+		public void RequestSoftCancellation() {
+			soft.Cancel();
+		}
+
+		public void RequestHardCancellation() {
+			soft.Cancel();
+			hard.Cancel();
+		}
+
+		public void Dispose() {
+			soft.Dispose();
+			hard.Dispose();
+		}
+	}
+}

app/Version.cs

@@ -7,6 +7,6 @@ using DHT.Utils;
 
 namespace DHT.Utils {
 	static class Version {
-		public const string Tag = "34.0.0.0";
+		public const string Tag = "37.0.0.0";
 	}
 }

build.py

@@ -1,225 +0,0 @@
-# Python 3
-
-import fileinput
-import glob
-import shutil
-import sys
-import os
-import re
-import distutils.dir_util
-
-VERSION_SHORT = "v.31a"
-VERSION_FULL = VERSION_SHORT + ", released 12 Feb 2022"
-
-EXEC_UGLIFYJS_WIN = "{2}/lib/uglifyjs.cmd --parse bare_returns --compress --mangle toplevel --mangle-props keep_quoted,reserved=[{3}] --output \"{1}\" \"{0}\""
-EXEC_UGLIFYJS_AUTO = "uglifyjs --parse bare_returns --compress --mangle toplevel --mangle-props keep_quoted,reserved=[{3}] --output \"{1}\" \"{0}\""
-
-USE_UGLIFYJS = "--nominify" not in sys.argv
-USE_MINIFICATION = "--nominify" not in sys.argv
-BUILD_WEBSITE = "--website" in sys.argv
-CLIPBOARD_TRACKER = "--copytracker" in sys.argv
-
-WORKING_DIR = os.getcwd()
-
-# UglifyJS Setup
-
-if os.name == "nt":
-    EXEC_UGLIFYJS = EXEC_UGLIFYJS_WIN
-else:
-    EXEC_UGLIFYJS = EXEC_UGLIFYJS_AUTO
-    
-    if USE_UGLIFYJS and shutil.which("uglifyjs") is None:
-        USE_UGLIFYJS = False
-        print("Could not find 'uglifyjs', JS minification will be disabled")
-
-if USE_UGLIFYJS:
-    with open("reserve.txt", "r") as reserved:
-        RESERVED_PROPS = ",".join(line.strip() for line in reserved.readlines())
-
-
-# File Utilities
-
-def combine_files(input_pattern, output_file):
-    is_first_file = True
-    
-    with fileinput.input(sorted(glob.glob(input_pattern))) as stream:
-        for line in stream:
-            if stream.isfirstline():
-                if is_first_file:
-                    is_first_file = False
-                else:
-                    output_file.write("\n")
-            
-            output_file.write(line.replace("{{{version:full}}}", VERSION_FULL))
-
-
-def minify_css(input_file, output_file):
-    if not USE_MINIFICATION:
-        if input_file != output_file:
-            shutil.copyfile(input_file, output_file)
-        
-        return
-    
-    with open(input_file, "r") as fin:
-        css = fin.read()
-    
-    css = re.sub(r"^\s+(.+?):\s*(.+?)(?:\s*(!important))?;\n", r"\1:\2\3;", css, flags = re.M)  # remove spaces after colons
-    css = re.sub(r"\{\n", r"{", css, flags = re.M)  # remove new lines after {
-    css = re.sub(r"\n\}", r"}", css, flags = re.M)  # remove new lines before }
-    css = re.sub(r"\n\n", r"\n", css, flags = re.M)  # remove empty lines
-    css = re.sub(r";\}$", r"}", css, flags = re.M)  # remove last semicolons
-    css = re.sub(r"rgb\((.*?),\s*(.*?),\s*(.*?)\)", r"rgb(\1,\2,\3)", css, flags = re.M)  # remove spaces after commas in rgb()
-    css = re.sub(r"rgba\((.*?),\s*(.*?),\s*(.*?),\s*(.*?)\)", r"rgba(\1,\2,\3,\4)", css, flags = re.M)  # remove spaces after commas in rgba()
-    
-    with open(output_file, "w") as out:
-        out.write(css)
-
-
-# Build System
-
-def build_tracker_html():
-    output_file_raw = "bld/track.js"
-    output_file_html = "bld/track.html"
-    
-    output_file_tmp = "bld/track.tmp.js"
-    input_pattern = "src/tracker/*.js"
-    
-    with open(output_file_raw, "w") as out:
-        if not USE_UGLIFYJS:
-            out.write("(function(){\n")
-        
-        combine_files(input_pattern, out)
-        
-        if not USE_UGLIFYJS:
-            out.write("})()")
-    
-    if USE_UGLIFYJS:
-        os.system(EXEC_UGLIFYJS.format(output_file_raw, output_file_tmp, WORKING_DIR, RESERVED_PROPS))
-        
-        with open(output_file_raw, "w") as out:
-            out.write("javascript:(function(){")
-            
-            with open(output_file_tmp, "r") as minified:
-                out.write(minified.read().replace("\n", " ").replace("\r", ""))
-            
-            out.write("})()")
-        
-        os.remove(output_file_tmp)
-    
-    with open(output_file_raw, "r") as raw:
-        script_contents = raw.read().replace("&", "&amp;").replace('"', "&quot;").replace("'", "&#x27;").replace("<", "&lt;").replace(">", "&gt;")
-    
-    with open(output_file_html, "w") as out:
-        out.write(script_contents)
-
-
-def build_tracker_userscript():
-    output_file = "bld/track.user.js"
-    
-    input_pattern = "src/tracker/*.js"
-    userscript_base = "src/base/track.user.js"
-    
-    with open(userscript_base, "r") as base:
-        userscript_contents = base.read().replace("{{{version}}}", VERSION_SHORT).split("{{{contents}}}")
-    
-    with open(output_file, "w") as out:
-        out.write(userscript_contents[0])
-        combine_files(input_pattern, out)
-        out.write(userscript_contents[1])
-
-
-def build_viewer():
-    output_file = "bld/viewer.html"
-    input_html = "src/viewer/index.html"
-    
-    input_css_pattern = "src/viewer/styles/*.css"
-    tmp_css_file_combined = "bld/viewer.tmp.css"
-    tmp_css_file_minified = "bld/viewer.min.css"
-    
-    with open(tmp_css_file_combined, "w") as out:
-        combine_files(input_css_pattern, out)
-    
-    minify_css(tmp_css_file_combined, tmp_css_file_minified)
-    os.remove(tmp_css_file_combined)
-    
-    input_js_pattern = "src/viewer/scripts/*.js"
-    tmp_js_file_combined = "bld/viewer.tmp.js"
-    tmp_js_file_minified = "bld/viewer.min.js"
-    
-    with open(tmp_js_file_combined, "w") as out:
-        combine_files(input_js_pattern, out)
-    
-    if USE_UGLIFYJS:
-        os.system(EXEC_UGLIFYJS.format(tmp_js_file_combined, tmp_js_file_minified, WORKING_DIR, RESERVED_PROPS))
-    else:
-        shutil.copyfile(tmp_js_file_combined, tmp_js_file_minified)
-    
-    os.remove(tmp_js_file_combined)
-    
-    tokens = {
-        "/*{js}*/": tmp_js_file_minified,
-        "/*{css}*/": tmp_css_file_minified
-    }
-    
-    with open(output_file, "w") as out:
-        with open(input_html, "r") as fin:
-            for line in fin:
-                token = None
-                
-                for token in (token for token in tokens if token in line):
-                    with open(tokens[token], "r") as token_file:
-                        embedded = token_file.read()
-                    
-                    out.write(embedded)
-                    os.remove(tokens[token])
-                
-                if token is None:
-                    out.write(line)
-
-
-def build_website():
-    tracker_file_html = "bld/track.html"
-    tracker_file_userscript = "bld/track.user.js"
-    viewer_file = "bld/viewer.html"
-    web_style_file = "bld/web/style.css"
-    
-    distutils.dir_util.copy_tree("web", "bld/web")
-    index_file = "bld/web/index.php"
-    
-    with open(index_file, "r") as index:
-        index_contents = index.read()
-    
-    with open(index_file, "w") as index:
-        index.write(index_contents.replace("{{{version:web}}}", VERSION_SHORT.replace(" ", "&nbsp;")))
-    
-    shutil.copyfile(tracker_file_html, "bld/web/build/track.html")
-    shutil.copyfile(tracker_file_userscript, "bld/web/build/track.user.js")
-    shutil.copyfile(viewer_file, "bld/web/build/viewer.html")
-    minify_css(web_style_file, web_style_file)
-
-
-# Build Process
-
-os.makedirs("bld", exist_ok = True)
-
-print("Building tracker html...")
-build_tracker_html()
-
-print("Building tracker userscript...")
-build_tracker_userscript()
-
-print("Building viewer...")
-build_viewer()
-
-if BUILD_WEBSITE:
-    print("Building website...")
-    build_website()
-
-if CLIPBOARD_TRACKER:
-    if os.name == "nt":
-        print("Copying to clipboard...")
-        os.system("clip < bld/track.js")
-    else:
-        print("Clipboard is only supported on Windows")
-
-print("Done")

reserve.txt

@@ -1,73 +0,0 @@
-autoscroll
-_autoscroll
-afterFirstMsg
-_afterFirstMsg
-afterSavedMsg
-_afterSavedMsg
-enableImagePreviews
-_enableImagePreviews
-enableFormatting
-_enableFormatting
-enableAnimatedEmoji
-_enableAnimatedEmoji
-enableUserAvatars
-_enableUserAvatars
-DHT_LOADED
-DHT_EMBEDDED
-meta
-data
-users
-userindex
-servers
-channels
-u
-t
-m
-f
-e
-a
-t
-te
-d
-r
-re
-c
-n
-an
-tag
-avatar
-author
-type
-state
-name
-position
-topic
-nsfw
-id
-username
-bot
-discriminator
-timestamp
-content
-editedTimestamp
-mentions
-embeds
-attachments
-title
-description
-reply
-reactions
-emoji
-count
-animated
-ext
-toDate
-memoizedProps
-props
-children
-channel
-messages
-msSaveBlob
-messageReference
-message_id
-guild_id

src/base/track.user.js

@@ -1,50 +0,0 @@
-// ==UserScript==
-// @name         Discord History Tracker
-// @version      {{{version}}}
-// @license      MIT
-// @namespace    https://chylex.com
-// @homepageURL  https://dht.chylex.com/
-// @supportURL   https://github.com/chylex/Discord-History-Tracker/issues
-// @include      https://discord.com/*
-// @run-at       document-idle
-// @grant        none
-// ==/UserScript==
-
-const start = function(){
-
-{{{contents}}}
-
-};
-
-const css = document.createElement("style");
-
-css.innerText = `
-#dht-userscript-trigger { cursor: pointer; margin-top: 5px }
-#dht-userscript-trigger svg { opacity: 0.6 }
-#dht-userscript-trigger:hover svg { opacity: 1 }
-`;
-
-document.head.appendChild(css);
-
-window.setInterval(function(){
-  if (document.getElementById("dht-userscript-trigger")){
-    return;
-  }
-  
-  const help = document.querySelector("section[class^='title'] a[href*='support.discord.com']");
-  
-  if (help){
-    help.insertAdjacentHTML("afterend", `
-<span id="dht-userscript-trigger">
-  <span style="margin: 0 4px" role="button">
-    <svg width="28" height="16" viewBox="0 0 11 6" fill="#fff">
-      <path d="M3.133,2.848c0,0.355 -0.044,0.668 -0.132,0.937c-0.088,0.27 -0.208,0.495 -0.36,0.677c-0.153,0.181 -0.333,0.319 -0.541,0.412c-0.207,0.092 -0.431,0.139 -0.672,0.139l-1.413,0l0,-4.266l1.265,0c0.27,0 0.519,0.042 0.746,0.124c0.227,0.083 0.423,0.21 0.586,0.382c0.164,0.171 0.291,0.389 0.383,0.654c0.092,0.264 0.138,0.578 0.138,0.941Zm-0.739,0c0,-0.248 -0.028,-0.461 -0.083,-0.639c-0.056,-0.177 -0.133,-0.323 -0.232,-0.437c-0.099,-0.114 -0.217,-0.198 -0.355,-0.253c-0.139,-0.054 -0.292,-0.082 -0.459,-0.082l-0.518,0l0,2.886l0.621,0c0.147,0 0.283,-0.032 0.409,-0.094c0.125,-0.063 0.233,-0.156 0.325,-0.28c0.092,-0.124 0.163,-0.278 0.215,-0.462c0.051,-0.184 0.077,-0.397 0.077,-0.639Z"></path>
-      <path d="M5.939,5.013l0,-1.829l-1.523,0l0,1.829l-0.732,0l0,-4.266l0.732,0l0,1.699l1.523,0l0,-1.699l0.733,0l0,4.266l-0.733,0Z"></path>
-      <path d="M8.933,1.437l0,3.576l-0.732,0l0,-3.576l-1.13,0l0,-0.69l2.994,0l0,0.69l-1.132,0Z"></path>
-    </svg>
-  </span>
-</span>`);
-    
-    document.getElementById("dht-userscript-trigger").addEventListener("click", start);
-  }
-}, 200);

src/tracker/discord.js

@@ -1,312 +0,0 @@
-var DISCORD = (function(){
-  var getMessageOuterElement = function(){
-    return DOM.queryReactClass("messagesWrapper");
-  };
-  
-  var getMessageScrollerElement = function(){
-    return getMessageOuterElement().querySelector("[class*='scroller-']");
-  };
-  
-  var getMessageElements = function() {
-    return getMessageOuterElement().querySelectorAll("[class*='message-']");
-  };
-  
-  var getReactProps = function(ele) {
-    var keys = Object.keys(ele || {});
-    var key = keys.find(key => key.startsWith("__reactInternalInstance"));
-    
-    if (key){
-      return ele[key].memoizedProps;
-    }
-    
-    key = keys.find(key => key.startsWith("__reactProps$"));
-    return key ? ele[key] : null;
-  };
-  
-  var getMessageElementProps = function(ele) {
-    const props = getReactProps(ele);
-    
-    if (props.children && props.children.length >= 4) {
-      const childProps = props.children[3].props;
-      
-      if ("message" in childProps && "channel" in childProps) {
-        return childProps;
-      }
-    }
-    
-    return null;
-  };
-  
-  var hasMoreMessages = function() {
-    return document.querySelector("#messagesNavigationDescription + [class^=container]") === null;
-  };
-  
-  var getMessages = function() {
-    try {
-      const messages = [];
-      
-      for (const ele of getMessageElements()) {
-        const props = getMessageElementProps(ele);
-        
-        if (props != null) {
-          messages.push(props.message);
-        }
-      }
-      
-      return messages;
-    } catch (e) {
-      console.error(e);
-      return [];
-    }
-  };
-  
-  return {
-    /**
-     * Calls the provided function with a list of messages whenever the currently loaded messages change,
-     * or with `false` if there are no more messages.
-     */
-    setupMessageCallback: function(callback) {
-      let skipsLeft = 0;
-      let waitForCleanup = false;
-      let hasReachedStart = false;
-      const previousMessages = new Set();
-  
-      const intervalId = window.setInterval(() => {
-        if (skipsLeft > 0) {
-          --skipsLeft;
-          return;
-        }
-      
-        const view = getMessageOuterElement();
-      
-        if (!view) {
-          skipsLeft = 2;
-          return;
-        }
-      
-        const anyMessage = DOM.queryReactClass("message", getMessageOuterElement());
-        const messageCount = anyMessage ? anyMessage.parentElement.children.length : 0;
-      
-        if (messageCount > 300) {
-          if (waitForCleanup) {
-            return;
-          }
-        
-          skipsLeft = 3;
-          waitForCleanup = true;
-        
-          window.setTimeout(() => {
-            const view = getMessageScrollerElement();
-            view.scrollTop = view.scrollHeight / 2;
-          }, 1);
-        }
-        else {
-          waitForCleanup = false;
-        }
-      
-        const messages = getMessages();
-        let hasChanged = false;
-      
-        for (const message of messages) {
-          if (!previousMessages.has(message.id)) {
-            hasChanged = true;
-            break;
-          }
-        }
-      
-        if (!hasChanged) {
-          if (!hasReachedStart && !hasMoreMessages()) {
-            hasReachedStart = true;
-            callback(false);
-          }
-          
-          return;
-        }
-      
-        previousMessages.clear();
-        for (const message of messages) {
-          previousMessages.add(message.id);
-        }
-        
-        hasReachedStart = false;
-        callback(messages);
-      }, 200);
-  
-      window.DHT_ON_UNLOAD.push(() => window.clearInterval(intervalId));
-    },
-    
-    /*
-     * Returns internal React state object of an element.
-     */
-    getReactProps: function(ele){
-      return getReactProps(ele);
-    },
-    
-    /*
-     * Returns an object containing the selected server name, selected channel name and ID, and the object type.
-     * For types DM and GROUP, the server and channel names are identical.
-     * For SERVER type, the channel has to be in view, otherwise Discord unloads it.
-     */
-    getSelectedChannel: function() {
-      try {
-        let obj;
-        
-        for (const ele of getMessageElements()) {
-          const props = getMessageElementProps(ele);
-          
-          if (props != null) {
-            obj = props.channel;
-            break;
-          }
-        }
-        
-        if (!obj) {
-          return null;
-        }
-        
-        var dms = DOM.queryReactClass("privateChannels");
-        
-        if (dms){
-          let name;
-          
-          for (const ele of dms.querySelectorAll("[class*='channel-'] [class*='selected-'] [class^='name-'] *, [class*='channel-'][class*='selected-'] [class^='name-'] *")) {
-            const node = Array.prototype.find.call(ele.childNodes, node => node.nodeType === Node.TEXT_NODE);
-            
-            if (node) {
-              name = node.nodeValue;
-              break;
-            }
-          }
-          
-          if (!name) {
-            return null;
-          }
-          
-          let type;
-          
-          // https://discord.com/developers/docs/resources/channel#channel-object-channel-types
-          switch (obj.type) {
-            case 1: type = "DM"; break;
-            case 3: type = "GROUP"; break;
-            default: return null;
-          }
-          
-          return {
-            "server": name,
-            "channel": name,
-            "id": obj.id,
-            "type": type,
-            "extra": {}
-          };
-        }
-        else if (obj.guild_id) {
-          return {
-            "server": document.querySelector("nav header > h1").innerText,
-            "channel": obj.name,
-            "id": obj.id,
-            "type": "SERVER",
-            "extra": {
-              "position": obj.position,
-              "topic": obj.topic,
-              "nsfw": obj.nsfw
-            }
-          };
-        }
-        else {
-          return null;
-        }
-      } catch(e) {
-        console.error(e);
-        return null;
-      }
-    },
-    
-    /*
-     * Returns an array containing currently loaded messages.
-     */
-    getMessages: function(){
-      return getMessages();
-    },
-    
-    /*
-     * Returns true if the message view is visible.
-     */
-    isInMessageView: () => !!getMessageOuterElement(),
-    
-    /*
-     * Returns true if there are more messages available or if they're still loading.
-     */
-    hasMoreMessages: function(){
-      return hasMoreMessages();
-    },
-    
-    /*
-     * Forces the message view to load older messages by scrolling all the way up.
-     */
-    loadOlderMessages: function(){
-      let view = getMessageScrollerElement();
-      
-      if (view.scrollTop > 0){
-        view.scrollTop = 0;
-      }
-    },
-    
-    /*
-     * Selects the next text channel and returns true, otherwise returns false if there are no more channels.
-     */
-    selectNextTextChannel: function(){
-      var dms = DOM.queryReactClass("privateChannels");
-      
-      if (dms){
-        var currentChannel = DOM.queryReactClass("selected", dms);
-        var nextChannel = currentChannel && currentChannel.nextElementSibling;
-        
-        if (!nextChannel || !nextChannel.getAttribute("class").includes("channel-") || !("href" in nextChannel) || !nextChannel.href.includes("/@me/")){
-          return false;
-        }
-        else{
-          nextChannel.click();
-          nextChannel.scrollIntoView(true);
-          return true;
-        }
-      }
-      else{
-        var channelIconNormal = "M5.88657 21C5.57547 21 5.3399 20.7189 5.39427 20.4126L6.00001 17H2.59511C2.28449 17 2.04905 16.7198 2.10259 16.4138L2.27759 15.4138C2.31946 15.1746 2.52722 15 2.77011 15H6.35001L7.41001 9H4.00511C3.69449 9 3.45905 8.71977 3.51259 8.41381L3.68759 7.41381C3.72946 7.17456 3.93722 7 4.18011 7H7.76001L8.39677 3.41262C8.43914 3.17391 8.64664 3 8.88907 3H9.87344C10.1845 3 10.4201 3.28107 10.3657 3.58738L9.76001 7H15.76L16.3968 3.41262C16.4391 3.17391 16.6466 3 16.8891 3H17.8734C18.1845 3 18.4201 3.28107 18.3657 3.58738L17.76 7H21.1649C21.4755 7 21.711 7.28023 21.6574 7.58619L21.4824 8.58619C21.4406 8.82544 21.2328 9 20.9899 9H17.41L16.35 15H19.7549C20.0655 15 20.301 15.2802 20.2474 15.5862L20.0724 16.5862C20.0306 16.8254 19.8228 17 19.5799 17H16L15.3632 20.5874C15.3209 20.8261 15.1134 21 14.8709 21H13.8866C13.5755 21 13.3399 20.7189 13.3943 20.4126L14 17H8.00001L7.36325 20.5874C7.32088 20.8261 7.11337 21 6.87094 21H5.88657ZM9.41045 9L8.35045 15H14.3504L15.4104 9H9.41045Z";
-        var channelIconSpecial = "M14 8C14 7.44772 13.5523 7 13 7H9.76001L10.3657 3.58738C10.4201 3.28107 10.1845 3 9.87344 3H8.88907C8.64664 3 8.43914 3.17391 8.39677 3.41262L7.76001 7H4.18011C3.93722 7 3.72946 7.17456 3.68759 7.41381L3.51259 8.41381C3.45905 8.71977 3.69449 9 4.00511 9H7.41001L6.35001 15H2.77011C2.52722 15 2.31946 15.1746 2.27759 15.4138L2.10259 16.4138C2.04905 16.7198 2.28449 17 2.59511 17H6.00001L5.39427 20.4126C5.3399 20.7189 5.57547 21 5.88657 21H6.87094C7.11337 21 7.32088 20.8261 7.36325 20.5874L8.00001 17H14L13.3943 20.4126C13.3399 20.7189 13.5755 21 13.8866 21H14.8709C15.1134 21 15.3209 20.8261 15.3632 20.5874L16 17H19.5799C19.8228 17 20.0306 16.8254 20.0724 16.5862L20.2474 15.5862C20.301 15.2802 20.0655 15 19.7549 15H16.35L16.6758 13.1558C16.7823 12.5529 16.3186 12 15.7063 12C15.2286 12 14.8199 12.3429 14.7368 12.8133L14.3504 15H8.35045L9.41045 9H13C13.5523 9 14 8.55228 14 8Z";
-        
-        var isValidChannelClass = cls => cls.includes("wrapper-") && !cls.includes("clickable-");
-        var isValidChannelType = ele => !!ele.querySelector('path[d="' + channelIconNormal + '"]') || !!ele.querySelector('path[d="' + channelIconSpecial + '"]');
-        var isValidChannel = ele => ele.childElementCount > 0 && isValidChannelClass(ele.children[0].className) && isValidChannelType(ele);
-        
-        var channelListEle = document.querySelector("div[class*='sidebar'] > nav[class*='container'] > div[class*='scroller']");
-        
-        if (!channelListEle){
-          return false;
-        }
-        
-        var allChannels = Array.prototype.filter.call(channelListEle.querySelectorAll("[class*='containerDefault']"), isValidChannel);
-        var nextChannel = null;
-        
-        for(var index = 0; index < allChannels.length-1; index++){
-          if (allChannels[index].children[0].className.includes("modeSelected")){
-            nextChannel = allChannels[index+1];
-            break;
-          }
-        }
-        
-        if (nextChannel === null){
-          return false;
-        }
-        
-        const nextChannelLink = nextChannel.querySelector("a[href^='/channels/']");
-        if (!nextChannelLink) {
-          return false;
-        }
-  
-        nextChannelLink.click();
-        nextChannel.scrollIntoView(true);
-        return true;
-      }
-    }
-  };
-})();

src/tracker/dom.js

@@ -1,85 +0,0 @@
-var DOM = (function(){
-  var createElement = (tag, parent, id, html) => {
-    var ele = document.createElement(tag);
-    ele.id = id || "";
-    ele.innerHTML = html || "";
-    parent.appendChild(ele);
-    return ele;
-  };
-  
-  return {
-    /*
-     * Returns a child element by its ID. Parent defaults to the entire document.
-     */
-    id: (id, parent) => (parent || document).getElementById(id),
-    
-    /*
-     * Returns the first child element containing the specified obfuscated class. Parent defaults to the entire document.
-     */
-    queryReactClass: (cls, parent) => (parent || document).querySelector(`[class*="${cls}-"]`),
-    
-    /*
-     * Creates an element, adds it to the DOM, and returns it.
-     */
-    createElement: (tag, parent, id, html) => createElement(tag, parent, id, html),
-    
-    /*
-     * Removes an element from the DOM.
-     */
-    removeElement: (ele) => ele.parentNode.removeChild(ele),
-    
-    /*
-     * Creates a new style element with the specified CSS and returns it.
-     */
-    createStyle: (styles) => createElement("style", document.head, "", styles),
-    
-    /*
-     * Convenience setTimeout function to save space after minification.
-     */
-    setTimer: (callback, timeout) => window.setTimeout(callback, timeout),
-    
-    /*
-     * Convenience addEventListener function to save space after minification.
-     */
-    listen: (ele, event, callback) => ele.addEventListener(event, callback),
-    
-    /*
-     * Utility function to save an object into a cookie.
-     */
-    saveToCookie: (name, obj, expiresInSeconds) => {
-      var expires = new Date(Date.now()+1000*expiresInSeconds).toUTCString();
-      document.cookie = name+"="+encodeURIComponent(JSON.stringify(obj))+";path=/;expires="+expires;
-    },
-    
-    /*
-     * Utility function to load an object from a cookie.
-     */
-    loadFromCookie: (name) => {
-      var value = document.cookie.replace(new RegExp("(?:(?:^|.*;\\s*)"+name+"\\s*\\=\\s*([^;]*).*$)|^.*$"), "$1");
-      return value.length ? JSON.parse(decodeURIComponent(value)) : null;
-    },
-    
-    /*
-     * Triggers a UTF-8 text file download.
-     */
-    downloadTextFile: (fileName, fileContents) => {
-      var blob = new Blob([fileContents], { "type": "octet/stream" });
-      
-      if ("msSaveBlob" in window.navigator){
-        return window.navigator.msSaveBlob(blob, fileName);
-      }
-      
-      var url = window.URL.createObjectURL(blob);
-      
-      var ele = createElement("a", document.body);
-      ele.href = url;
-      ele.download = fileName;
-      ele.style.display = "none";
-      
-      ele.click();
-      
-      document.body.removeChild(ele);
-      window.URL.revokeObjectURL(url);
-    }
-  };
-})();

src/tracker/gui.js

@@ -1,277 +0,0 @@
-var GUI = (function(){
-  var controller;
-  var settings;
-  
-  var updateButtonState = () => {
-    if (STATE.isTracking()){
-      controller.ui.btnUpload.disabled = true;
-      controller.ui.btnSettings.disabled = true;
-      controller.ui.btnReset.disabled = true;
-    }
-    else{
-      controller.ui.btnUpload.disabled = false;
-      controller.ui.btnSettings.disabled = false;
-      controller.ui.btnDownload.disabled = controller.ui.btnReset.disabled = !STATE.hasSavedData();
-    }
-  };
-  
-  var stateChangedEvent = (type, detail) => {
-    if (controller){
-      var force = type === "gui" && detail === "controller";
-      
-      if (type === "data" || force){
-        updateButtonState();
-      }
-      
-      if (type === "tracking" || force){
-        updateButtonState();
-        controller.ui.btnToggleTracking.innerHTML = STATE.isTracking() ? "Pause Tracking" : "Start Tracking";
-      }
-      
-      if (type === "data" || force){
-        var messageCount = 0;
-        var channelCount = 0;
-        
-        if (STATE.hasSavedData()){
-          messageCount = STATE.getSavefile().countMessages();
-          channelCount = STATE.getSavefile().countChannels();
-        }
-        
-        controller.ui.textStatus.innerHTML = [
-          messageCount, " message", (messageCount === 1 ? "" : "s"),
-          " from ",
-          channelCount, " channel", (channelCount === 1 ? "" : "s")
-        ].join("");
-      }
-    }
-    
-    if (settings){
-      var force = type === "gui" && detail === "settings";
-      
-      if (force){
-        settings.ui.cbAutoscroll.checked = SETTINGS.autoscroll;
-        settings.ui.optsAfterFirstMsg[SETTINGS.afterFirstMsg].checked = true;
-        settings.ui.optsAfterSavedMsg[SETTINGS.afterSavedMsg].checked = true;
-      }
-      
-      if (type === "setting" || force){
-        var autoscrollRev = !SETTINGS.autoscroll;
-        
-        // discord polyfills Object.values
-        Object.values(settings.ui.optsAfterFirstMsg).forEach(ele => ele.disabled = autoscrollRev);
-        Object.values(settings.ui.optsAfterSavedMsg).forEach(ele => ele.disabled = autoscrollRev);
-      }
-    }
-  };
-  
-  var registeredEvent = false;
-  
-  var setupStateChanged = function(detail){
-    if (!registeredEvent){
-      STATE.onStateChanged(stateChangedEvent);
-      SETTINGS.onSettingsChanged(stateChangedEvent);
-      registeredEvent = true;
-    }
-    
-    stateChangedEvent("gui", detail);
-  };
-  
-  var root = {
-    showController: function(){
-      controller = {};
-      
-      // styles
-      
-      controller.styles = DOM.createStyle(`
-#app-mount > div[class*="app-"] { margin-bottom: 48px !important; }
-#dht-ctrl { position: absolute; bottom: 0; width: 100%; height: 48px; background-color: #FFF; }
-#dht-ctrl button { height: 32px; margin: 8px 0 8px 8px; font-size: 16px; padding: 0 12px; background-color: #7289DA; color: #FFF; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.75); }
-#dht-ctrl button:disabled { background-color: #7A7A7A; cursor: default; }
-#dht-ctrl-close { margin: 8px 8px 8px 0 !important; float: right; }
-#dht-ctrl p { display: inline-block; margin: 14px 12px; }
-#dht-ctrl input { display: none; }`);
-      
-      // main
-      
-      var btn = (id, title) => "<button id='dht-ctrl-"+id+"'>"+title+"</button>";
-      
-      controller.ele = DOM.createElement("div", document.body, "dht-ctrl", `
-${btn("upload", "Upload &amp; Combine")}
-${btn("settings", "Settings")}
-${btn("track", "")}
-${btn("download", "Download")}
-${btn("reset", "Reset")}
-<p id='dht-ctrl-status'></p>
-<input id='dht-ctrl-upload-input' type='file' multiple>
-${btn("close", "X")}`);
-      
-      // elements
-      
-      controller.ui = {
-        btnUpload: DOM.id("dht-ctrl-upload"),
-        btnSettings: DOM.id("dht-ctrl-settings"),
-        btnToggleTracking: DOM.id("dht-ctrl-track"),
-        btnDownload: DOM.id("dht-ctrl-download"),
-        btnReset: DOM.id("dht-ctrl-reset"),
-        btnClose: DOM.id("dht-ctrl-close"),
-        textStatus: DOM.id("dht-ctrl-status"),
-        inputUpload: DOM.id("dht-ctrl-upload-input")
-      };
-      
-      // events
-      
-      DOM.listen(controller.ui.btnUpload, "click", () => {
-        controller.ui.inputUpload.click();
-      });
-      
-      DOM.listen(controller.ui.btnSettings, "click", () => {
-        root.showSettings();
-      });
-      
-      DOM.listen(controller.ui.btnToggleTracking, "click", () => {
-        STATE.setIsTracking(!STATE.isTracking());
-      });
-      
-      DOM.listen(controller.ui.btnDownload, "click", () => {
-        STATE.downloadSavefile();
-      });
-      
-      DOM.listen(controller.ui.btnReset, "click", () => {
-        STATE.resetState();
-      });
-      
-      DOM.listen(controller.ui.btnClose, "click", () => {
-        root.hideController();
-        window.DHT_ON_UNLOAD.forEach(f => f());
-        window.DHT_LOADED = false;
-      });
-      
-      DOM.listen(controller.ui.inputUpload, "change", () => {
-        Array.prototype.forEach.call(controller.ui.inputUpload.files, file => {
-          var reader = new FileReader();
-          
-          reader.onload = function(){
-            var obj = {};
-
-            try{
-              obj = JSON.parse(reader.result);
-            }catch(e){
-              alert("Could not parse '"+file.name+"', see console for details.");
-              console.error(e);
-              return;
-            }
-            
-            if (SAVEFILE.isValid(obj)){
-              STATE.uploadSavefile(file.name, new SAVEFILE(obj));
-            }
-            else{
-              alert("File '"+file.name+"' has an invalid format.");
-            }
-          };
-          
-          reader.readAsText(file, "UTF-8");
-        });
-
-        controller.ui.inputUpload.value = null;
-      });
-      
-      setupStateChanged("controller");
-    },
-    
-    hideController: function(){
-      if (controller){
-        DOM.removeElement(controller.ele);
-        DOM.removeElement(controller.styles);
-        controller = null;
-      }
-    },
-    
-    showSettings: function(){
-      settings = {};
-      
-      // styles
-      
-      settings.styles = DOM.createStyle(`
-#dht-cfg-overlay { position: absolute; left: 0; top: 0; width: 100%; height: 100%; background-color: #000; opacity: 0.5; display: block; z-index: 1000; }
-#dht-cfg { position: absolute; left: 50%; top: 50%; width: 800px; height: 262px; margin-left: -400px; margin-top: -131px; padding: 8px; background-color: #fff; z-index: 1001; }
-#dht-cfg-note { margin-top: 22px; }
-#dht-cfg sub { color: #666; font-size: 13px; }`);
-      
-      // overlay
-      
-      settings.overlay = DOM.createElement("div", document.body, "dht-cfg-overlay");
-      
-      DOM.listen(settings.overlay, "click", () => {
-        root.hideSettings();
-      });
-      
-      // main
-      
-      var radio = (type, id, label) => "<label><input id='dht-cfg-"+type+"-"+id+"' name='dht-"+type+"' type='radio'> "+label+"</label><br>";
-      
-      settings.ele = DOM.createElement("div", document.body, "dht-cfg", `
-<label><input id='dht-cfg-autoscroll' type='checkbox'> Autoscroll</label><br>
-<br>
-<label>After reaching the first message in channel...</label><br>
-${radio("afm", "nothing", "Do Nothing")}
-${radio("afm", "pause", "Pause Tracking")}
-${radio("afm", "switch", "Switch to Next Channel")}
-<br>
-<label>After reaching a previously saved message...</label><br>
-${radio("asm", "nothing", "Do Nothing")}
-${radio("asm", "pause", "Pause Tracking")}
-${radio("asm", "switch", "Switch to Next Channel")}
-<p id='dht-cfg-note'>
-It is recommended to disable link and image previews to avoid putting unnecessary strain on your browser.<br><br>
-<sub>{{{version:full}}}</sub>
-</p>`);
-      
-      // elements
-      
-      settings.ui = {
-        cbAutoscroll: DOM.id("dht-cfg-autoscroll"),
-        optsAfterFirstMsg: {},
-        optsAfterSavedMsg: {}
-      };
-      
-      settings.ui.optsAfterFirstMsg[CONSTANTS.AUTOSCROLL_ACTION_NOTHING] = DOM.id("dht-cfg-afm-nothing");
-      settings.ui.optsAfterFirstMsg[CONSTANTS.AUTOSCROLL_ACTION_PAUSE] = DOM.id("dht-cfg-afm-pause");
-      settings.ui.optsAfterFirstMsg[CONSTANTS.AUTOSCROLL_ACTION_SWITCH] = DOM.id("dht-cfg-afm-switch");
-      
-      settings.ui.optsAfterSavedMsg[CONSTANTS.AUTOSCROLL_ACTION_NOTHING] = DOM.id("dht-cfg-asm-nothing");
-      settings.ui.optsAfterSavedMsg[CONSTANTS.AUTOSCROLL_ACTION_PAUSE] = DOM.id("dht-cfg-asm-pause");
-      settings.ui.optsAfterSavedMsg[CONSTANTS.AUTOSCROLL_ACTION_SWITCH] = DOM.id("dht-cfg-asm-switch");
-      
-      // events
-      
-      settings.ui.cbAutoscroll.addEventListener("change", () => {
-        SETTINGS.autoscroll = settings.ui.cbAutoscroll.checked;
-      });
-      
-      Object.keys(settings.ui.optsAfterFirstMsg).forEach(key => {
-        DOM.listen(settings.ui.optsAfterFirstMsg[key], "click", () => {
-          SETTINGS.afterFirstMsg = key;
-        });
-      });
-      
-      Object.keys(settings.ui.optsAfterSavedMsg).forEach(key => {
-        DOM.listen(settings.ui.optsAfterSavedMsg[key], "click", () => {
-          SETTINGS.afterSavedMsg = key;
-        });
-      });
-      
-      setupStateChanged("settings");
-    },
-    
-    hideSettings: function(){
-      if (settings){
-        DOM.removeElement(settings.overlay);
-        DOM.removeElement(settings.ele);
-        DOM.removeElement(settings.styles);
-        settings = null;
-      }
-    }
-  };
-  
-  return root;
-})();

src/tracker/savefile.js

@@ -1,349 +0,0 @@
-/*
- * SAVEFILE STRUCTURE
- * ==================
- *
- * {
- *   meta: {
- *     users: {
- *       <discord user id>: {
- *         name: <user name>,
- *         avatar: <user icon>,
- *         tag: <user discriminator> // only present if not a bot
- *       }, ...
- *     },
- *
- *     // the user index is an array of discord user ids,
- *     // these indexes are used in the message objects to save space
- *     userindex: [
- *       <discord user id>, ...
- *     ],
- *
- *     servers: [
- *       {
- *         name: <server name>,
- *         type: <"SERVER"|"GROUP"|DM">
- *       }, ...
- *     ],
- *
- *     channels: {
- *       <discord channel id>: {
- *         server: <server index in the meta.servers array>,
- *         name: <channel name>,
- *         position: <order in channel list>, // only present if server type == SERVER
- *         topic: <channel topic>,            // only present if server type == SERVER
- *         nsfw: <channel NSFW status>        // only present if server type == SERVER
- *       }, ...
- *     }
- *   },
- *
- *   data: {
- *     <discord channel id>: {
- *       <discord message id>: {
- *         u: <user index of the sender>,
- *         t: <message timestamp>,
- *         m: <message content>, // only present if not empty
- *         f: <message flags>,   // only present if edited in which case it equals 1, deprecated (use 'te' instead)
- *         te: <edit timestamp>, // only present if edited
- *         e: [ // omit for no embeds
- *           {
- *             url: <embed url>,
- *             type: <embed type>,
- *             t: <rich embed title>,      // only present if type == rich, and if not empty
- *             d: <rich embed description> // only present if type == rich, and if the embed has a simple description text
- *           }, ...
- *         ],
- *         a: [ // omit for no attachments
- *           {
- *             url: <attachment url>
- *           }, ...
- *         ],
- *         r: <reply message id>, // only present if referencing another message (reply)
- *         re: [ // omit for no reactions
- *           {
- *             c: <react count>
- *             n: <emoji name>,
- *             id: <emoji id>,          // only present for custom emoji
- *             an: <emoji is animated>, // only present for custom animated emoji
- *           }, ...
- *         ]
- *       }, ...
- *     }, ...
- *   }
- * }
- *
- *
- * TEMPORARY OBJECT STRUCTURE
- * ==========================
- *
- * {
- *   userlookup: {
- *     <discord user id>: <user index in the meta.userindex array>
- *   },
- *   channelkeys: Set<channel id>,
- *   messagekeys: Set<message id>,
- *   freshmsgs: Set<message id> // only messages which were newly added to the savefile in the current session
- * }
- */
-
-class SAVEFILE{
-  constructor(parsedObj){
-    var me = this;
-    
-    if (!SAVEFILE.isValid(parsedObj)){
-      parsedObj = {
-        meta: {},
-        data: {}
-      };
-    }
-    
-    me.meta = parsedObj.meta;
-    me.data = parsedObj.data;
-    
-    me.meta.users = me.meta.users || {};
-    me.meta.userindex = me.meta.userindex || [];
-    me.meta.servers = me.meta.servers || [];
-    me.meta.channels = me.meta.channels || {};
-    
-    me.tmp = {
-      userlookup: {},
-      channelkeys: new Set(),
-      messagekeys: new Set(),
-      freshmsgs: new Set()
-    };
-  }
-  
-  static isValid(parsedObj){
-    return parsedObj && typeof parsedObj.meta === "object" && typeof parsedObj.data === "object";
-  }
-  
-  findOrRegisterUser(userId, userName, userDiscriminator, userAvatar){
-    var wasPresent = userId in this.meta.users;
-    var userObj = wasPresent ? this.meta.users[userId] : {};
-    
-    userObj.name = userName;
-    
-    if (userDiscriminator){
-      userObj.tag = userDiscriminator;
-    }
-    
-    if (userAvatar){
-      userObj.avatar = userAvatar;
-    }
-    
-    if (!wasPresent){
-      this.meta.users[userId] = userObj;
-      this.meta.userindex.push(userId);
-      return this.tmp.userlookup[userId] = this.meta.userindex.length-1;
-    }
-    else if (!(userId in this.tmp.userlookup)){
-      return this.tmp.userlookup[userId] = this.meta.userindex.findIndex(id => id == userId);
-    }
-    else{
-      return this.tmp.userlookup[userId];
-    }
-  }
-  
-  findOrRegisterServer(serverName, serverType){
-    var index = this.meta.servers.findIndex(server => server.name === serverName && server.type === serverType);
-    
-    if (index === -1){
-      this.meta.servers.push({
-        "name": serverName,
-        "type": serverType
-      });
-      
-      return this.meta.servers.length-1;
-    }
-    else{
-      return index;
-    }
-  }
-  
-  tryRegisterChannel(serverIndex, channelId, channelName, extraInfo){
-    if (!this.meta.servers[serverIndex]){
-      return undefined;
-    }
-    
-    var wasPresent = channelId in this.meta.channels;
-    var channelObj = wasPresent ? this.meta.channels[channelId] : { "server": serverIndex };
-    
-    channelObj.name = channelName;
-    
-    if (extraInfo.position){
-      channelObj.position = extraInfo.position;
-    }
-    
-    if (extraInfo.topic){
-      channelObj.topic = extraInfo.topic;
-    }
-    
-    if (extraInfo.nsfw){
-      channelObj.nsfw = extraInfo.nsfw;
-    }
-    
-    if (wasPresent){
-      return false;
-    }
-    else{
-      this.meta.channels[channelId] = channelObj;
-      this.tmp.channelkeys.add(channelId);
-      return true;
-    }
-  }
-  
-  addMessage(channelId, messageId, messageObject){
-    var container = this.data[channelId] || (this.data[channelId] = {});
-    var wasPresent = messageId in container;
-    
-    container[messageId] = messageObject;
-    this.tmp.messagekeys.add(messageId);
-    return !wasPresent;
-  }
-  
-  convertToMessageObject(discordMessage){
-    var author = discordMessage.author;
-    
-    var obj = {
-      u: this.findOrRegisterUser(author.id, author.username, author.bot ? null : author.discriminator, author.avatar),
-      t: discordMessage.timestamp.toDate().getTime()
-    };
-    
-    if (discordMessage.content.length > 0){
-      obj.m = discordMessage.content;
-    }
-    
-    if (discordMessage.editedTimestamp !== null){
-      obj.te = discordMessage.editedTimestamp.toDate().getTime();
-    }
-    
-    if (discordMessage.embeds.length > 0){
-      obj.e = discordMessage.embeds.map(embed => {
-        let conv = {
-          url: embed.url,
-          type: embed.type
-        };
-        
-        if (embed.type === "rich"){
-          if (Array.isArray(embed.title) && embed.title.length === 1 && typeof embed.title[0] === "string"){
-            conv.t = embed.title[0];
-            
-            if (Array.isArray(embed.description) && embed.description.length === 1 && typeof embed.description[0] === "string"){
-              conv.d = embed.description[0];
-            }
-          }
-        }
-        
-        return conv;
-      });
-    }
-    
-    if (discordMessage.attachments.length > 0){
-      obj.a = discordMessage.attachments.map(attachment => ({
-        url: attachment.url
-      }));
-    }
-    
-    if (discordMessage.messageReference !== null){
-      obj.r = discordMessage.messageReference.message_id;
-    }
-    
-    if (discordMessage.reactions.length > 0) {
-      obj.re = discordMessage.reactions.map(reaction => {
-	      let conv = {
-		      c: reaction.count,
-		      n: reaction.emoji.name
-	      };
-	      
-	      if (reaction.emoji.id !== null) {
-	      	conv.id = reaction.emoji.id;
-	      }
-	      
-	      if (reaction.emoji.animated) {
-	      	conv.an = true;
-	      }
-	      
-	      return conv;
-      });
-    }
-    
-    return obj;
-  }
-  
-  isMessageFresh(id){
-    return this.tmp.freshmsgs.has(id);
-  }
-  
-  addMessagesFromDiscord(channelId, discordMessageArray){
-    var hasNewMessages = false;
-    
-    for(var discordMessage of discordMessageArray){
-      var type = discordMessage.type;
-      
-      // https://discord.com/developers/docs/resources/channel#message-object-message-reference-structure
-      if ((type === 0 || type === 19) && discordMessage.state === "SENT" && this.addMessage(channelId, discordMessage.id, this.convertToMessageObject(discordMessage))){
-        this.tmp.freshmsgs.add(discordMessage.id);
-        hasNewMessages = true;
-      }
-    }
-    
-    return hasNewMessages;
-  }
-  
-  countChannels(){
-    return this.tmp.channelkeys.size;
-  }
-  
-  countMessages(){
-    return this.tmp.messagekeys.size;
-  }
-  
-  combineWith(obj){
-    var userMap = {};
-    var shownError = false;
-    
-    for(var userId in obj.meta.users){
-      var oldUser = obj.meta.users[userId];
-      userMap[obj.meta.userindex.findIndex(id => id == userId)] = this.findOrRegisterUser(userId, oldUser.name, oldUser.tag, oldUser.avatar);
-    }
-    
-    for(var channelId in obj.meta.channels){
-      var oldServer = obj.meta.servers[obj.meta.channels[channelId].server];
-      var oldChannel = obj.meta.channels[channelId];
-      this.tryRegisterChannel(this.findOrRegisterServer(oldServer.name, oldServer.type), channelId, oldChannel.name, oldChannel /* filtered later */);
-    }
-    
-    for(var channelId in obj.data){
-      var oldChannel = obj.data[channelId];
-      
-      for(var messageId in oldChannel){
-        var oldMessage = oldChannel[messageId];
-        var oldUser = oldMessage.u;
-        
-        if (oldUser in userMap){
-          oldMessage.u = userMap[oldUser];
-          this.addMessage(channelId, messageId, oldMessage);
-        }
-        else{
-          if (!shownError){
-            shownError = true;
-            alert("The uploaded archive appears to be corrupted, some messages will be skipped. See console for details.");
-            
-            console.error("User list:", obj.meta.users);
-            console.error("User index:", obj.meta.userindex);
-            console.error("Generated mapping:", userMap);
-            console.error("Missing user for the following messages:");
-          }
-          
-          console.error(oldMessage);
-        }
-      }
-    }
-  }
-  
-  toJson(){
-    return JSON.stringify({
-      "meta": this.meta,
-      "data": this.data
-    });
-  }
-}

src/tracker/settings.js

@@ -1,64 +0,0 @@
-var CONSTANTS = {
-  AUTOSCROLL_ACTION_NOTHING: "optNothing",
-  AUTOSCROLL_ACTION_PAUSE: "optPause",
-  AUTOSCROLL_ACTION_SWITCH: "optSwitch"
-};
-
-var IS_FIRST_RUN = false;
-
-var SETTINGS = (function(){
-  var root = {};
-  var settingsChangedEvents = [];
-  
-  var saveSettings = function(){
-    DOM.saveToCookie("DHT_SETTINGS", root, 60*60*24*365*5);
-  };
-  
-  var triggerSettingsChanged = function(changeType, changeDetail){
-    for(var callback of settingsChangedEvents){
-      callback(changeType, changeDetail);
-    }
-    
-    saveSettings();
-  };
-  
-  var defineTriggeringProperty = function(obj, property, value){
-    var name = "_"+property;
-    
-    Object.defineProperty(obj, property, {
-      get: (() => obj[name]),
-      set: (value => {
-        obj[name] = value;
-        triggerSettingsChanged("setting", property);
-      })
-    });
-    
-    obj[name] = value;
-  };
-  
-  var loaded = DOM.loadFromCookie("DHT_SETTINGS");
-  
-  if (!loaded){
-    loaded = {
-      "_autoscroll": true,
-      "_afterFirstMsg": CONSTANTS.AUTOSCROLL_ACTION_PAUSE,
-      "_afterSavedMsg": CONSTANTS.AUTOSCROLL_ACTION_PAUSE
-    };
-    
-    IS_FIRST_RUN = true;
-  }
-  
-  defineTriggeringProperty(root, "autoscroll", loaded._autoscroll);
-  defineTriggeringProperty(root, "afterFirstMsg", loaded._afterFirstMsg);
-  defineTriggeringProperty(root, "afterSavedMsg", loaded._afterSavedMsg);
-  
-  root.onSettingsChanged = function(callback){
-    settingsChangedEvents.push(callback);
-  };
-  
-  if (IS_FIRST_RUN){
-    saveSettings();
-  }
-  
-  return root;
-})();

src/tracker/state.js

@@ -1,119 +0,0 @@
-var STATE = (function(){
-  var stateChangedEvents = [];
-  
-  var triggerStateChanged = function(changeType, changeDetail){
-    for(var callback of stateChangedEvents){
-      callback(changeType, changeDetail);
-    }
-  };
-  
-  /*
-   * Internal class constructor.
-   */
-  class CLS{
-    constructor(){
-      this.resetState();
-    };
-    
-    /*
-     * Resets the state to default values.
-     */
-    resetState(){
-      this._savefile = null;
-      this._isTracking = false;
-      this._lastFileName = null;
-      triggerStateChanged("data", "reset");
-    }
-    
-    /*
-     * Returns the savefile object, creates a new one if needed.
-     */
-    getSavefile(){
-      if (!this._savefile){
-        this._savefile = new SAVEFILE();
-      }
-      
-      return this._savefile;
-    }
-    
-    /*
-     * Returns true if the database file contains any data.
-     */
-    hasSavedData(){
-      return this._savefile != null;
-    }
-    
-    /*
-     * Returns true if currently tracking message.
-     */
-    isTracking(){
-      return this._isTracking;
-    }
-    
-    /*
-     * Sets the tracking state.
-     */
-    setIsTracking(state){
-      this._isTracking = state;
-      triggerStateChanged("tracking", state);
-    }
-    
-    /*
-     * Combines current savefile with the provided one.
-     */
-    uploadSavefile(fileName, fileObject){
-      this._lastFileName = fileName;
-      this.getSavefile().combineWith(fileObject);
-      triggerStateChanged("data", "upload");
-    }
-    
-    /*
-     * Triggers a savefile download, if available.
-     */
-    downloadSavefile(){
-      if (this.hasSavedData()){
-        DOM.downloadTextFile(this._lastFileName || "dht.txt", this._savefile.toJson());
-      }
-    }
-    
-    /*
-     * Registers a Discord server and channel.
-     */
-    addDiscordChannel(serverName, serverType, channelId, channelName, extraInfo){
-      var serverIndex = this.getSavefile().findOrRegisterServer(serverName, serverType);
-      
-      if (this.getSavefile().tryRegisterChannel(serverIndex, channelId, channelName, extraInfo) === true){
-        triggerStateChanged("data", "channel");
-      }
-    }
-    
-    /*
-     * Adds all messages from the array to the specified channel. Returns true if the savefile was updated.
-     */
-    addDiscordMessages(channelId, discordMessageArray){
-      if (this.getSavefile().addMessagesFromDiscord(channelId, discordMessageArray)){
-        triggerStateChanged("data", "messages");
-        return true;
-      }
-      else{
-        return false;
-      }
-    }
-    
-    /*
-     * Returns true if the message was added during this session.
-     */
-    isMessageFresh(id){
-      return this.getSavefile().isMessageFresh(id);
-    }
-    
-    /*
-     * Adds a listener that is called whenever the state changes. The callback is a function that takes subject (generic type) and detail (specific type or data).
-     */
-    onStateChanged(callback){
-      stateChangedEvents.push(callback);
-    }
-  }
-  
-  return new CLS();
-})();

src/tracker/Σ.js

@@ -1,136 +0,0 @@
-const url = window.location.href;
-
-if (!url.includes("discord.com/") && !url.includes("discordapp.com/") && !confirm("Could not detect Discord in the URL, do you want to run the script anyway?")){
-  return;
-}
-
-if (window.DHT_LOADED){
-  alert("Discord History Tracker is already loaded.");
-  return;
-}
-
-window.DHT_LOADED = true;
-window.DHT_ON_UNLOAD = [];
-
-// Execution
-
-let ignoreMessageCallback = new Set();
-let frozenMessageLoadingTimer = null;
-
-let stopTrackingDelayed = function(callback){
-  ignoreMessageCallback.add("stopping");
-  
-  DOM.setTimer(() => {
-    STATE.setIsTracking(false);
-    ignoreMessageCallback.delete("stopping");
-    
-    if (callback){
-      callback();
-    }
-  }, 200); // give the user visual feedback after clicking the button before switching off
-};
-
-DISCORD.setupMessageCallback(messages => {
-  if (STATE.isTracking() && ignoreMessageCallback.size === 0){
-    let info = DISCORD.getSelectedChannel();
-    
-    if (!info){
-      stopTrackingDelayed();
-      return;
-    }
-    
-    STATE.addDiscordChannel(info.server, info.type, info.id, info.channel, info.extra);
-    
-     if (messages !== false && !messages.length){
-      DISCORD.loadOlderMessages();
-      return;
-    }
-    
-    let hasUpdatedFile = messages !== false && STATE.addDiscordMessages(info.id, messages);
-    
-    if (SETTINGS.autoscroll){
-      let action = null;
-      
-      if (messages === false) {
-        action = SETTINGS.afterFirstMsg;
-      }
-      else if (!hasUpdatedFile && !STATE.isMessageFresh(messages[0].id)){
-        action = SETTINGS.afterSavedMsg;
-      }
-      
-      if (action === null){
-        if (hasUpdatedFile){
-          DISCORD.loadOlderMessages();
-          window.clearTimeout(frozenMessageLoadingTimer);
-          frozenMessageLoadingTimer = null;
-        }
-        else{
-          frozenMessageLoadingTimer = window.setTimeout(DISCORD.loadOlderMessages, 2500);
-        }
-      }
-      else{
-        ignoreMessageCallback.add("stalling");
-        
-        DOM.setTimer(() => {
-          ignoreMessageCallback.delete("stalling");
-          
-          let updatedInfo = DISCORD.getSelectedChannel();
-          
-          if (updatedInfo && updatedInfo.id === info.id){
-            let lastMessages = DISCORD.getMessages(); // sometimes needed to catch the last few messages before switching
-            
-            if (lastMessages != null){
-              STATE.addDiscordMessages(info.id, lastMessages);
-            }
-          }
-          
-          if ((action === CONSTANTS.AUTOSCROLL_ACTION_SWITCH && !DISCORD.selectNextTextChannel()) || action === CONSTANTS.AUTOSCROLL_ACTION_PAUSE){
-            STATE.setIsTracking(false);
-          }
-        }, 250);
-      }
-    }
-  }
-});
-
-STATE.onStateChanged((type, enabled) => {
-  if (type === "tracking" && enabled){
-    let info = DISCORD.getSelectedChannel();
-    
-    if (info){
-      let messages = DISCORD.getMessages();
-      
-      if (messages != null){
-        STATE.addDiscordChannel(info.server, info.type, info.id, info.channel, info.extra);
-        STATE.addDiscordMessages(info.id, messages);
-      }
-      else{
-        stopTrackingDelayed(() => alert("Cannot see any messages."));
-        return;
-      }
-    }
-    else{
-      stopTrackingDelayed(() => alert("The selected channel is not visible in the channel list."));
-      return;
-    }
-    
-    if (SETTINGS.autoscroll && DISCORD.isInMessageView()){
-      if (DISCORD.hasMoreMessages()){
-        DISCORD.loadOlderMessages();
-      }
-      else{
-        let action = SETTINGS.afterFirstMsg;
-        
-        if ((action === CONSTANTS.AUTOSCROLL_ACTION_SWITCH && !DISCORD.selectNextTextChannel()) || action === CONSTANTS.AUTOSCROLL_ACTION_PAUSE){
-          stopTrackingDelayed();
-        }
-      }
-    }
-  }
-});
-
-GUI.showController();
-
-if (IS_FIRST_RUN){
-  GUI.showSettings();
-}

src/viewer/index.html

@@ -1,85 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-    <title>Discord Offline History</title>
-    
-    <script type="text/javascript">
-      /*{js}*/
-    </script>
-    <style type="text/css">
-      /*{css}*/
-    </style>
-  </head>
-  <body>
-    <div id="menu">
-      <input id="uploaded-file" type="file" style="display:none">
-      <button id="btn-upload-file" class="hide-embedded">Load File</button>
-      
-      <div class="splitter hide-embedded"></div>
-      
-      <button id="btn-settings">Settings</button>
-      
-      <div> <!-- needed to stop the select from messing up -->
-        <select id="opt-messages-per-page">
-          <option value="50">50 messages per page&nbsp;</option>
-          <option value="100">100 messages per page&nbsp;</option>
-          <option value="250">250 messages per page&nbsp;</option>
-          <option value="500">500 messages per page&nbsp;</option>
-          <option value="1000">1000 messages per page&nbsp;</option>
-          <option value="0">All messages&nbsp;</option>
-        </select>
-      </div>
-
-      <div class="nav">
-        <button id="nav-first" data-nav="first" class="icon">&laquo;</button>
-        <button id="nav-prev" data-nav="prev" class="icon">&lsaquo;</button>
-        <button id="nav-pick" data-nav="pick">Page <span id="nav-page-current">1</span>/<span id="nav-page-total">?</span></button>
-        <button id="nav-next" data-nav="next" class="icon">&rsaquo;</button>
-        <button id="nav-last" data-nav="last" class="icon">&raquo;</button>
-      </div>
-      
-      <div class="splitter"></div>
-      
-      <div> <!-- needed to stop the select from messing up -->
-        <select id="opt-messages-filter">
-          <option value="">No filter&nbsp;</option>
-          <option value="user">Filter messages by user&nbsp;</option>
-          <option value="contents">Filter messages by contents&nbsp;</option>
-          <option value="withimages">Only messages with images&nbsp;</option>
-          <option value="withdownloads">Only messages with downloads&nbsp;</option>
-          <option value="edited">Only edited messages&nbsp;</option>
-        </select>
-      </div>
-      
-      <div id="opt-filter-list">
-        <select id="opt-filter-user" data-filter-type="user">
-          <option value="">Select user...</option>
-        </select>
-        <input id="opt-filter-contents" type="text" data-filter-type="contents" placeholder="Messages containing...">
-        <input type="hidden" data-filter-type="withimages" value="1">
-        <input type="hidden" data-filter-type="withdownloads" value="1">
-        <input type="hidden" data-filter-type="edited" value="1">
-      </div>
-      
-      <div id="opt-save-filtered">
-        <div class="splitter"></div>
-        <button id="btn-save-filtered">Save Filtered Messages</button>
-      </div>
-      
-      <div class="separator"></div>
-      
-      <button id="btn-about">About</button>
-    </div>
-    
-    <div id="app">
-      <div id="channels"></div>
-      <div id="messages"></div>
-    </div>
-    
-    <div id="modal">
-      <div id="overlay"></div>
-      <div id="dialog"></div>
-    </div>
-  </body>
-</html>

src/viewer/scripts/discord.js

@@ -1,246 +0,0 @@
-var DISCORD = (function(){
-  var REGEX = {
-    formatBold: /\*\*([\s\S]+?)\*\*(?!\*)/g,
-    formatItalic: /(.)?\*([\s\S]+?)\*(?!\*)/g,
-    formatUnderline: /__([\s\S]+?)__(?!_)/g,
-    formatStrike: /~~([\s\S]+?)~~(?!~)/g,
-    formatCodeInline: /(`+)\s*([\s\S]*?[^`])\s*\1(?!`)/g,
-    formatCodeBlock: /```(?:([A-z0-9\-]+?)\n+)?\n*([^]+?)\n*```/g,
-    formatUrl: /(\b(?:https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig,
-    formatUrlNoEmbed: /<(\b(?:https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])>/ig,
-    specialEscapedBacktick: /\\`/g,
-    specialEscapedSingle: /\\([*\\])/g,
-    specialEscapedDouble: /\\__|_\\_|\\_\\_|\\~~|~\\~|\\~\\~/g,
-    specialUnescaped: /([*_~\\])/g,
-    mentionRole: /&lt;@&(\d+?)&gt;/g,
-    mentionUser: /&lt;@!?(\d+?)&gt;/g,
-    mentionChannel: /&lt;#(\d+?)&gt;/g,
-    customEmojiStatic: /&lt;:([^:]+):(\d+?)&gt;/g,
-    customEmojiAnimated: /&lt;a:([^:]+):(\d+?)&gt;/g
-  };
-  
-  var isImageAttachment = function(attachment){
-    var dot = attachment.url.lastIndexOf(".");
-    var ext = dot === -1 ? "" : attachment.url.substring(dot).toLowerCase();
-    return ext === ".png" || ext === ".gif" || ext === ".jpg" || ext === ".jpeg";
-  };
-  
-  var getHumanReadableTime = function(timestamp){
-    var date = new Date(timestamp);
-    return date.toLocaleDateString() + ", " + date.toLocaleTimeString();
-  };
-  
-  var templateChannelServer;
-  var templateChannelPrivate;
-  var templateMessageNoAvatar;
-  var templateMessageWithAvatar;
-  var templateUserAvatar;
-  var templateEmbedImage;
-  var templateEmbedRich;
-  var templateEmbedRichNoDescription;
-  var templateEmbedRichUnsupported;
-  var templateEmbedDownload;
-  
-  var processMessageContents = function(contents){
-    var processed = DOM.escapeHTML(contents.replace(REGEX.formatUrlNoEmbed, "$1"));
-    
-    if (STATE.settings.enableFormatting){
-      var escapeHtmlMatch = (full, match) => "&#"+match.charCodeAt(0)+";";
-      
-      processed = processed
-        .replace(REGEX.specialEscapedBacktick, "&#96;")
-        .replace(REGEX.formatCodeBlock, (full, ignore, match) => "<code class='block'>"+match.replace(REGEX.specialUnescaped, escapeHtmlMatch)+"</code>")
-        .replace(REGEX.formatCodeInline, (full, ignore, match) => "<code class='inline'>"+match.replace(REGEX.specialUnescaped, escapeHtmlMatch)+"</code>")
-        .replace(REGEX.specialEscapedSingle, escapeHtmlMatch)
-        .replace(REGEX.specialEscapedDouble, full => full.replace(/\\/g, "").replace(/(.)/g, escapeHtmlMatch))
-        .replace(REGEX.formatBold, "<b>$1</b>")
-        .replace(REGEX.formatItalic, (full, pre, match) => pre === '\\' ? full : (pre || "")+"<i>"+match+"</i>")
-        .replace(REGEX.formatUnderline, "<u>$1</u>")
-        .replace(REGEX.formatStrike, "<s>$1</s>");
-    }
-    
-    var animatedEmojiExtension = STATE.settings.enableAnimatedEmoji ? "gif" : "png";
-    
-    processed = processed
-      .replace(REGEX.formatUrl, "<a href='$1' target='_blank' rel='noreferrer'>$1</a>")
-      .replace(REGEX.mentionChannel, (full, match) => "<span class='link mention-chat'>#"+STATE.getChannelName(match)+"</span>")
-      .replace(REGEX.mentionUser, (full, match) => "<span class='link mention-user' title='#"+(STATE.getUserTag(match) || "????")+"'>@"+STATE.getUserName(match)+"</span>")
-      .replace(REGEX.customEmojiStatic, "<img src='https://cdn.discordapp.com/emojis/$2.png' alt=':$1:' title=':$1:' class='emoji'>")
-      .replace(REGEX.customEmojiAnimated, "<img src='https://cdn.discordapp.com/emojis/$2."+animatedEmojiExtension+"' alt=':$1:' title=':$1:' class='emoji'>");
-    
-    return "<p>"+processed+"</p>";
-  };
-  
-  return {
-    setup: function(){
-      templateChannelServer = new TEMPLATE([
-        "<div data-channel='{id}'>",
-        "<div class='info' title='{topic}'><strong class='name'>#{name}</strong>{nsfw}<span class='tag'>{msgcount}</span></div>",
-        "<span class='server'>{server.name} ({server.type})</span>",
-        "</div>"
-      ].join(""));
-      
-      templateChannelPrivate = new TEMPLATE([
-        "<div data-channel='{id}'>",
-        "<div class='info'><strong class='name'>{name}</strong><span class='tag'>{msgcount}</span></div>",
-        "<span class='server'>({server.type})</span>",
-        "</div>"
-      ].join(""));
-      
-      templateMessageNoAvatar = new TEMPLATE([
-        "<div>",
-        "<div class='reply-message'>{reply}</div>",
-        "<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
-        "<div class='message'>{contents}{embeds}{attachments}</div>",
-        "<div class='reactions'>{reactions}</div>",
-        "</div>"
-      ].join(""));
-      
-      templateMessageWithAvatar = new TEMPLATE([
-        "<div>",
-        "<div class='reply-message reply-message-with-avatar'>{reply}</div>",
-        "<div class='avatar-wrapper'>",
-        "<div class='avatar'>{avatar}</div>",
-        "<div>",
-        "<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
-        "<div class='message'>{contents}{embeds}{attachments}</div>",
-        "<div class='reactions'>{reactions}</div>",
-        "</div>",
-        "</div>",
-        "</div>"
-      ].join(""));
-      
-      templateUserAvatar = new TEMPLATE([
-        "<img src='https://cdn.discordapp.com/avatars/{id}/{path}.webp?size=128'>"
-      ].join(""));
-      
-      templateEmbedImage = new TEMPLATE([
-        "<a href='{url}' class='embed thumbnail'><img src='{url}' alt='(image attachment not found)'></a><br>"
-      ].join(""));
-      
-      templateEmbedRich = new TEMPLATE([
-        "<div class='embed download'><a href='{url}' class='title'>{t}</a><p class='desc'>{d}</p></div>"
-      ].join(""));
-      
-      templateEmbedRichNoDescription = new TEMPLATE([
-        "<div class='embed download'><a href='{url}' class='title'>{t}</a></div>"
-      ].join(""));
-      
-      templateEmbedRichUnsupported = new TEMPLATE([
-        "<div class='embed download'><p>(Formatted embeds are currently not supported)</p></div>"
-      ].join(""));
-      
-      templateEmbedDownload = new TEMPLATE([
-        "<a href='{url}' class='embed download'>Download {filename}</a>"
-      ].join(""));
-      
-      templateReaction = new TEMPLATE([
-        "<span class='reaction-wrapper'><span class='reaction-emoji'>{n}</span><span class='count'>{c}</span></span>"
-      ].join(""));
-      
-      templateReactionCustom = new TEMPLATE([
-        "<span class='reaction-wrapper'><img src='https://cdn.discordapp.com/emojis/{id}.{ext}' alt=':{n}:' title=':{n}:' class='reaction-emoji-custom'><span class='count'>{c}</span></span>"
-      ].join(""));
-    },
-    
-    isImageAttachment: isImageAttachment,
-    
-    getChannelHTML: function(channel){
-      return (channel.server.type === "SERVER" ? templateChannelServer : templateChannelPrivate).apply(channel, (property, value) => {
-        if (property === "server.type"){
-          switch(value){
-            case "SERVER": return "server";
-            case "GROUP": return "group";
-            case "DM": return "user";
-          }
-        }
-        else if (property === "nsfw"){
-          return value ? "<span class='tag'>NSFW</span>" : "";
-        }
-      });
-    },
-    
-    getMessageHTML: function(message){
-      return (STATE.settings.enableUserAvatars ? templateMessageWithAvatar : templateMessageNoAvatar).apply(message, (property, value) => {
-        if (property === "avatar"){
-          return value ? templateUserAvatar.apply(value) : "";
-        }
-        else if (property === "user.tag"){
-          return value ? value : "????";
-        }
-        else if (property === "timestamp"){
-          return getHumanReadableTime(value);
-        }
-        else if (property === "contents"){
-          return value == null || value.length === 0 ? "" : processMessageContents(value);
-        }
-        else if (property === "embeds"){
-          if (!value){
-            return "";
-          }
-          
-          return value.map(embed => {
-            switch(embed.type){
-              case "image":
-                return STATE.settings.enableImagePreviews ? templateEmbedImage.apply(embed) : "";
-                
-              case "rich":
-                return (embed.t ? (embed.d ? templateEmbedRich : templateEmbedRichNoDescription) : templateEmbedRichUnsupported).apply(embed);
-            }
-          }).join("");
-        }
-        else if (property === "attachments"){
-          if (!value){
-            return "";
-          }
-          
-          return value.map(attachment => {
-            if (isImageAttachment(attachment) && STATE.settings.enableImagePreviews){
-              return templateEmbedImage.apply(attachment);
-            }
-            else{
-              var sliced = attachment.url.split("/");
-
-              return templateEmbedDownload.apply({
-                "url": attachment.url,
-                "filename": sliced[sliced.length-1]
-              });
-            }
-          }).join("");
-        }
-        else if (property === "edit"){
-          return value ? "<span class='info edited'>Edited" + (value > 1 ? " " + getHumanReadableTime(value) : "") + "</span>" : "";
-        }
-        else if (property === "jump"){
-          return STATE.hasActiveFilter ? "<span class='info jump' data-jump='" + value + "'>Jump to message</span>" : "";
-        }
-        else if (property === "reply"){
-          if (value === null) {
-            return "";
-          }
-          
-          var user = "<span class='reply-username' title='#" + (value.user.tag ? value.user.tag : "????") + "'>" + value.user.name + "</span>";
-          var avatar = STATE.settings.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(value.avatar) + "</span>" : "";
-          var contents = value.contents ? "<span class='reply-contents'>" + processMessageContents(value.contents) + "</span>" : "";
-          
-          return "<span class='jump' data-jump='" + value.id + "'>Jump to reply</span><span class='user'>" + avatar + user + "</span>" + contents;
-        }
-        else if (property === "reactions"){
-          if (value === null){
-            return "";
-          }
-          
-          return value.map(reaction => {
-            if ("id" in reaction){
-              reaction.ext = reaction.an && STATE.settings.enableAnimatedEmoji ? "gif" : "png";
-              return templateReactionCustom.apply(reaction);
-            }
-            else {
-              return templateReaction.apply(reaction);
-            }
-          }).join("");
-        }
-      });
-    }
-  };
-})();

src/viewer/scripts/dom.js

@@ -1,77 +0,0 @@
-var DOM = (function(){
-  var createElement = (tag, parent) => {
-    var ele = document.createElement(tag);
-    parent.appendChild(ele);
-    return ele;
-  };
-  
-  var entityMap = {
-    "&": "&",
-    "<": "&lt;",
-    ">": "&gt;",
-    '"': '&quot;',
-    "'": '&#39;'
-  };
-  
-  var entityRegex = /[&<>"']/g;
-  
-  return {
-    /*
-     * Returns a child element by its ID. Parent defaults to the entire document.
-     */
-    id: (id, parent) => (parent || document).getElementById(id),
-    
-    /*
-     * Returns an array of all child elements containing the specified class. Parent defaults to the entire document.
-     */
-    cls: (cls, parent) => Array.prototype.slice.call((parent || document).getElementsByClassName(cls)),
-    
-    /*
-     * Returns an array of all child elements that have the specified tag. Parent defaults to the entire document.
-     */
-    tag: (tag, parent) => Array.prototype.slice.call((parent || document).getElementsByTagName(tag)),
-    
-    /*
-     * Returns the first child element containing the specified class. Parent defaults to the entire document.
-     */
-    fcls: (cls, parent) => (parent || document).getElementsByClassName(cls)[0],
-    
-    /*
-     * Creates an element, adds it to the DOM, and returns it.
-     */
-    createElement: (tag, parent) => createElement(tag, parent),
-    
-    /*
-     * Removes an element from the DOM.
-     */
-    removeElement: (ele) => ele.parentNode.removeChild(ele),
-    
-    /*
-     * Converts characters to their HTML entity form.
-     */
-    escapeHTML: (html) => String(html).replace(entityRegex, s => entityMap[s]),
-    
-    /*
-     * Triggers a UTF-8 text file download.
-     */
-    downloadTextFile: (fileName, fileContents) => {
-      var blob = new Blob([fileContents], { "type": "octet/stream" });
-      
-      if ("msSaveBlob" in window.navigator){
-        return window.navigator.msSaveBlob(blob, fileName);
-      }
-      
-      var url = window.URL.createObjectURL(blob);
-      
-      var ele = createElement("a", document.body);
-      ele.href = url;
-      ele.download = fileName;
-      ele.style.display = "none";
-      
-      ele.click();
-      
-      document.body.removeChild(ele);
-      window.URL.revokeObjectURL(url);
-    }
-  };
-})();

src/viewer/scripts/embed.js

@@ -1,58 +0,0 @@
-var EMBED = (function(){
-  var enabled = false;
-  
-  var html;
-  var generated;
-  
-  var downloadTextFile = function(fileName, fileContents){
-    var blob = new Blob([fileContents], { "type": "octet/stream" });
-    
-    if ("msSaveBlob" in window.navigator){
-      return window.navigator.msSaveBlob(blob, fileName);
-    }
-    
-    var url = window.URL.createObjectURL(blob);
-    
-    var ele = DOM.createElement("a", document.body);
-    ele.href = url;
-    ele.download = fileName;
-    ele.style.display = "none";
-    
-    ele.click();
-    
-    document.body.removeChild(ele);
-    window.URL.revokeObjectURL(url);
-  };
-  
-  var utoa = function(str){
-    return window.btoa(unescape(encodeURIComponent(str)));
-  };
-  
-  var atou = function(str){
-    return decodeURIComponent(escape(window.atob(str)));
-  };
-  
-  return {
-    setup: function(){
-      enabled = true;
-      html = "<!DOCTYPE html>\n" + document.documentElement.outerHTML;
-      
-      DOM.id("btn-upload-file").insertAdjacentHTML("afterend", `<button id="btn-embed-file" disabled>Embed File</button>`);
-      DOM.id("btn-embed-file").addEventListener("click", () => downloadTextFile("embed.html", generated));
-    },
-    
-    onFileRead: function(json){
-      if (!enabled){
-        return;
-      }
-      
-      DOM.id("btn-embed-file").disabled = false;
-      generated = html.replace("</title>", `</title>\n<script type="text/javascript">window.DHT_EMBEDDED = "${utoa(json)}";<\/script>`).replace(`<${document.body.tagName.toLowerCase()}>`, `<body class="embedded">`);
-    },
-    
-    getEmbeddedJSON: function(){
-      var embed = window.DHT_EMBEDDED;
-      return embed ? atou(embed) : null;
-    }
-  };
-})();

src/viewer/scripts/exec.js

@@ -1,80 +0,0 @@
-document.addEventListener("DOMContentLoaded", () => {
-  var embedded = EMBED.getEmbeddedJSON();
-  
-  if (location.search === "?embed" && !embedded){
-    EMBED.setup();
-  }
-  
-  DISCORD.setup();
-  GUI.setup();
-  
-  GUI.onOptionMessagesPerPageChanged(() => {
-    STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage());
-  });
-  
-  STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage());
-  
-  GUI.onOptMessageFilterChanged(filter => {
-    STATE.setActiveFilter(filter);
-  });
-  
-  GUI.onNavigationButtonClicked(action => {
-    STATE.updateCurrentPage(action);
-  });
-  
-  STATE.onUsersRefreshed(users => {
-    GUI.updateUserList(users);
-  });
-  
-  STATE.onChannelsRefreshed((channels, selected) => {
-    GUI.updateChannelList(channels, selected, STATE.selectChannel);
-  });
-  
-  STATE.onMessagesRefreshed(messages => {
-    GUI.updateNavigation(STATE.getCurrentPage(), STATE.getPageCount());
-    GUI.updateMessageList(messages);
-    GUI.scrollMessagesToTop();
-  });
-  
-  var loadJSON = function(json, errParse, errInvalid){
-    var obj;
-    
-    try{
-      obj = JSON.parse(json);
-      EMBED.onFileRead(json);
-    }catch(e){
-      console.error(e);
-      alert(errParse);
-      return;
-    }
-    
-    if (SAVEFILE.isValid(obj)){
-      STATE.uploadFile(new SAVEFILE(obj));
-    }
-    else{
-      alert(errInvalid);
-    }
-  };
-  
-  if (embedded){
-    loadJSON(embedded, "Could not parse embedded file, see console for details.", "Embedded file has an invalid format.");
-  }
-  else{
-    GUI.onFileUploaded(files => {
-      if (files.length === 1){
-        var file = files[0];
-        var reader = new FileReader();
-        
-        STATE.setUploadedFileName(file.name);
-
-        reader.onload = () => loadJSON(reader.result, "Could not parse '"+file.name+"', see console for details.", "File '"+file.name+"' has an invalid format.");
-        reader.readAsText(file, "UTF-8");
-      }
-      else{
-        alert("Please, select only one file.");
-      }
-      
-      return true;
-    });
-  }
-});

src/viewer/scripts/gui.js

@@ -1,295 +0,0 @@
-var GUI = (function(){
-  var eventOnFileUploaded;
-  var eventOnOptMessagesPerPageChanged;
-  var eventOnOptMessageFilterChanged;
-  var eventOnNavButtonClicked;
-  
-  var getActiveFilter = function(){
-    var active = DOM.fcls("active", DOM.id("opt-filter-list"));
-    
-    return active && active.value !== "" ? {
-      "type": active.getAttribute("data-filter-type"),
-      "value": active.value
-    } : null;
-  };
-  
-  var triggerFilterChanged = function(){
-    var activeFilter = getActiveFilter();
-    DOM.id("opt-save-filtered").classList.toggle("active", activeFilter != null);
-    
-    eventOnOptMessageFilterChanged && eventOnOptMessageFilterChanged(activeFilter);
-  };
-  
-  var showModal = function(width, html){
-    var dialog = DOM.id("dialog");
-    dialog.innerHTML = html;
-    dialog.style.width = width+"px";
-    dialog.style.marginLeft = (-width/2)+"px";
-    
-    DOM.id("modal").classList.add("visible");
-    return dialog;
-  };
-  
-  // -------------
-  // Modal dialogs
-  // -------------
-  
-  var showSettingsModal = function(){
-    showModal(560, `
-<label><input id='dht-cfg-imgpreviews' type='checkbox'> Image Previews</label><br>
-<label><input id='dht-cfg-formatting' type='checkbox'> Message Formatting</label><br>
-<label><input id='dht-cfg-useravatars' type='checkbox'> User Avatars</label><br>
-<label><input id='dht-cfg-animemoji' type='checkbox'> Animated Emoji</label><br>`);
-    
-    var setupCheckBox = function(id, settingName){
-      var ele = DOM.id(id);
-      ele.checked = STATE.settings[settingName];
-      ele.addEventListener("change", () => STATE.settings[settingName] = ele.checked);
-    };
-    
-    setupCheckBox("dht-cfg-imgpreviews", "enableImagePreviews");
-    setupCheckBox("dht-cfg-formatting", "enableFormatting");
-    setupCheckBox("dht-cfg-useravatars", "enableUserAvatars");
-    setupCheckBox("dht-cfg-animemoji", "enableAnimatedEmoji");
-  };
-  
-  var showInfoModal = function(){
-    var linkGH = "https://github.com/chylex/Discord-History-Tracker";
-    
-    showModal(560, `
-<p>Discord History Tracker is developed by <a href='https://chylex.com'>chylex</a> as an <a href='${linkGH}/blob/master/LICENSE.md'>open source</a> project.</p>
-<sub>{{{version:full}}}</sub>
-<p>Please, report any issues and suggestions to the <a href='${linkGH}/issues'>tracker</a>. If you want to support the development, please spread the word and consider <a href='https://www.patreon.com/chylex'>becoming a patron</a> or <a href='https://ko-fi.com/chylex'>buying me a coffee</a>. Any support is appreciated!</p>
-<p><a href='${linkGH}/issues'>Issue Tracker</a> &nbsp;&mdash;&nbsp; <a href='${linkGH}'>GitHub Repository</a> &nbsp;&mdash;&nbsp; <a href='https://twitter.com/chylexmc'>Developer's Twitter</a></p>`);
-  };
-  
-  return {
-    // ---------
-    // GUI setup
-    // ---------
-    
-    /*
-     * Hooks all event listeners into the DOM.
-     */
-    setup: function(){
-      var inputUploadedFile = DOM.id("uploaded-file");
-      var inputMessageFilter = DOM.id("opt-messages-filter");
-      var containerFilterList = DOM.id("opt-filter-list");
-      
-      var resetActiveFilter = function(){
-        inputMessageFilter.value = "";
-        inputMessageFilter.dispatchEvent(new Event("change"));
-        
-        DOM.id("opt-filter-contents").value = "";
-        DOM.id("opt-save-filtered").classList.remove("active");
-      };
-      
-      DOM.id("btn-upload-file").addEventListener("click", () => {
-        inputUploadedFile.click();
-      });
-      
-      inputUploadedFile.addEventListener("change", () => {
-        if (eventOnFileUploaded && eventOnFileUploaded(inputUploadedFile.files)){
-          inputUploadedFile.value = null;
-          resetActiveFilter();
-        }
-      });
-      
-      inputMessageFilter.value = ""; // required to prevent browsers from remembering old value
-      
-      inputMessageFilter.addEventListener("change", () => {
-        DOM.cls("active", containerFilterList).forEach(ele => ele.classList.remove("active"));
-        
-        if (inputMessageFilter.value){
-          containerFilterList.querySelector("[data-filter-type='"+inputMessageFilter.value+"']").classList.add("active");
-        }
-        
-        triggerFilterChanged();
-      });
-      
-      Array.prototype.forEach.call(containerFilterList.children, ele => {
-        ele.addEventListener(ele.tagName === "SELECT" ? "change" : "input", e => triggerFilterChanged());
-      });
-      
-      DOM.id("opt-messages-per-page").addEventListener("change", () => {
-        eventOnOptMessagesPerPageChanged && eventOnOptMessagesPerPageChanged();
-      });
-      
-      DOM.id("btn-save-filtered").addEventListener("click", () => {
-        if (confirm("Filtering only removes messages, all users and servers will remain in the new archive. Continue?")){
-          STATE.saveFilteredMessages();
-        }
-      });
-
-      DOM.tag("button", DOM.fcls("nav")).forEach(button => {
-        button.disabled = true;
-        
-        button.addEventListener("click", () => {
-          eventOnNavButtonClicked && eventOnNavButtonClicked(button.getAttribute("data-nav"));
-        });
-      });
-      
-      DOM.id("btn-settings").addEventListener("click", () => {
-        showSettingsModal();
-      });
-      
-      DOM.id("btn-about").addEventListener("click", () => {
-        showInfoModal();
-      });
-      
-      DOM.id("messages").addEventListener("click", e => {
-        var jump = e.target.getAttribute("data-jump");
-        
-        if (jump){
-          resetActiveFilter();
-          
-          var index = STATE.navigateToMessage(jump);
-          DOM.id("messages").children[index].scrollIntoView();
-        }
-      });
-      
-      DOM.id("overlay").addEventListener("click", () => {
-        DOM.id("modal").classList.remove("visible");
-        DOM.id("dialog").innerHTML = "";
-      });
-    },
-    
-    // -----------------
-    // Event registering
-    // -----------------
-    
-    /*
-     * Sets a callback for when a file is uploaded. The callback takes a single argument, which is the file object array, and should return true to reset the input.
-     */
-    onFileUploaded: function(callback){
-      eventOnFileUploaded = callback;
-    },
-    
-    /*
-     * Sets a callback for when the user changes the messages per page option. The callback is not passed any arguments.
-     */
-    onOptionMessagesPerPageChanged: function(callback){
-      eventOnOptMessagesPerPageChanged = callback;
-    },
-    
-    /*
-     * Sets a callback for when the user changes the active filter. The callback is passed either null or an object such as { type: <filter type>, value: <filter value> }.
-     */
-    onOptMessageFilterChanged: function(callback){
-      eventOnOptMessageFilterChanged = callback;
-    },
-
-    /*
-     * Sets a callback for when the user clicks a navigation button. The callback is passed one of the following strings: first, prev, next, last.
-     */
-    onNavigationButtonClicked: function(callback){
-      eventOnNavButtonClicked = callback;
-    },
-    
-    // ----------------------
-    // Options and navigation
-    // ----------------------
-    
-    /*
-     * Returns the selected amount of messages per page.
-     */
-    getOptionMessagesPerPage: function(){
-      return parseInt(DOM.id("opt-messages-per-page").value, 10);
-    },
-
-    /*
-     * Updates the navigation text and buttons.
-     */
-    updateNavigation: function(currentPage, totalPages){
-      DOM.id("nav-page-current").innerHTML = currentPage;
-      DOM.id("nav-page-total").innerHTML = totalPages || "?";
-      
-      DOM.id("nav-first").disabled = currentPage === 1;
-      DOM.id("nav-prev").disabled = currentPage === 1;
-      DOM.id("nav-pick").disabled = (totalPages || 0) <= 1;
-      DOM.id("nav-next").disabled = currentPage === (totalPages || 1);
-      DOM.id("nav-last").disabled = currentPage === (totalPages || 1);
-    },
-    
-    // --------------
-    // Updating lists
-    // --------------
-    
-    /*
-     * Updates the channel list and sets up their click events. The callback is triggered whenever a channel is selected, and takes the channel ID as its argument.
-     */
-    updateChannelList: function(channels, selected, callback){
-      var eleChannels = DOM.id("channels");
-      
-      if (!channels){
-        eleChannels.innerHTML = "";
-      }
-      else{
-        if (getActiveFilter() != null){
-          channels = channels.filter(channel => channel.msgcount > 0);
-        }
-        
-        eleChannels.innerHTML = channels.map(channel => DISCORD.getChannelHTML(channel)).join("");
-        
-        Array.prototype.forEach.call(eleChannels.children, ele => {
-          ele.addEventListener("click", e => {
-            var currentChannel = DOM.fcls("active", eleChannels);
-
-            if (currentChannel){
-              currentChannel.classList.remove("active");
-            }
-
-            ele.classList.add("active");
-            callback(ele.getAttribute("data-channel"));
-          });
-        });
-        
-        if (selected){
-          var activeChannel = eleChannels.querySelector("[data-channel='"+selected+"']");
-          activeChannel && activeChannel.classList.add("active");
-        }
-      }
-    },
-    
-    // ------------
-    // Message list
-    // ------------
-    
-    /*
-     * Updates the message list.
-     */
-    updateMessageList: function(messages){
-      DOM.id("messages").innerHTML = messages ? messages.map(message => DISCORD.getMessageHTML(message)).join("") : "";
-    },
-    
-    /*
-     * Updates the user filter list.
-     */
-    updateUserList: function(users){
-      var eleSelect = DOM.id("opt-filter-user");
-      
-      while(eleSelect.length > 1){
-        eleSelect.remove(1);
-      }
-      
-      var options = [];
-      
-      for(var key of Object.keys(users)){
-        var option = document.createElement("option");
-        option.value = key;
-        option.text = users[key].name;
-        options.push(option);
-      }
-      
-      options.sort((a, b) => a.text.toLocaleLowerCase().localeCompare(b.text.toLocaleLowerCase()));
-      options.forEach(option => eleSelect.add(option));
-    },
-
-    /*
-     * Scrolls the message div to the top.
-     */
-    scrollMessagesToTop: function(){
-      DOM.id("messages").scrollTop = 0;
-    }
-  };
-})();

src/viewer/scripts/processor.js

@@ -1,41 +0,0 @@
-var PROCESSOR = {};
-
-// ------------------------
-// Global filter generators
-// ------------------------
-
-PROCESSOR.FILTER = {
-  byUser: ((userindex) => message => message.u === userindex),
-  byTime: ((timeStart, timeEnd) => message => message.t >= timeStart && message.t <= timeEnd),
-  byContents: ((substr) => message => ("m" in message ? message.m : "").indexOf(substr) !== -1),
-  byRegex: ((regex) => message => regex.test("m" in message ? message.m : "")),
-  withImages: (() => message => (message.e && message.e.some(embed => embed.type === "image")) || (message.a && message.a.some(DISCORD.isImageAttachment))),
-  withDownloads: (() => message => message.a && message.a.some(attachment => !DISCORD.isImageAttachment(attachment))),
-  withEmbeds: (() => message => message.e && message.e.length > 0),
-  withAttachments: (() => message => message.a && message.a.length > 0),
-  isEdited: (() => message => ("te" in message) ? message.te : (message.f & 1) === 1)
-};
-
-// --------------
-// Global sorters
-// --------------
-
-PROCESSOR.SORTER = {
-  oldestToNewest: (key1, key2) => {
-    if (key1.length === key2.length){
-      return key1 > key2 ? 1 : key1 < key2 ? -1 : 0;
-    }
-    else{
-      return key1.length > key2.length ? 1 : -1;
-    }
-  },
-  
-  newestToOldest: (key1, key2) => {
-    if (key1.length === key2.length){
-      return key1 > key2 ? -1 : key1 < key2 ? 1 : 0;
-    }
-    else{
-      return key1.length > key2.length ? -1 : 1;
-    }
-  }
-};

src/viewer/scripts/savefile.js

@@ -1,83 +0,0 @@
-class SAVEFILE{
-  constructor(parsedObj){
-    var me = this;
-    
-    me.meta = parsedObj.meta;
-    me.data = parsedObj.data;
-    
-    me.meta.users = me.meta.users || {};
-    me.meta.userindex = me.meta.userindex || [];
-    me.meta.servers = me.meta.servers || [];
-    me.meta.channels = me.meta.channels || {};
-  };
-
-  static isValid(parsedObj){
-    return parsedObj && typeof parsedObj.meta === "object" && typeof parsedObj.data === "object";
-  };
-
-  getServer(index){
-    return this.meta.servers[index] || { "name": "<unknown>", "type": "ERROR" };
-  }
-
-  getChannels(){
-    return this.meta.channels;
-  }
-
-  getChannelById(channel){
-    return this.meta.channels[channel] || { "id": channel, "name": channel };
-  }
-
-  getUsers(){
-    return this.meta.users;
-  }
-
-  getUser(index){
-    return this.meta.users[this.meta.userindex[index]] || { "name": "<unknown>" };
-  }
-
-  getUserId(index){
-    return this.meta.userindex[index];
-  }
-
-  getUserById(user){
-    return this.meta.users[user] || { "name": user };
-  }
-
-  getUserIndex(user){
-    return this.meta.userindex.indexOf(user);
-  }
-
-  getMessages(channel){
-    return this.data[channel] || {};
-  }
-
-  filterToJson(filterFunction){
-    var newMeta = JSON.parse(JSON.stringify(this.meta));
-    var newData = {};
-    
-    for(let channel of Object.keys(this.getChannels())){
-      var messages = this.getMessages(channel);
-      var retained = {};
-      
-      for(let key of Object.keys(messages)){
-        var message = messages[key];
-        
-        if (filterFunction(message)){
-          retained[key] = message;
-        }
-      }
-      
-      if (Object.keys(retained).length > 0){
-        newData[channel] = retained;
-      }
-      else{
-        delete newMeta.channels[channel];
-      }
-    }
-    
-    return JSON.stringify({
-      "meta": newMeta,
-      "data": newData
-    });
-  }
-}