Change image alt text in viewer to indicate when images are loading, and when loading fails

Related to #79

app/.idea/.idea.DiscordHistoryTracker/.idea/avalonia.xml

@@ -7,6 +7,7 @@
         <entry key="Desktop/Dialogs/CheckBox/CheckBoxDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Dialogs/Message/MessageDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Dialogs/TextBox/TextBoxDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/AttachmentFilterPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />

app/Desktop/App.axaml

@@ -36,6 +36,20 @@
         <Style Selector="TextBox:disabled"><!-- TODO bug in Avalonia (https://github.com/AvaloniaUI/Avalonia/pull/7792) -->
             <Setter Property="Foreground" Value="{DynamicResource TextControlForegroundDisabled}" />
         </Style>
+        <Style Selector="TextBox:error DataValidationErrors">
+            <Style.Resources>
+                <ControlTemplate x:Key="InlineDataValidationContentTemplate" TargetType="DataValidationErrors">
+                    <ContentPresenter Name="PART_ContentPresenter"
+                                      Padding="{TemplateBinding Padding}"
+                                      Background="{TemplateBinding Background}"
+                                      BorderThickness="{TemplateBinding BorderThickness}"
+                                      CornerRadius="{TemplateBinding CornerRadius}"
+                                      Content="{TemplateBinding Content}"
+                                      ContentTemplate="{TemplateBinding ContentTemplate}" />
+                </ControlTemplate>
+            </Style.Resources>
+            <Setter Property="Template" Value="{StaticResource InlineDataValidationContentTemplate}" />
+        </Style>
 
         <Style Selector="Expander /template/ ToggleButton#ExpanderHeader">
             <Setter Property="HorizontalContentAlignment" Value="Left" />

app/Desktop/Desktop.csproj

@@ -31,6 +31,10 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Version.cs" Link="Version.cs" />
+    <Compile Update="Dialogs\TextBox\TextBoxDialog.axaml.cs">
+      <DependentUpon>CheckBoxDialog.axaml</DependentUpon>
+      <SubType>Code</SubType>
+    </Compile>
   </ItemGroup>
   <ItemGroup>
     <AvaloniaResource Include="Resources/icon.ico" />

app/Desktop/Dialogs/TextBox/TextBoxDialog.axaml

@@ -0,0 +1,56 @@
+<Window xmlns="https://github.com/avaloniaui"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:namespace="clr-namespace:DHT.Desktop.Dialogs.TextBox"
+        mc:Ignorable="d" d:DesignWidth="500"
+        x:Class="DHT.Desktop.Dialogs.TextBox.TextBoxDialog"
+        Title="{Binding Title}"
+        Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
+        Width="500" SizeToContent="Height" CanResize="False"
+        WindowStartupLocation="CenterOwner">
+
+    <Window.DataContext>
+        <namespace:TextBoxDialogModel />
+    </Window.DataContext>
+
+    <Window.Styles>
+        <Style Selector="Panel.buttons">
+            <Setter Property="Margin" Value="0 20 0 0" />
+        </Style>
+        <Style Selector="Panel.buttons > WrapPanel.right">
+            <Setter Property="HorizontalAlignment" Value="Right" />
+        </Style>
+        <Style Selector="Panel.buttons Button">
+            <Setter Property="MinWidth" Value="80" />
+            <Setter Property="Margin" Value="8 0 0 0" />
+        </Style>
+    </Window.Styles>
+
+    <StackPanel Margin="20">
+        <ScrollViewer MaxHeight="400">
+            <StackPanel Spacing="10">
+                <TextBlock Text="{Binding Description}" TextWrapping="Wrap" />
+                <ItemsRepeater Items="{Binding Items}">
+                    <ItemsRepeater.ItemTemplate>
+                        <DataTemplate>
+                            <DockPanel Margin="0 5 25 0">
+                                <TextBox Name="Input" Text="{Binding Value}" Width="180" VerticalAlignment="Top" DockPanel.Dock="Right" />
+                                <Label Target="Input" VerticalAlignment="Center" DockPanel.Dock="Left">
+                                    <TextBlock Text="{Binding Title}" TextWrapping="Wrap" />
+                                </Label>
+                            </DockPanel>
+                        </DataTemplate>
+                    </ItemsRepeater.ItemTemplate>
+                </ItemsRepeater>
+            </StackPanel>
+        </ScrollViewer>
+        <Panel Classes="buttons">
+            <WrapPanel Classes="right">
+                <Button Click="ClickOk" IsEnabled="{Binding !HasErrors}">OK</Button>
+                <Button Click="ClickCancel">Cancel</Button>
+            </WrapPanel>
+        </Panel>
+    </StackPanel>
+
+</Window>

app/Desktop/Dialogs/TextBox/TextBoxDialog.axaml.cs

@@ -0,0 +1,31 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia;
+using Avalonia.Controls;
+using Avalonia.Interactivity;
+using Avalonia.Markup.Xaml;
+using DHT.Desktop.Dialogs.Message;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+	public sealed class TextBoxDialog : Window {
+		public TextBoxDialog() {
+			InitializeComponent();
+			#if DEBUG
+			this.AttachDevTools();
+			#endif
+		}
+
+		private void InitializeComponent() {
+			AvaloniaXamlLoader.Load(this);
+		}
+
+		public void ClickOk(object? sender, RoutedEventArgs e) {
+			Close(DialogResult.OkCancel.Ok);
+		}
+
+		public void ClickCancel(object? sender, RoutedEventArgs e) {
+			Close(DialogResult.OkCancel.Cancel);
+		}
+	}
+}
+

app/Desktop/Dialogs/TextBox/TextBoxDialogModel.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	class TextBoxDialogModel : BaseModel {
+		public string Title { get; init; } = "";
+		public string Description { get; init; } = "";
+
+		private IReadOnlyList<TextBoxItem> items = Array.Empty<TextBoxItem>();
+
+		public IReadOnlyList<TextBoxItem> Items {
+			get => items;
+
+			protected set {
+				foreach (var item in items) {
+					item.ErrorsChanged -= OnItemErrorsChanged;
+				}
+
+				items = value;
+
+				foreach (var item in items) {
+					item.ErrorsChanged += OnItemErrorsChanged;
+				}
+			}
+		}
+		
+		public bool HasErrors => Items.Any(static item => !item.IsValid);
+
+		private void OnItemErrorsChanged(object? sender, DataErrorsChangedEventArgs e) {
+			OnPropertyChanged(nameof(HasErrors));
+		}
+	}
+
+	sealed class TextBoxDialogModel<T> : TextBoxDialogModel {
+		public new IReadOnlyList<TextBoxItem<T>> Items { get; }
+
+		public IEnumerable<TextBoxItem<T>> ValidItems => Items.Where(static item => item.IsValid);
+
+		public TextBoxDialogModel(IEnumerable<TextBoxItem<T>> items) {
+			this.Items = new List<TextBoxItem<T>>(items);
+			base.Items = this.Items;
+		}
+	}
+}

app/Desktop/Dialogs/TextBox/TextBoxItem.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections;
+using System.ComponentModel;
+using DHT.Utils.Models;
+
+namespace DHT.Desktop.Dialogs.TextBox {
+	class TextBoxItem : BaseModel, INotifyDataErrorInfo {
+		public string Title { get; init; } = "";
+		public object? Item { get; init; } = null;
+		
+		public Func<string, bool> ValidityCheck { get; init; } = static _ => true;
+		public bool IsValid => ValidityCheck(Value);
+
+		private string value = string.Empty;
+
+		public string Value {
+			get => this.value;
+			set {
+				Change(ref this.value, value);
+				ErrorsChanged?.Invoke(this, new DataErrorsChangedEventArgs(nameof(Value)));
+			}
+		}
+
+		public IEnumerable GetErrors(string? propertyName) {
+			if (propertyName == nameof(Value) && !IsValid) {
+				yield return string.Empty;
+			}
+		}
+
+		public bool HasErrors => !IsValid;
+		public event EventHandler<DataErrorsChangedEventArgs>? ErrorsChanged;
+	}
+
+	sealed class TextBoxItem<T> : TextBoxItem {
+		public new T Item { get; }
+
+		public TextBoxItem(T item) {
+			this.Item = item;
+			base.Item = item;
+		}
+	}
+}

app/Desktop/Main/Pages/DatabasePage.axaml

@@ -24,6 +24,7 @@
         <WrapPanel>
             <Button Command="{Binding OpenDatabaseFolder}">Open Database Folder</Button>
             <Button Command="{Binding MergeWithDatabase}">Merge with Database(s)...</Button>
+            <Button Command="{Binding ImportLegacyArchive}">Import Legacy Archive(s)...</Button>
         </WrapPanel>
     </StackPanel>
 

app/Desktop/Main/Pages/DatabasePageModel.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -9,7 +11,10 @@ using Avalonia.Threading;
 using DHT.Desktop.Common;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
+using DHT.Desktop.Dialogs.TextBox;
+using DHT.Server.Data;
 using DHT.Server.Database;
+using DHT.Server.Database.Import;
 using DHT.Utils.Logging;
 using DHT.Utils.Models;
 
@@ -112,6 +117,73 @@ namespace DHT.Desktop.Main.Pages {
 			});
 		}
 
+		public async void ImportLegacyArchive() {
+			var fileDialog = new OpenFileDialog {
+				Title = "Open Legacy DHT Archive",
+				Directory = Path.GetDirectoryName(Db.Path),
+				AllowMultiple = true
+			};
+
+			string[]? paths = await fileDialog.ShowAsync(window);
+			if (paths == null || paths.Length == 0) {
+				return;
+			}
+
+			ProgressDialog progressDialog = new ProgressDialog();
+			progressDialog.DataContext = new ProgressDialogModel(async callback => await ImportLegacyArchiveFromPaths(Db, paths, progressDialog, callback)) {
+				Title = "Legacy Archive Import"
+			};
+
+			await progressDialog.ShowDialog(window);
+		}
+
+		private static async Task ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
+			var fakeSnowflake = new FakeSnowflake();
+
+			await PerformImport(target, paths, dialog, callback, "Legacy Archive Import", "Legacy Archive Error", "archive file", async path => {
+				await using var jsonStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+				
+				return await LegacyArchiveImport.Read(jsonStream, target, fakeSnowflake, async servers => {
+					SynchronizationContext? prevSyncContext = SynchronizationContext.Current;
+					SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext());
+					Dictionary<DHT.Server.Data.Server, ulong>? result = await Dispatcher.UIThread.InvokeAsync(() => AskForServerIds(dialog, servers));
+					SynchronizationContext.SetSynchronizationContext(prevSyncContext);
+					return result;
+				});
+			});
+		}
+
+		private static async Task<Dictionary<DHT.Server.Data.Server, ulong>?> AskForServerIds(Window window, DHT.Server.Data.Server[] servers) {
+			static bool IsValidSnowflake(string value) {
+				return string.IsNullOrEmpty(value) || ulong.TryParse(value, out _);
+			}
+			
+			var items = new List<TextBoxItem<DHT.Server.Data.Server>>();
+
+			foreach (var server in servers.OrderBy(static server => server.Type).ThenBy(static server => server.Name)) {
+				items.Add(new TextBoxItem<DHT.Server.Data.Server>(server) {
+					Title = server.Name + " (" + ServerTypes.ToNiceString(server.Type) + ")",
+					ValidityCheck = IsValidSnowflake
+				});
+			}
+
+			var model = new TextBoxDialogModel<DHT.Server.Data.Server>(items) {
+				Title = "Imported Server IDs",
+				Description = "Please fill in the IDs of servers and direct messages. First enable Developer Mode in Discord, then right-click each server or direct message, click 'Copy ID', and paste it into the input field. If a server no longer exists, leave its input field empty to use a random ID."
+			};
+
+			var dialog = new TextBoxDialog { DataContext = model };
+			var result = await dialog.ShowDialog<DialogResult.OkCancel>(window);
+
+			if (result != DialogResult.OkCancel.Ok) {
+				return null;
+			}
+
+			return model.ValidItems
+			            .Where(static item => !string.IsNullOrEmpty(item.Value))
+			            .ToDictionary(static item => item.Item, static item => ulong.Parse(item.Value));
+		}
+
 		private static async Task PerformImport(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback, string neutralDialogTitle, string errorDialogTitle, string itemName, Func<string, Task<bool>> performImport) {
 			int total = paths.Length;
 			var oldStatistics = target.SnapshotStatistics();

app/Resources/Viewer/scripts/discord.js

@@ -26,6 +26,7 @@ const DISCORD = (function() {
 	let templateUserAvatar;
 	let templateAttachmentDownload;
 	let templateEmbedImage;
+	let templateEmbedImageWithSize;
 	let templateEmbedRich;
 	let templateEmbedRichNoDescription;
 	let templateEmbedUrl;
@@ -64,6 +65,19 @@ const DISCORD = (function() {
 		return "<p>" + processed + "</p>";
 	};
 	
+	const getImageEmbed = function(url, image) {
+		if (!SETTINGS.enableImagePreviews) {
+			return "";
+		}
+		
+		if (image.width && image.height) {
+			return templateEmbedImageWithSize.apply({ url, src: image.url, width: image.width, height: image.height });
+		}
+		else {
+			return templateEmbedImage.apply({ url, src: image.url });
+		}
+	};
+	
 	return {
 		setup() {
 			templateChannelServer = new TEMPLATE([
@@ -114,7 +128,12 @@ const DISCORD = (function() {
 			
 			// noinspection HtmlUnknownTarget
 			templateEmbedImage = new TEMPLATE([
-				"<a href='{url}' class='embed thumbnail'><img src='{src}' alt='(image attachment not found)'></a><br>"
+				"<a href='{url}' class='embed thumbnail'><img src='{src}' alt='(image attachment is loading...)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
+			].join(""));
+			
+			// noinspection HtmlUnknownTarget
+			templateEmbedImageWithSize = new TEMPLATE([
+				"<a href='{url}' class='embed thumbnail'><img src='{src}' width='{width}' height='{height}' alt='(image attachment is loading...)' onerror='DISCORD.handleImageLoadError(this)'></a><br>"
 			].join(""));
 			
 			// noinspection HtmlUnknownTarget
@@ -145,6 +164,12 @@ const DISCORD = (function() {
 			].join(""));
 		},
 		
+		handleImageLoadError(ele) {
+			// noinspection JSUnusedGlobalSymbols
+			ele.onerror = null;
+			ele.setAttribute("alt", "(image attachment not found)");
+		},
+		
 		isImageAttachment(attachment) {
 			const dot = attachment.url.lastIndexOf(".");
 			const ext = dot === -1 ? "" : attachment.url.substring(dot).toLowerCase();
@@ -183,10 +208,10 @@ const DISCORD = (function() {
 							return templateEmbedUnsupported.apply(embed);
 						}
 						else if ("image" in embed && embed.image.url) {
-							return SETTINGS.enableImagePreviews ? templateEmbedImage.apply({ url: embed.url, src: embed.image.url }) : "";
+							return getImageEmbed(embed.url, embed.image);
 						}
 						else if ("thumbnail" in embed && embed.thumbnail.url) {
-							return SETTINGS.enableImagePreviews ? templateEmbedImage.apply({ url: embed.url, src: embed.thumbnail.url }) : "";
+							return getImageEmbed(embed.url, embed.thumbnail);
 						}
 						else if ("title" in embed && "description" in embed) {
 							return templateEmbedRich.apply(embed);

app/Resources/Viewer/styles/messages.css

@@ -112,6 +112,7 @@
 }
 
 .message .thumbnail img {
+  width: auto;
   max-width: 100%;
   max-height: 320px;
   border-radius: 3px;

app/Server/Data/ServerType.cs

@@ -24,6 +24,15 @@ namespace DHT.Server.Data {
 			};
 		}
 
+		public static string ToNiceString(ServerType? type) {
+			return type switch {
+				ServerType.Server        => "Server",
+				ServerType.Group         => "Group",
+				ServerType.DirectMessage => "DM",
+				_                        => "Unknown"
+			};
+		}
+
 		internal static string ToJsonViewerString(ServerType? type) {
 			return type switch {
 				ServerType.Server        => "server",

app/Server/Database/DatabaseExtensions.cs

@@ -1,16 +1,11 @@
+using System.Collections.Generic;
 using DHT.Server.Data;
 
 namespace DHT.Server.Database {
 	public static class DatabaseExtensions {
 		public static void AddFrom(this IDatabaseFile target, IDatabaseFile source) {
-			foreach (var server in source.GetAllServers()) {
+			target.AddServers(source.GetAllServers());
-				target.AddServer(server);
+			target.AddChannels(source.GetAllChannels());
-			}
-
-			foreach (var channel in source.GetAllChannels()) {
-				target.AddChannel(channel);
-			}
-
 			target.AddUsers(source.GetAllUsers().ToArray());
 			target.AddMessages(source.GetMessages().ToArray());
 			
@@ -18,5 +13,17 @@ namespace DHT.Server.Database {
 				target.AddDownload(download.Status == DownloadStatus.Success ? source.GetDownloadWithData(download) : download);
 			}
 		}
+
+		internal static void AddServers(this IDatabaseFile target, IEnumerable<Data.Server> servers) {
+			foreach (var server in servers) {
+				target.AddServer(server);
+			}
+		}
+
+		internal static void AddChannels(this IDatabaseFile target, IEnumerable<Channel> channels) {
+			foreach (var channel in channels) {
+				target.AddChannel(channel);
+			}
+		}
 	}
 }

app/Server/Database/DummyDatabaseFile.cs

@@ -45,6 +45,10 @@ namespace DHT.Server.Database {
 			return new();
 		}
 
+		public HashSet<ulong> GetMessageIds(MessageFilter? filter = null) {
+			return new();
+		}
+
 		public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {}
 
 		public int CountAttachments(AttachmentFilter? filter = null) {

app/Server/Database/IDatabaseFile.cs

@@ -23,6 +23,7 @@ namespace DHT.Server.Database {
 		void AddMessages(Message[] messages);
 		int CountMessages(MessageFilter? filter = null);
 		List<Message> GetMessages(MessageFilter? filter = null);
+		HashSet<ulong> GetMessageIds(MessageFilter? filter = null);
 		void RemoveMessages(MessageFilter filter, FilterRemovalMode mode);
 
 		int CountAttachments(AttachmentFilter? filter = null);

app/Server/Database/Import/FakeSnowflake.cs

@@ -0,0 +1,21 @@
+using System;
+
+namespace DHT.Server.Database.Import {
+	/// <summary>
+	/// https://discord.com/developers/docs/reference#snowflakes
+	/// </summary>
+	public sealed class FakeSnowflake {
+		private const ulong DiscordEpoch = 1420070400000UL;
+			
+		private ulong id;
+
+		public FakeSnowflake() {
+			var unixMillis = (ulong) (DateTime.UtcNow.Subtract(DateTime.UnixEpoch).Ticks / TimeSpan.TicksPerMillisecond);
+			this.id = (unixMillis - DiscordEpoch) << 22;
+		}
+
+		internal ulong Next() {
+			return id++;
+		}
+	}
+}

app/Server/Database/Import/LegacyArchiveImport.cs

@@ -0,0 +1,263 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using DHT.Server.Data;
+using DHT.Utils.Collections;
+using DHT.Utils.Http;
+using DHT.Utils.Logging;
+using Microsoft.AspNetCore.StaticFiles;
+
+namespace DHT.Server.Database.Import {
+	public static class LegacyArchiveImport {
+		private static readonly Log Log = Log.ForType(typeof(LegacyArchiveImport));
+
+		private static readonly FileExtensionContentTypeProvider ContentTypeProvider = new ();
+
+		public static async Task<bool> Read(Stream stream, IDatabaseFile db, FakeSnowflake fakeSnowflake, Func<Data.Server[], Task<Dictionary<Data.Server, ulong>?>> askForServerIds) {
+			var perf = Log.Start();
+			var root = await JsonSerializer.DeserializeAsync<JsonElement>(stream);
+
+			try {
+				var meta = root.RequireObject("meta");
+				var data = root.RequireObject("data");
+
+				perf.Step("Deserialize JSON");
+
+				var users = ReadUserList(meta);
+				var servers = ReadServerList(meta, fakeSnowflake);
+
+				var newServersOnly = new HashSet<Data.Server>(servers);
+				var oldServersById = db.GetAllServers().ToDictionary(static server => server.Id, static server => server);
+
+				var oldChannels = db.GetAllChannels();
+				var oldChannelsById = oldChannels.ToDictionary(static channel => channel.Id, static channel => channel);
+
+				foreach (var (channelId, serverIndex) in ReadChannelToServerIndexMapping(meta, servers)) {
+					if (oldChannelsById.TryGetValue(channelId, out var oldChannel) && oldServersById.TryGetValue(oldChannel.Server, out var oldServer) && newServersOnly.Remove(servers[serverIndex])) {
+						servers[serverIndex] = oldServer;
+					}
+				}
+
+				perf.Step("Read server and user list");
+
+				if (newServersOnly.Count > 0) {
+					var askedServerIds = await askForServerIds(newServersOnly.ToArray());
+					if (askedServerIds == null) {
+						return false;
+					}
+
+					perf.Step("Ask for server IDs");
+
+					for (var i = 0; i < servers.Length; i++) {
+						var server = servers[i];
+						if (askedServerIds.TryGetValue(server, out var serverId)) {
+							servers[i] = new Data.Server {
+								Id = serverId,
+								Name = server.Name,
+								Type = server.Type
+							};
+						}
+					}
+				}
+
+				var channels = ReadChannelList(meta, servers);
+
+				perf.Step("Read channel list");
+
+				var oldMessageIds = db.GetMessageIds();
+				var newMessages = channels.SelectMany(channel => ReadMessages(data, channel, users, fakeSnowflake))
+				                          .Where(message => !oldMessageIds.Contains(message.Id))
+				                          .ToArray();
+
+				perf.Step("Read messages");
+
+				db.AddUsers(users);
+				db.AddServers(servers);
+				db.AddChannels(channels);
+				db.AddMessages(newMessages);
+
+				perf.Step("Import into database");
+			} catch (HttpException e) {
+				throw new JsonException(e.Message);
+			}
+
+			perf.End();
+			return true;
+		}
+
+		private static User[] ReadUserList(JsonElement meta) {
+			const string UsersPath = "meta.users[]";
+
+			static ulong ParseUserIndex(JsonElement element, int index) {
+				return ulong.Parse(element.GetString() ?? throw new JsonException("Expected key 'meta.userindex[" + index + "]' to be a string."));
+			}
+
+			var userindex = meta.RequireArray("userindex", "meta")
+			                    .Select(static (item, index) => (ParseUserIndex(item, index), index))
+			                    .ToDictionary();
+
+			var users = new User[userindex.Count];
+
+			foreach (var item in meta.RequireObject("users", "meta").EnumerateObject()) {
+				var path = UsersPath + "." + item.Name;
+				var userId = ulong.Parse(item.Name);
+				var userObj = item.Value;
+
+				users[userindex[userId]] = new User {
+					Id = userId,
+					Name = userObj.RequireString("name", path),
+					AvatarUrl = userObj.HasKey("avatar") ? userObj.RequireString("avatar", path) : null,
+					Discriminator = userObj.HasKey("tag") ? userObj.RequireString("tag", path) : null
+				};
+			}
+
+			return users;
+		}
+
+		private static Data.Server[] ReadServerList(JsonElement meta, FakeSnowflake fakeSnowflake) {
+			const string ServersPath = "meta.servers[]";
+
+			return meta.RequireArray("servers", "meta").Select(serverObj => new Data.Server {
+				Id = fakeSnowflake.Next(),
+				Name = serverObj.RequireString("name", ServersPath),
+				Type = ServerTypes.FromString(serverObj.RequireString("type", ServersPath))
+			}).ToArray();
+		}
+
+		private const string ChannelsPath = "meta.channels";
+
+		private static Dictionary<ulong, int> ReadChannelToServerIndexMapping(JsonElement meta, Data.Server[] servers) {
+			return meta.RequireObject("channels", "meta").EnumerateObject().Select(item => {
+				var path = ChannelsPath + "." + item.Name;
+				var channelId = ulong.Parse(item.Name);
+				var channelObj = item.Value;
+
+				return (channelId, channelObj.RequireInt("server", path, min: 0, max: servers.Length - 1));
+			}).ToDictionary();
+		}
+
+		private static Channel[] ReadChannelList(JsonElement meta, Data.Server[] servers) {
+			return meta.RequireObject("channels", "meta").EnumerateObject().Select(item => {
+				var path = ChannelsPath + "." + item.Name;
+				var channelId = ulong.Parse(item.Name);
+				var channelObj = item.Value;
+
+				return new Channel {
+					Id = channelId,
+					Server = servers[channelObj.RequireInt("server", path, min: 0, max: servers.Length - 1)].Id,
+					Name = channelObj.RequireString("name", path),
+					Position = channelObj.HasKey("position") ? channelObj.RequireInt("position", path, min: 0) : null,
+					Topic = channelObj.HasKey("topic") ? channelObj.RequireString("topic", path) : null,
+					Nsfw = channelObj.HasKey("nsfw") ? channelObj.RequireBool("nsfw", path) : null
+				};
+			}).ToArray();
+		}
+
+		private static Message[] ReadMessages(JsonElement data, Channel channel, User[] users, FakeSnowflake fakeSnowflake) {
+			const string DataPath = "data";
+
+			var channelId = channel.Id;
+			var channelIdStr = channelId.ToString();
+
+			var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null;
+			if (messagesObj == null) {
+				return Array.Empty<Message>();
+			}
+
+			return messagesObj.Value.EnumerateObject().Select(item => {
+				var path = DataPath + "." + item.Name;
+				var messageId = ulong.Parse(item.Name);
+				var messageObj = item.Value;
+
+				return new Message {
+					Id = messageId,
+					Sender = users[messageObj.RequireInt("u", path, min: 0, max: users.Length - 1)].Id,
+					Channel = channelId,
+					Text = messageObj.HasKey("m") ? messageObj.RequireString("m", path) : string.Empty,
+					Timestamp = messageObj.RequireLong("t", path),
+					EditTimestamp = messageObj.HasKey("te") ? messageObj.RequireLong("te", path) : null,
+					RepliedToId = messageObj.HasKey("r") ? messageObj.RequireSnowflake("r", path) : null,
+					Attachments = messageObj.HasKey("a") ? ReadMessageAttachments(messageObj.RequireArray("a", path), fakeSnowflake, path + ".a[]").ToImmutableArray() : ImmutableArray<Attachment>.Empty,
+					Embeds = messageObj.HasKey("e") ? ReadMessageEmbeds(messageObj.RequireArray("e", path), path + ".e[]").ToImmutableArray() : ImmutableArray<Embed>.Empty,
+					Reactions = messageObj.HasKey("re") ? ReadMessageReactions(messageObj.RequireArray("re", path), path + ".re[]").ToImmutableArray() : ImmutableArray<Reaction>.Empty
+				};
+			}).ToArray();
+		}
+
+		[SuppressMessage("ReSharper", "ConvertToLambdaExpression")]
+		private static IEnumerable<Attachment> ReadMessageAttachments(JsonElement.ArrayEnumerator attachmentsArray, FakeSnowflake fakeSnowflake, string path) {
+			return attachmentsArray.Select(attachmentObj => {
+				string url = attachmentObj.RequireString("url", path);
+				string name = url[(url.LastIndexOf('/') + 1)..];
+				string? type = ContentTypeProvider.TryGetContentType(name, out var contentType) ? contentType : null;
+
+				return new Attachment {
+					Id = fakeSnowflake.Next(),
+					Name = name,
+					Type = type,
+					Url = url,
+					Size = 0 // unknown size
+				};
+			}).DistinctByKeyStable(static attachment => {
+				// Some Discord messages have duplicate attachments with the same id for unknown reasons.
+				return attachment.Id;
+			});
+		}
+
+		private static IEnumerable<Embed> ReadMessageEmbeds(JsonElement.ArrayEnumerator embedsArray, string path) {
+			// Some rich embeds are missing URLs which causes a missing 'url' key.
+			return embedsArray.Where(static embedObj => embedObj.HasKey("url")).Select(embedObj => {
+				string url = embedObj.RequireString("url", path);
+				string type = embedObj.RequireString("type", path);
+
+				var embedJson = new Dictionary<string, object> {
+					{ "url", url },
+					{ "type", type },
+					{ "dht_legacy", true }
+				};
+
+				if (type == "image") {
+					embedJson["image"] = new Dictionary<string, string> {
+						{ "url", url }
+					};
+				}
+				else if (type == "rich") {
+					if (embedObj.HasKey("t")) {
+						embedJson["title"] = embedObj.RequireString("t", path);
+					}
+
+					if (embedObj.HasKey("d")) {
+						embedJson["description"] = embedObj.RequireString("d", path);
+					}
+				}
+
+				return new Embed {
+					Json = JsonSerializer.Serialize(embedJson)
+				};
+			});
+		}
+
+		private static IEnumerable<Reaction> ReadMessageReactions(JsonElement.ArrayEnumerator reactionsArray, string path) {
+			return reactionsArray.Select(reactionObj => {
+				var id = reactionObj.HasKey("id") ? reactionObj.RequireSnowflake("id", path) : (ulong?) null;
+				var name = reactionObj.HasKey("n") ? reactionObj.RequireString("n", path) : null;
+
+				if (id == null && name == null) {
+					throw new JsonException("Expected key '" + path + ".id' and/or '" + path + ".n' to be present.");
+				}
+
+				return new Reaction {
+					EmojiId = id,
+					EmojiName = name,
+					EmojiFlags = reactionObj.HasKey("an") && reactionObj.RequireBool("an", path) ? EmojiFlags.Animated : EmojiFlags.None,
+					Count = reactionObj.RequireInt("c", path, min: 0)
+				};
+			});
+		}
+	}
+}

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -386,6 +386,22 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 			return list;
 		}
 
+		public HashSet<ulong> GetMessageIds(MessageFilter? filter = null) {
+			var perf = log.Start();
+			var ids = new HashSet<ulong>();
+
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT message_id FROM messages" + filter.GenerateWhereClause());
+			using var reader = cmd.ExecuteReader();
+
+			while (reader.Read()) {
+				ids.Add(reader.GetUint64(0));
+			}
+
+			perf.End();
+			return ids;
+		}
+
 		public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {
 			var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
 			

app/Utils/Collections/LinqExtensions.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace DHT.Utils.Collections {
 	public static class LinqExtensions {
@@ -14,5 +15,9 @@ namespace DHT.Utils.Collections {
 				}
 			}
 		}
+
+		public static Dictionary<TKey, TValue> ToDictionary<TKey, TValue>(this IEnumerable<(TKey, TValue)> collection) where TKey : notnull {
+			return collection.ToDictionary(static item => item.Item1, static item => item.Item2);
+		}
 	}
 }

app/Version.cs

@@ -7,6 +7,6 @@ using DHT.Utils;
 
 namespace DHT.Utils {
 	static class Version {
-		public const string Tag = "36.0.0.0";
+		public const string Tag = "36.1.0.0";
 	}
 }