Release v39.0

Closes #232

app/.idea/.idea.DiscordHistoryTracker/.idea/codeStyles/Project.xml

@@ -29,9 +29,6 @@
     <H2CodeStyleSettings version="6">
       <option name="USE_GENERIC_STYLE" value="true" />
     </H2CodeStyleSettings>
-    <H2CodeStyleSettings version="6">
-      <option name="USE_GENERIC_STYLE" value="true" />
-    </H2CodeStyleSettings>
     <HSQLCodeStyleSettings version="6">
       <option name="USE_GENERIC_STYLE" value="true" />
     </HSQLCodeStyleSettings>

app/.idea/.idea.DiscordHistoryTracker/.idea/runConfigurations/Desktop.xml

@@ -1,8 +1,8 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Desktop" type="DotNetProject" factoryName=".NET Project">
-    <option name="EXE_PATH" value="$PROJECT_DIR$/Desktop/bin/Debug/net5.0/DiscordHistoryTracker.exe" />
+    <option name="EXE_PATH" value="$PROJECT_DIR$/.artifacts/bin/Desktop/debug/DiscordHistoryTracker.exe" />
     <option name="PROGRAM_PARAMETERS" value="" />
-    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/Desktop/bin/Debug/net5.0" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.artifacts/bin/Desktop/debug" />
     <option name="PASS_PARENT_ENVS" value="1" />
     <option name="USE_EXTERNAL_CONSOLE" value="0" />
     <option name="USE_MONO" value="0" />
@@ -12,7 +12,7 @@
     <option name="PROJECT_ARGUMENTS_TRACKING" value="1" />
     <option name="PROJECT_WORKING_DIRECTORY_TRACKING" value="1" />
     <option name="PROJECT_KIND" value="DotNetCore" />
-    <option name="PROJECT_TFM" value="net5.0" />
+    <option name="PROJECT_TFM" value="net8.0" />
     <method v="2">
       <option name="Build" />
     </method>

app/.idea/.idea.DiscordHistoryTracker/.idea/runConfigurations/Minify.xml

@@ -1,23 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Minify" type="PythonConfigurationType" factoryName="Python">
-    <module name="rider.module" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <envs>
-      <env name="PYTHONUNBUFFERED" value="1" />
-    </envs>
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
-    <option name="IS_MODULE_SDK" value="false" />
-    <option name="ADD_CONTENT_ROOTS" value="false" />
-    <option name="ADD_SOURCE_ROOTS" value="false" />
-    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/minify.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2" />
-  </configuration>
-</component>

app/Desktop/App.axaml

@@ -2,7 +2,8 @@
              xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
              xmlns:common="clr-namespace:DHT.Desktop.Common"
              xmlns:system="clr-namespace:System;assembly=System.Runtime"
-             x:Class="DHT.Desktop.App">
+             x:Class="DHT.Desktop.App"
+             RequestedThemeVariant="Light">
 
     <Application.Styles>
 

app/Desktop/Desktop.csproj

@@ -14,13 +14,13 @@
   </PropertyGroup>
   
   <ItemGroup>
-    <PackageReference Include="Avalonia" Version="11.0.5" />
+    <PackageReference Include="Avalonia" Version="11.0.6" />
-    <PackageReference Include="Avalonia.Controls.DataGrid" Version="11.0.5" />
+    <PackageReference Include="Avalonia.Controls.DataGrid" Version="11.0.6" />
-    <PackageReference Include="Avalonia.Controls.ItemsRepeater" Version="11.0.5" />
+    <PackageReference Include="Avalonia.Controls.ItemsRepeater" Version="11.0.6" />
-    <PackageReference Include="Avalonia.Desktop" Version="11.0.5" />
+    <PackageReference Include="Avalonia.Desktop" Version="11.0.6" />
-    <PackageReference Include="Avalonia.Diagnostics" Version="11.0.5" Condition=" '$(Configuration)' == 'Debug' " />
+    <PackageReference Include="Avalonia.Diagnostics" Version="11.0.6" Condition=" '$(Configuration)' == 'Debug' " />
-    <PackageReference Include="Avalonia.Fonts.Inter" Version="11.0.5" />
+    <PackageReference Include="Avalonia.Fonts.Inter" Version="11.0.6" />
-    <PackageReference Include="Avalonia.Themes.Fluent" Version="11.0.5" />
+    <PackageReference Include="Avalonia.Themes.Fluent" Version="11.0.6" />
   </ItemGroup>
   
   <ItemGroup>

app/Resources/Tracker/scripts/discord.js

@@ -1,5 +1,23 @@
 // noinspection JSUnresolvedVariable
+// noinspection LocalVariableNamingConventionJS
 class DISCORD {
+	
+	// https://discord.com/developers/docs/resources/channel#channel-object-channel-types
+	static CHANNEL_TYPE = {
+		DM: 1,
+		GROUP_DM: 3,
+		ANNOUNCEMENT_THREAD: 10,
+		PUBLIC_THREAD: 11,
+		PRIVATE_THREAD: 12
+	};
+	
+	// https://discord.com/developers/docs/resources/channel#message-object-message-types
+	static MESSAGE_TYPE = {
+		DEFAULT: 0,
+		REPLY: 19,
+		THREAD_STARTER: 21
+	};
+	
 	static getMessageOuterElement() {
 		return DOM.queryReactClass("messagesWrapper");
 	}
@@ -53,7 +71,7 @@ class DISCORD {
 		 */
 		const onMessageElementsChangedLater = function() {
 			window.clearTimeout(debounceTimer);
-			debounceTimer = window.setTimeout(onMessageElementsChanged, 200);
+			debounceTimer = window.setTimeout(onMessageElementsChanged, 100);
 		};
 		
 		const observer = new MutationObserver(function () {
@@ -191,8 +209,8 @@ class DISCORD {
 				
 				// https://discord.com/developers/docs/resources/channel#channel-object-channel-types
 				switch (obj.type) {
-					case 1: type = "DM"; break;
+					case DISCORD.CHANNEL_TYPE.DM: type = "DM"; break;
-					case 3: type = "GROUP"; break;
+					case DISCORD.CHANNEL_TYPE.GROUP_DM: type = "GROUP"; break;
 					default: return null;
 				}
 				
@@ -230,7 +248,7 @@ class DISCORD {
 					}
 				};
 				
-				if (obj.parent_id) {
+				if (obj.type === DISCORD.CHANNEL_TYPE.ANNOUNCEMENT_THREAD || obj.type === DISCORD.CHANNEL_TYPE.PUBLIC_THREAD || obj.type === DISCORD.CHANNEL_TYPE.PRIVATE_THREAD) {
 					channel["extra"]["parent"] = obj.parent_id;
 				}
 				else {

app/Resources/Tracker/scripts/gui.js

@@ -86,12 +86,12 @@ const GUI = (function() {
 <label><input id='dht-cfg-autoscroll' type='checkbox'> Autoscroll</label><br>
 <br>
 <label>After reaching the first message in channel...</label><br>
-${radio("afm", "nothing", "Do Nothing")}
+${radio("afm", "nothing", "Continue Tracking")}
 ${radio("afm", "pause", "Pause Tracking")}
 ${radio("afm", "switch", "Switch to Next Channel")}
 <br>
 <label>After reaching a previously saved message...</label><br>
-${radio("asm", "nothing", "Do Nothing")}
+${radio("asm", "nothing", "Continue Tracking")}
 ${radio("asm", "pause", "Pause Tracking")}
 ${radio("asm", "switch", "Switch to Next Channel")}
 <p id='dht-cfg-note'>It is recommended to disable link and image previews to avoid putting unnecessary strain on your browser.</p>`;

app/Resources/Tracker/scripts/state.js

@@ -177,8 +177,7 @@ const STATE = (function() {
 		 * @param {DiscordMessage[]} discordMessageArray
 		 */
 		async addDiscordMessages(discordMessageArray) {
-			// https://discord.com/developers/docs/resources/channel#message-object-message-types
+			discordMessageArray = discordMessageArray.filter(msg => (msg.type === DISCORD.MESSAGE_TYPE.DEFAULT || msg.type === DISCORD.MESSAGE_TYPE.REPLY || msg.type === DISCORD.MESSAGE_TYPE.THREAD_STARTER) && msg.state === "SENT");
-			discordMessageArray = discordMessageArray.filter(msg => (msg.type === 0 || msg.type === 19 || msg.type === 21) && msg.state === "SENT");
 			
 			if (discordMessageArray.length === 0) {
 				return false;

app/Resources/Viewer/scripts/discord.js

@@ -1,7 +1,8 @@
 const DISCORD = (function() {
 	const regex = {
 		formatBold: /\*\*([\s\S]+?)\*\*(?!\*)/g,
-		formatItalic: /(.)?([_*])([\s\S]+?)\2(?!\2)/g,
+		formatItalic1: /\*([\s\S]+?)\*(?!\*)/g,
+		formatItalic2: /_([\s\S]+?)_(?!_)\b/g,
 		formatUnderline: /__([\s\S]+?)__(?!_)/g,
 		formatStrike: /~~([\s\S]+?)~~(?!~)/g,
 		formatCodeInline: /(`+)\s*([\s\S]*?[^`])\s*\1(?!`)/g,
@@ -48,7 +49,8 @@ const DISCORD = (function() {
 				.replace(regex.specialEscapedDouble, full => full.replace(/\\/g, "").replace(/(.)/g, escapeHtmlMatch))
 				.replace(regex.formatBold, "<b>$1</b>")
 				.replace(regex.formatUnderline, "<u>$1</u>")
-				.replace(regex.formatItalic, (full, pre, char, match) => pre === "\\" ? full : (pre || "") + "<i>" + match + "</i>")
+				.replace(regex.formatItalic1, "<i>$1</i>")
+				.replace(regex.formatItalic2, "<i>$1</i>")
 				.replace(regex.formatStrike, "<s>$1</s>");
 		}
 		

app/Server/Data/Attachment.cs

@@ -4,7 +4,8 @@ public readonly struct Attachment {
 	public ulong Id { get; internal init; }
 	public string Name { get; internal init; }
 	public string? Type { get; internal init; }
-	public string Url { get; internal init; }
+	public string NormalizedUrl { get; internal init; }
+	public string DownloadUrl { get; internal init; }
 	public ulong Size { get; internal init; }
 	public int? Width { get; internal init; }
 	public int? Height { get; internal init; }

app/Server/Data/Download.cs

@@ -1,30 +1,33 @@
 using System;
 using System.Net;
+using DHT.Server.Download;
 
 namespace DHT.Server.Data;
 
 public readonly struct Download {
-	internal static Download NewSuccess(string url, byte[] data) {
+	internal static Download NewSuccess(DownloadItem item, byte[] data) {
-		return new Download(url, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
+		return new Download(item.NormalizedUrl, item.DownloadUrl, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
 	}
 
-	internal static Download NewFailure(string url, HttpStatusCode? statusCode, ulong size) {
+	internal static Download NewFailure(DownloadItem item, HttpStatusCode? statusCode, ulong size) {
-		return new Download(url, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
+		return new Download(item.NormalizedUrl, item.DownloadUrl, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
 	}
 
-	public string Url { get; }
+	public string NormalizedUrl { get; }
+	public string DownloadUrl { get; }
 	public DownloadStatus Status { get; }
 	public ulong Size { get; }
 	public byte[]? Data { get; }
 
-	internal Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
+	internal Download(string normalizedUrl, string downloadUrl, DownloadStatus status, ulong size, byte[]? data = null) {
-		Url = url;
+		NormalizedUrl = normalizedUrl;
+		DownloadUrl = downloadUrl;
 		Status = status;
 		Size = size;
 		Data = data;
 	}
 
 	internal Download WithData(byte[] data) {
-		return new Download(Url, Status, Size, data);
+		return new Download(NormalizedUrl, DownloadUrl, Status, Size, data);
 	}
 }

app/Server/Database/Export/Strategy/LiveViewerExportStrategy.cs

@@ -13,6 +13,6 @@ public sealed class LiveViewerExportStrategy : IViewerExportStrategy {
 	}
 
 	public string GetAttachmentUrl(Attachment attachment) {
-		return "http://127.0.0.1:" + safePort + "/get-attachment/" + WebUtility.UrlEncode(attachment.Url) + "?token=" + safeToken;
+		return "http://127.0.0.1:" + safePort + "/get-attachment/" + WebUtility.UrlEncode(attachment.NormalizedUrl) + "?token=" + safeToken;
 	}
 }

app/Server/Database/Export/Strategy/StandaloneViewerExportStrategy.cs

@@ -8,6 +8,11 @@ public sealed class StandaloneViewerExportStrategy : IViewerExportStrategy {
 	private StandaloneViewerExportStrategy() {}
 
 	public string GetAttachmentUrl(Attachment attachment) {
-		return attachment.Url;
+		// The normalized URL will not load files from Discord CDN once the time limit is enforced.
+		
+		// The downloaded URL would work, but only for a limited time, so it is better for the links to not work
+		// rather than give users a false sense of security.
+		
+		return attachment.NormalizedUrl;
 	}
 }

app/Server/Database/Export/ViewerJsonExport.cs

@@ -170,7 +170,7 @@ public static class ViewerJsonExport {
 					obj["a"] = message.Attachments.Select(attachment => {
 						var a = new Dictionary<string, object> {
 							{ "url", strategy.GetAttachmentUrl(attachment) },
-							{ "name", Uri.TryCreate(attachment.Url, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.Url },
+							{ "name", Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl },
 						};
 
 						if (attachment is { Width: not null, Height: not null }) {

app/Server/Database/Import/LegacyArchiveImport.cs

@@ -197,7 +197,8 @@ public static class LegacyArchiveImport {
 				Id = fakeSnowflake.Next(),
 				Name = name,
 				Type = type,
-				Url = url,
+				NormalizedUrl = url,
+				DownloadUrl = url,
 				Size = 0, // unknown size
 			};
 		}).DistinctByKeyStable(static attachment => {

app/Server/Database/Sqlite/Schema.cs

@@ -1,13 +1,16 @@
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using DHT.Server.Database.Exceptions;
 using DHT.Server.Database.Sqlite.Utils;
+using DHT.Server.Download;
 using DHT.Utils.Logging;
+using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite;
 
 sealed class Schema {
-	internal const int Version = 5;
+	internal const int Version = 6;
 
 	private static readonly Log Log = Log.ForType<Schema>();
 
@@ -47,57 +50,88 @@ sealed class Schema {
 	}
 
 	private void InitializeSchemas() {
-		Execute(@"CREATE TABLE users (
+		Execute("""
-			          id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE users (
-			          name TEXT NOT NULL,
+					id            INTEGER PRIMARY KEY NOT NULL,
-			          avatar_url TEXT,
+					name          TEXT NOT NULL,
-			          discriminator TEXT)");
+					avatar_url    TEXT,
+					discriminator TEXT
+				)
+				""");
 
-		Execute(@"CREATE TABLE servers (
+		Execute("""
-			          id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE servers (
-			          name TEXT NOT NULL,
+					id   INTEGER PRIMARY KEY NOT NULL,
-			          type TEXT NOT NULL)");
+					name TEXT NOT NULL,
+					type TEXT NOT NULL
+				)
+				""");
 
-		Execute(@"CREATE TABLE channels (
+		Execute("""
-			          id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE channels (
-			          server INTEGER NOT NULL,
+					id        INTEGER PRIMARY KEY NOT NULL,
-			          name TEXT NOT NULL,
+					server    INTEGER NOT NULL,
-			          parent_id INTEGER,
+					name      TEXT NOT NULL,
-			          position INTEGER,
+					parent_id INTEGER,
-			          topic TEXT,
+					position  INTEGER,
-			          nsfw INTEGER)");
+					topic     TEXT,
+					nsfw      INTEGER
+				)
+				""");
 
-		Execute(@"CREATE TABLE messages (
+		Execute("""
-			        message_id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE messages (
-			        sender_id INTEGER NOT NULL,
+					message_id INTEGER PRIMARY KEY NOT NULL,
-			        channel_id INTEGER NOT NULL,
+					sender_id  INTEGER NOT NULL,
-			        text TEXT NOT NULL,
+					channel_id INTEGER NOT NULL,
-			        timestamp INTEGER NOT NULL)");
+					text       TEXT NOT NULL,
+					timestamp  INTEGER NOT NULL
+				)
+				""");
 
-		Execute(@"CREATE TABLE attachments (
+		Execute("""
-			        message_id INTEGER NOT NULL,
+				CREATE TABLE attachments (
-			        attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
+					message_id     INTEGER NOT NULL,
-			        name TEXT NOT NULL,
+					attachment_id  INTEGER NOT NULL PRIMARY KEY NOT NULL,
-			        type TEXT,
+					name           TEXT NOT NULL,
-			        url TEXT NOT NULL,
+					type           TEXT,
-			        size INTEGER NOT NULL,
+					normalized_url TEXT NOT NULL,
-			        width INTEGER,
+					download_url   TEXT,
-			        height INTEGER)");
+					size           INTEGER NOT NULL,
+					width          INTEGER,
+					height         INTEGER
+				)
+				""");
 
-		Execute(@"CREATE TABLE embeds (
+		Execute("""
-			        message_id INTEGER NOT NULL,
+				CREATE TABLE embeds (
-			        json TEXT NOT NULL)");
-
-		Execute(@"CREATE TABLE reactions (
 					message_id INTEGER NOT NULL,
-					emoji_id INTEGER,
+					json       TEXT NOT NULL
-					emoji_name TEXT,
+				)
+				""");
+
+		Execute("""
+				CREATE TABLE downloads (
+					normalized_url TEXT NOT NULL PRIMARY KEY,
+					download_url   TEXT,
+					status         INTEGER NOT NULL,
+					size           INTEGER NOT NULL,
+					blob           BLOB
+				)
+				""");
+		
+		Execute("""
+				CREATE TABLE reactions (
+					message_id  INTEGER NOT NULL,
+					emoji_id    INTEGER,
+					emoji_name  TEXT,
 					emoji_flags INTEGER NOT NULL,
-					count INTEGER NOT NULL)");
+					count       INTEGER NOT NULL
+				)
+				""");
 
 		CreateMessageEditTimestampTable();
 		CreateMessageRepliedToTable();
-		CreateDownloadsTable();
 
 		Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
 		Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
@@ -107,23 +141,90 @@ sealed class Schema {
 	}
 
 	private void CreateMessageEditTimestampTable() {
-		Execute(@"CREATE TABLE edit_timestamps (
+		Execute("""
-			        message_id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE edit_timestamps (
-			        edit_timestamp INTEGER NOT NULL)");
+					message_id     INTEGER PRIMARY KEY NOT NULL,
+					edit_timestamp INTEGER NOT NULL
+				)
+				""");
 	}
 
 	private void CreateMessageRepliedToTable() {
-		Execute(@"CREATE TABLE replied_to (
+		Execute("""
-			        message_id INTEGER PRIMARY KEY NOT NULL,
+				CREATE TABLE replied_to (
-			        replied_to_id INTEGER NOT NULL)");
+					message_id    INTEGER PRIMARY KEY NOT NULL,
+					replied_to_id INTEGER NOT NULL
+				)
+				""");
 	}
 
-	private void CreateDownloadsTable() {
+	private void NormalizeAttachmentUrls() {
-		Execute(@"CREATE TABLE downloads (
+		var normalizedUrls = new Dictionary<long, string>();
-                      url TEXT NOT NULL PRIMARY KEY,
+
-                      status INTEGER NOT NULL,
+		using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
-                      size INTEGER NOT NULL,
+			using var reader = selectCmd.ExecuteReader();
-                      blob BLOB)");
+			
+			while (reader.Read()) {
+				var attachmentId = reader.GetInt64(0);
+				var originalUrl = reader.GetString(1);
+				normalizedUrls[attachmentId] = DiscordCdn.NormalizeUrl(originalUrl);
+			}
+		}
+
+		using var tx = conn.BeginTransaction();
+		
+		using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
+			updateCmd.Parameters.Add(":attachment_id", SqliteType.Integer);
+			updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
+				
+			foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
+				updateCmd.Set(":attachment_id", attachmentId);
+				updateCmd.Set(":normalized_url", normalizedUrl);
+				updateCmd.ExecuteNonQuery();
+			}
+		}
+			
+		tx.Commit();
+	}
+
+	private void NormalizeDownloadUrls() {
+		var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
+		var duplicateUrlsToDelete = new HashSet<string>();
+
+		using (var selectCmd = conn.Command("SELECT url FROM downloads ORDER BY CASE WHEN status = 200 THEN 0 ELSE 1 END")) {
+			using var reader = selectCmd.ExecuteReader();
+
+			while (reader.Read()) {
+				var originalUrl = reader.GetString(0);
+				var normalizedUrl = DiscordCdn.NormalizeUrl(originalUrl);
+
+				if (!normalizedUrlsToOriginalUrls.TryAdd(normalizedUrl, originalUrl)) {
+					duplicateUrlsToDelete.Add(originalUrl);
+				}
+			}
+		}
+
+		using var tx = conn.BeginTransaction();
+		
+		using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) {
+			foreach (var duplicateUrl in duplicateUrlsToDelete) {
+				deleteCmd.Set(":url", duplicateUrl);
+				deleteCmd.ExecuteNonQuery();
+			}
+		}
+			
+		using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
+			updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
+			updateCmd.Parameters.Add(":download_url", SqliteType.Text);
+				
+			foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
+				updateCmd.Set(":normalized_url", normalizedUrl);
+				updateCmd.Set(":download_url", downloadUrl);
+				updateCmd.ExecuteNonQuery();
+			}
+		}
+			
+		tx.Commit();
 	}
 
 	private void UpgradeSchemas(int dbVersion) {
@@ -140,13 +241,19 @@ sealed class Schema {
 			CreateMessageEditTimestampTable();
 			CreateMessageRepliedToTable();
 
-			Execute(@"INSERT INTO edit_timestamps (message_id, edit_timestamp)
+			Execute("""
-						SELECT message_id, edit_timestamp FROM messages
+					INSERT INTO edit_timestamps (message_id, edit_timestamp)
-						WHERE edit_timestamp IS NOT NULL");
+					SELECT message_id, edit_timestamp
+					FROM messages
+					WHERE edit_timestamp IS NOT NULL
+					""");
 
-			Execute(@"INSERT INTO replied_to (message_id, replied_to_id)
+			Execute("""
-						SELECT message_id, replied_to_id FROM messages
+					INSERT INTO replied_to (message_id, replied_to_id)
-						WHERE replied_to_id IS NOT NULL");
+					SELECT message_id, replied_to_id
+					FROM messages
+					WHERE replied_to_id IS NOT NULL
+					""");
 
 			Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
 			Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
@@ -158,7 +265,15 @@ sealed class Schema {
 		}
 
 		if (dbVersion <= 3) {
-			CreateDownloadsTable();
+			Execute("""
+					CREATE TABLE downloads (
+						url    TEXT NOT NULL PRIMARY KEY,
+						status INTEGER NOT NULL,
+						size   INTEGER NOT NULL,
+						blob   BLOB
+					)
+					""");
+			
 			perf.Step("Upgrade to version 4");
 		}
 
@@ -168,6 +283,19 @@ sealed class Schema {
 			perf.Step("Upgrade to version 5");
 		}
 
+		if (dbVersion <= 5) {
+			Execute("ALTER TABLE attachments ADD download_url TEXT");
+			Execute("ALTER TABLE downloads ADD download_url TEXT");
+			
+			NormalizeAttachmentUrls();
+			NormalizeDownloadUrls();
+			
+			Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
+			Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
+			
+			perf.Step("Upgrade to version 6");
+		}
+
 		perf.End();
 	}
 }

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -252,7 +252,8 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 				("attachment_id", SqliteType.Integer),
 				("name", SqliteType.Text),
 				("type", SqliteType.Text),
-				("url", SqliteType.Text),
+				("normalized_url", SqliteType.Text),
+				("download_url", SqliteType.Text),
 				("size", SqliteType.Integer),
 				("width", SqliteType.Integer),
 				("height", SqliteType.Integer),
@@ -308,7 +309,8 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 						attachmentCmd.Set(":attachment_id", attachment.Id);
 						attachmentCmd.Set(":name", attachment.Name);
 						attachmentCmd.Set(":type", attachment.Type);
-						attachmentCmd.Set(":url", attachment.Url);
+						attachmentCmd.Set(":normalized_url", attachment.NormalizedUrl);
+						attachmentCmd.Set(":download_url", attachment.DownloadUrl);
 						attachmentCmd.Set(":size", attachment.Size);
 						attachmentCmd.Set(":width", attachment.Width);
 						attachmentCmd.Set(":height", attachment.Height);
@@ -363,11 +365,13 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 		var reactions = GetAllReactions();
 
 		using var conn = pool.Take();
-		using var cmd = conn.Command(@"
+		using var cmd = conn.Command($"""
-SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, et.edit_timestamp, rt.replied_to_id
+		                              SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, et.edit_timestamp, rt.replied_to_id
-FROM messages m
+		                              FROM messages m
-LEFT JOIN edit_timestamps et ON m.message_id = et.message_id
+		                              LEFT JOIN edit_timestamps et ON m.message_id = et.message_id
-LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereClause("m"));
+		                              LEFT JOIN replied_to rt ON m.message_id = rt.message_id
+		                              {filter.GenerateWhereClause("m")}
+		                              """);
 		using var reader = cmd.ExecuteReader();
 
 		while (reader.Read()) {
@@ -418,7 +422,7 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 
 	public int CountAttachments(AttachmentFilter? filter = null) {
 		using var conn = pool.Take();
-		using var cmd = conn.Command("SELECT COUNT(DISTINCT url) FROM attachments a" + filter.GenerateWhereClause("a"));
+		using var cmd = conn.Command("SELECT COUNT(DISTINCT normalized_url) FROM attachments a" + filter.GenerateWhereClause("a"));
 		using var reader = cmd.ExecuteReader();
 
 		return reader.Read() ? reader.GetInt32(0) : 0;
@@ -427,13 +431,15 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 	public void AddDownload(Data.Download download) {
 		using var conn = pool.Take();
 		using var cmd = conn.Upsert("downloads", new[] {
-			("url", SqliteType.Text),
+			("normalized_url", SqliteType.Text),
+			("download_url", SqliteType.Text),
 			("status", SqliteType.Integer),
 			("size", SqliteType.Integer),
 			("blob", SqliteType.Blob),
 		});
 
-		cmd.Set(":url", download.Url);
+		cmd.Set(":normalized_url", download.NormalizedUrl);
+		cmd.Set(":download_url", download.DownloadUrl);
 		cmd.Set(":status", (int) download.Status);
 		cmd.Set(":size", download.Size);
 		cmd.Set(":blob", download.Data);
@@ -446,15 +452,16 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 		var list = new List<Data.Download>();
 
 		using var conn = pool.Take();
-		using var cmd = conn.Command("SELECT url, status, size FROM downloads");
+		using var cmd = conn.Command("SELECT normalized_url, download_url, status, size FROM downloads");
 		using var reader = cmd.ExecuteReader();
 
 		while (reader.Read()) {
-			string url = reader.GetString(0);
+			string normalizedUrl = reader.GetString(0);
-			var status = (DownloadStatus) reader.GetInt32(1);
+			string downloadUrl = reader.GetString(1);
-			ulong size = reader.GetUint64(2);
+			var status = (DownloadStatus) reader.GetInt32(2);
+			ulong size = reader.GetUint64(3);
 
-			list.Add(new Data.Download(url, status, size));
+			list.Add(new Data.Download(normalizedUrl, downloadUrl, status, size));
 		}
 
 		return list;
@@ -462,8 +469,8 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 
 	public Data.Download GetDownloadWithData(Data.Download download) {
 		using var conn = pool.Take();
-		using var cmd = conn.Command("SELECT blob FROM downloads WHERE url = :url");
+		using var cmd = conn.Command("SELECT blob FROM downloads WHERE normalized_url = :url");
-		cmd.AddAndSet(":url", SqliteType.Text, download.Url);
+		cmd.AddAndSet(":url", SqliteType.Text, download.NormalizedUrl);
 
 		using var reader = cmd.ExecuteReader();
 
@@ -475,14 +482,15 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 		}
 	}
 
-	public DownloadedAttachment? GetDownloadedAttachment(string url) {
+	public DownloadedAttachment? GetDownloadedAttachment(string normalizedUrl) {
 		using var conn = pool.Take();
-		using var cmd = conn.Command(@"
+		using var cmd = conn.Command("""
-SELECT a.type, d.blob FROM downloads d
+		                             SELECT a.type, d.blob FROM downloads d
-LEFT JOIN attachments a ON d.url = a.url
+		                             LEFT JOIN attachments a ON d.normalized_url = a.normalized_url
-WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
+		                             WHERE d.normalized_url = :normalized_url AND d.status = :success AND d.blob IS NOT NULL
+		                             """);
 
-		cmd.AddAndSet(":url", SqliteType.Text, url);
+		cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl);
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 
 		using var reader = cmd.ExecuteReader();
@@ -499,7 +507,13 @@ WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
 
 	public void EnqueueDownloadItems(AttachmentFilter? filter = null) {
 		using var conn = pool.Take();
-		using var cmd = conn.Command("INSERT INTO downloads (url, status, size) SELECT a.url, :enqueued, MAX(a.size) FROM attachments a" + filter.GenerateWhereClause("a") + " GROUP BY a.url");
+		using var cmd = conn.Command($"""
+		                              INSERT INTO downloads (normalized_url, download_url, status, size)
+		                              SELECT a.normalized_url, a.download_url, :enqueued, MAX(a.size)
+		                              FROM attachments a
+		                              {filter.GenerateWhereClause("a")}
+		                              GROUP BY a.normalized_url
+		                              """);
 		cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
 		cmd.ExecuteNonQuery();
 	}
@@ -508,7 +522,7 @@ WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
 		var list = new List<DownloadItem>();
 
 		using var conn = pool.Take();
-		using var cmd = conn.Command("SELECT url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
+		using var cmd = conn.Command("SELECT normalized_url, download_url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
 		cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
 		cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
 
@@ -516,8 +530,9 @@ WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
 
 		while (reader.Read()) {
 			list.Add(new DownloadItem {
-				Url = reader.GetString(0),
+				NormalizedUrl = reader.GetString(0),
-				Size = reader.GetUint64(1),
+				DownloadUrl = reader.GetString(1),
+				Size = reader.GetUint64(2),
 			});
 		}
 
@@ -531,7 +546,7 @@ WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
 
 	public DownloadStatusStatistics GetDownloadStatusStatistics() {
 		static void LoadUndownloadedStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
-			using var cmd = conn.Command("SELECT IFNULL(COUNT(size), 0), IFNULL(SUM(size), 0) FROM (SELECT MAX(a.size) size FROM attachments a WHERE a.url NOT IN (SELECT d.url FROM downloads d) GROUP BY a.url)");
+			using var cmd = conn.Command("SELECT IFNULL(COUNT(size), 0), IFNULL(SUM(size), 0) FROM (SELECT MAX(a.size) size FROM attachments a WHERE a.normalized_url NOT IN (SELECT d.normalized_url FROM downloads d) GROUP BY a.normalized_url)");
 			using var reader = cmd.ExecuteReader();
 
 			if (reader.Read()) {
@@ -541,14 +556,16 @@ WHERE d.url = :url AND d.status = :success AND d.blob IS NOT NULL");
 		}
 
 		static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
-			using var cmd = conn.Command(@"SELECT
+			using var cmd = conn.Command("""
-IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
+			                             SELECT
-IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
+			                             IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
-IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
+			                             IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
-IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
+			                             IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
-IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
+			                             IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
-IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
+			                             IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
-FROM downloads");
+			                             IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
+			                             FROM downloads
+			                             """);
 			cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
 			cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 
@@ -576,7 +593,7 @@ FROM downloads");
 		var dict = new MultiDictionary<ulong, Attachment>();
 
 		using var conn = pool.Take();
-		using var cmd = conn.Command("SELECT message_id, attachment_id, name, type, url, size, width, height FROM attachments");
+		using var cmd = conn.Command("SELECT message_id, attachment_id, name, type, normalized_url, download_url, size, width, height FROM attachments");
 		using var reader = cmd.ExecuteReader();
 
 		while (reader.Read()) {
@@ -586,10 +603,11 @@ FROM downloads");
 				Id = reader.GetUint64(1),
 				Name = reader.GetString(2),
 				Type = reader.IsDBNull(3) ? null : reader.GetString(3),
-				Url = reader.GetString(4),
+				NormalizedUrl = reader.GetString(4),
-				Size = reader.GetUint64(5),
+				DownloadUrl = reader.GetString(5),
-				Width = reader.IsDBNull(6) ? null : reader.GetInt32(6),
+				Size = reader.GetUint64(6),
-				Height = reader.IsDBNull(7) ? null : reader.GetInt32(7),
+				Width = reader.IsDBNull(7) ? null : reader.GetInt32(7),
+				Height = reader.IsDBNull(8) ? null : reader.GetInt32(8),
 			});
 		}
 
@@ -677,7 +695,7 @@ FROM downloads");
 
 	private long ComputeAttachmentStatistics() {
 		using var conn = pool.Take();
-		return conn.SelectScalar("SELECT COUNT(DISTINCT url) FROM attachments") as long? ?? 0L;
+		return conn.SelectScalar("SELECT COUNT(DISTINCT normalized_url) FROM attachments") as long? ?? 0L;
 	}
 
 	private void UpdateAttachmentStatistics(long totalAttachments) {

app/Server/Database/Sqlite/SqliteFilters.cs

@@ -50,10 +50,10 @@ static class SqliteFilters {
 		}
 
 		if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyNotPresent) {
-			where.AddCondition("url NOT IN (SELECT url FROM downloads)");
+			where.AddCondition("normalized_url NOT IN (SELECT normalized_url FROM downloads)");
 		}
 		else if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyPresent) {
-			where.AddCondition("url IN (SELECT url FROM downloads)");
+			where.AddCondition("normalized_url IN (SELECT normalized_url FROM downloads)");
 		}
 
 		return where.Generate();

app/Server/Download/BackgroundDownloadThread.cs

@@ -87,16 +87,16 @@ public sealed class BackgroundDownloadThread : BaseModel {
 					FillQueue(db, queue, cancellationToken);
 
 					while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var item)) {
-						var url = item.Url;
+						var downloadUrl = item.DownloadUrl;
-						Log.Debug("Downloading " + url + "...");
+						Log.Debug("Downloading " + downloadUrl + "...");
 
 						try {
-							db.AddDownload(Data.Download.NewSuccess(url, await client.GetByteArrayAsync(url, cancellationToken)));
+							db.AddDownload(Data.Download.NewSuccess(item, await client.GetByteArrayAsync(downloadUrl, cancellationToken)));
 						} catch (HttpRequestException e) {
-							db.AddDownload(Data.Download.NewFailure(url, e.StatusCode, item.Size));
+							db.AddDownload(Data.Download.NewFailure(item, e.StatusCode, item.Size));
 							Log.Error(e);
 						} catch (Exception e) {
-							db.AddDownload(Data.Download.NewFailure(url, null, item.Size));
+							db.AddDownload(Data.Download.NewFailure(item, null, item.Size));
 							Log.Error(e);
 						} finally {
 							parameters.FireOnItemFinished(item);

app/Server/Download/DiscordCdn.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Frozen;
+
+namespace DHT.Server.Download; 
+
+static class DiscordCdn {
+	private static FrozenSet<string> CdnHosts { get; } = new [] {
+		"cdn.discordapp.com",
+		"cdn.discord.com",
+	}.ToFrozenSet();
+
+	public static string NormalizeUrl(string originalUrl) {
+		return Uri.TryCreate(originalUrl, UriKind.Absolute, out var uri) && CdnHosts.Contains(uri.Host) ? uri.GetLeftPart(UriPartial.Path) : originalUrl;
+	}
+}

app/Server/Download/DownloadItem.cs

@@ -1,6 +1,7 @@
 namespace DHT.Server.Download;
 
 public readonly struct DownloadItem {
-	public string Url { get; init; }
+	public string NormalizedUrl { get; init; }
+	public string DownloadUrl { get; init; }
 	public ulong Size { get; init; }
 }

app/Server/Endpoints/TrackMessagesEndpoint.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Server.Database;
+using DHT.Server.Download;
 using DHT.Server.Service;
 using DHT.Utils.Collections;
 using DHT.Utils.Http;
@@ -57,14 +58,18 @@ sealed class TrackMessagesEndpoint : BaseEndpoint {
 	};
 
 	[SuppressMessage("ReSharper", "ConvertToLambdaExpression")]
-	private static IEnumerable<Attachment> ReadAttachments(JsonElement.ArrayEnumerator array, string path) => array.Select(ele => new Attachment {
+	private static IEnumerable<Attachment> ReadAttachments(JsonElement.ArrayEnumerator array, string path) => array.Select(ele => {
-		Id = ele.RequireSnowflake("id", path),
+		var downloadUrl = ele.RequireString("url", path);
-		Name = ele.RequireString("name", path),
+		return new Attachment {
-		Type = ele.HasKey("type") ? ele.RequireString("type", path) : null,
+			Id = ele.RequireSnowflake("id", path),
-		Url = ele.RequireString("url", path),
+			Name = ele.RequireString("name", path),
-		Size = (ulong) ele.RequireLong("size", path),
+			Type = ele.HasKey("type") ? ele.RequireString("type", path) : null,
-		Width = ele.HasKey("width") ? ele.RequireInt("width", path) : null,
+			NormalizedUrl = DiscordCdn.NormalizeUrl(downloadUrl),
-		Height = ele.HasKey("height") ? ele.RequireInt("height", path) : null,
+			DownloadUrl = downloadUrl,
+			Size = (ulong) ele.RequireLong("size", path),
+			Width = ele.HasKey("width") ? ele.RequireInt("width", path) : null,
+			Height = ele.HasKey("height") ? ele.RequireInt("height", path) : null,
+		};
 	}).DistinctByKeyStable(static attachment => {
 		// Some Discord messages have duplicate attachments with the same id for unknown reasons.
 		return attachment.Id;

app/Version.cs

@@ -8,5 +8,5 @@ using DHT.Utils;
 namespace DHT.Utils; 
 
 static class Version {
-	public const string Tag = "38.0.0.0";
+	public const string Tag = "39.0.0.0";
 }