Fix not updating download URLs and metadata when re-tracking messages with failed downloads

References #216

app/Desktop/Main/Controls/MessageFilterPanelModel.cs

@@ -249,11 +249,8 @@ sealed partial class MessageFilterPanelModel : ObservableObject, IDisposable {
 			var checkBoxItems = new List<CheckBoxItem<ulong>>();
 
 			await foreach (var user in state.Db.Users.Get()) {
-				var name = user.Name;
-				var discriminator = user.Discriminator;
-
 				checkBoxItems.Add(new CheckBoxItem<ulong>(user.Id) {
-					Title = discriminator == null ? name : name + " #" + discriminator,
+					Title = user.DisplayName == null ? user.Name : $"{user.DisplayName} ({user.Name})",
 					IsChecked = IncludedUsers == null || IncludedUsers.Contains(user.Id)
 				});
 			}

app/Desktop/Main/Pages/DebugPageModel.cs

@@ -73,6 +73,7 @@ sealed class DebugPageModel {
 		var users = Enumerable.Range(0, userCount).Select(_ => new User {
 			Id = RandomId(rand),
 			Name = RandomName("u"),
+			DisplayName = RandomName("u"),
 			AvatarUrl = null,
 			Discriminator = rand.Next(0, 9999).ToString(),
 		}).ToArray();

app/Resources/Schemas/track-users.yml

@@ -9,6 +9,8 @@ items:
           pattern: "^[0-9]+$"
         name:
           type: string
+        displayName:
+          type: string
         avatar:
           type: string
         discriminator:

app/Resources/Tracker/scripts/state.js

@@ -69,6 +69,7 @@ const STATE = (function() {
 	 * @property {String} id
 	 * @property {String} username
 	 * @property {String} discriminator
+	 * @property {String} [globalName]
 	 * @property {String} [avatar]
 	 * @property {Boolean} [bot]
 	 */
@@ -200,6 +201,10 @@ const STATE = (function() {
 						name: user.username
 					};
 					
+					if (user.globalName) {
+						obj.displayName = user.globalName;
+					}
+					
 					if (user.avatar) {
 						obj.avatar = user.avatar;
 					}

app/Resources/Viewer/index.html

@@ -25,6 +25,8 @@
     <div id="menu">
       <button id="btn-settings">Settings</button>
       
+      <div class="splitter"></div>
+      
       <div> <!-- needed to stop the select from messing up -->
         <select id="opt-messages-per-page">
           <option value="50">50 messages per page&nbsp;</option>

app/Resources/Viewer/scripts/bootstrap.mjs

@@ -1,6 +1,7 @@
 import discord from "./discord.mjs";
 import gui from "./gui.mjs";
 import state from "./state.mjs";
+import "./polyfills.mjs";
 
 window.DISCORD = discord;
 

app/Resources/Viewer/scripts/discord.mjs

@@ -80,7 +80,7 @@ export default (function() {
 		processed = processed
 			.replace(regex.formatUrl, "<a href='$1' target='_blank' rel='noreferrer'>$1</a>")
 			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + state.getChannelName(match) + "</span>")
-			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (state.getUserTag(match) || "????") + "'>@" + state.getUserName(match) + "</span>")
+			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='" + state.getUserName(match) + "'>@" + state.getUserDisplayName(match) + "</span>")
 			.replace(regex.customEmojiStatic, (full, m1, m2) => getEmoji(m1, m2, "webp"))
 			.replace(regex.customEmojiAnimated, (full, m1, m2) => getEmoji(m1, m2, animatedEmojiExtension));
 		
@@ -129,7 +129,7 @@ export default (function() {
 			templateMessageNoAvatar = new template([
 				"<div>",
 				"<div class='reply-message'>{reply}</div>",
-				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
+				"<h2><strong class='username' title='{user.name}'>{user.displayName}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
 				"<div class='message'>{contents}{embeds}{attachments}</div>",
 				"{reactions}",
 				"</div>"
@@ -141,7 +141,7 @@ export default (function() {
 				"<div class='avatar-wrapper'>",
 				"<div class='avatar'>{avatar}</div>",
 				"<div>",
-				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
+				"<h2><strong class='username' title='{user.name}'>{user.displayName}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
 				"<div class='message'>{contents}{embeds}{attachments}</div>",
 				"{reactions}",
 				"</div>",
@@ -227,8 +227,8 @@ export default (function() {
 				if (property === "avatar") {
 					return value ? templateUserAvatar.apply(getAvatarUrlObject(value)) : "";
 				}
-				else if (property === "user.tag") {
-					return value ? value : "????";
+				else if (property === "user.displayName") {
+					return value ? value : message.user.name;
 				}
 				else if (property === "timestamp") {
 					return dom.getHumanReadableTime(value);
@@ -292,7 +292,7 @@ export default (function() {
 						return value === null ? "<span class='reply-contents reply-missing'>(replies to an unknown message)</span>" : "";
 					}
 					
-					const user = "<span class='reply-username' title='#" + (value.user.tag ? value.user.tag : "????") + "'>" + value.user.name + "</span>";
+					const user = "<span class='reply-username' title='" + value.user.name + "'>" + (value.user.displayName ?? value.user.name) + "</span>";
 					const avatar = settings.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(getAvatarUrlObject(value.avatar)) + "</span>" : "";
 					const contents = value.contents ? "<span class='reply-contents'>" + processMessageContents(value.contents) + "</span>" : "";
 					

app/Resources/Viewer/scripts/gui.mjs

@@ -243,10 +243,11 @@ export default (function() {
 			
 			const options = [];
 			
-			for (const key of Object.keys(users)) {
+			for (const id of Object.keys(users)) {
+				const user = users[id];
 				const option = document.createElement("option");
-				option.value = key;
-				option.text = users[key].name;
+				option.value = id;
+				option.text = user.displayName ? `${user.displayName} (${user.name})` : user.name;
 				options.push(option);
 			}
 			

app/Resources/Viewer/scripts/polyfills.mjs

@@ -0,0 +1,35 @@
+// https://gist.github.com/MattiasBuelens/496fc1d37adb50a733edd43853f2f60e/088f061ab79b296f29225467ae9ba86ff990195d
+
+ReadableStream.prototype.values ??= function({ preventCancel = false } = {}) {
+	const reader = this.getReader();
+	return {
+		async next() {
+			try {
+				const result = await reader.read();
+				if (result.done) {
+					reader.releaseLock();
+				}
+				return result;
+			} catch (e) {
+				reader.releaseLock();
+				throw e;
+			}
+		},
+		async return(value) {
+			if (!preventCancel) {
+				const cancelPromise = reader.cancel(value);
+				reader.releaseLock();
+				await cancelPromise;
+			}
+			else {
+				reader.releaseLock();
+			}
+			return { done: true, value };
+		},
+		[Symbol.asyncIterator]() {
+			return this;
+		}
+	};
+};
+
+ReadableStream.prototype[Symbol.asyncIterator] ??= ReadableStream.prototype.values;

app/Resources/Viewer/scripts/state.mjs

@@ -316,16 +316,16 @@ export default (function() {
 			return (channelObj && channelObj.name) || channel;
 		},
 		
-		getUserTag(user) {
-			const userObj = loadedFileMeta.users[user];
-			return (userObj && userObj.tag) || "????";
-		},
-		
 		getUserName(user) {
 			const userObj = loadedFileMeta.users[user];
 			return (userObj && userObj.name) || user;
 		},
 		
+		getUserDisplayName(user) {
+			const userObj = loadedFileMeta.users[user];
+			return (userObj && (userObj.displayName || userObj.name)) || user;
+		},
+		
 		selectChannel(channel) {
 			currentPage = 1;
 			selectedChannel = channel;

app/Resources/Viewer/styles/menu.css

@@ -1,15 +1,16 @@
 #menu {
-  width: 100%;
-  height: 48px;
   display: flex;
   flex-direction: row;
+  align-items: stretch;
+  gap: 8px;
+  padding: 8px;
   background-color: #17181c;
   border-bottom: 1px dotted #5d626b;
 }
 
 #menu .splitter {
-  width: 1px;
-  margin: 9px 4px;
+  flex: 0 0 1px;
+  margin: 9px 1px;
   background-color: #5d626b;
 }
 
@@ -23,7 +24,8 @@
 }
 
 #menu button, #menu select, #menu input[type="text"] {
-  margin: 8px;
+  height: 31px;
+  padding: 0 10px;
   background-color: #7289da;
   color: #fff;
   text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.75);
@@ -31,28 +33,25 @@
 
 #menu button {
   font-size: 17px;
-  padding: 0 12px;
   border: 0;
   cursor: pointer;
+  white-space: nowrap;
 }
 
 #menu select {
   font-size: 14px;
-  padding: 6px;
   border: 0;
   cursor: pointer;
 }
 
 #menu input[type="text"] {
   font-size: 14px;
-  padding: 7px 12px;
   border: 0;
 }
 
 #menu .nav {
   display: flex;
   flex-direction: row;
-  margin: 0 8px;
 }
 
 #menu .nav > button {
@@ -66,7 +65,7 @@
 }
 
 #menu .nav > button, #menu .nav > p {
-  margin: 8px 1px;
+  margin: 0 1px;
 }
 
 #opt-filter-list > select, #opt-filter-list > input {
@@ -76,3 +75,7 @@
 #opt-filter-list > .active {
   display: block;
 }
+
+#btn-about {
+  margin-left: auto;
+}

app/Server/Data/User.cs

@@ -3,6 +3,7 @@ namespace DHT.Server.Data;
 public readonly struct User {
 	public ulong Id { get; init; }
 	public string Name { get; init; }
+	public string? DisplayName { get; init; }
 	public string? AvatarUrl { get; init; }
 	public string? Discriminator { get; init; }
 }

app/Server/Database/Export/ViewerJson.cs

@@ -14,10 +14,10 @@ static class ViewerJson {
 		public required string Name { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-		public string? Avatar { get; init; }
+		public string? DisplayName { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-		public string? Tag { get; init; }
+		public string? Avatar { get; init; }
 	}
 
 	public sealed class JsonServer {

app/Server/Database/Export/ViewerJsonExport.cs

@@ -68,8 +68,8 @@ static class ViewerJsonExport {
 		await foreach (var user in db.Users.Get(cancellationToken)) {
 			users[user.Id] = new ViewerJson.JsonUser {
 				Name = user.Name,
+				DisplayName = user.DisplayName,
 				Avatar = user.AvatarUrl,
-				Tag = user.Discriminator
 			};
 		}
 

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -20,7 +20,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 
 	internal sealed class NewDownloadCollector : IAsyncDisposable {
 		private readonly SqliteDownloadRepository repository;
-		private bool hasAdded = false;
+		private bool hasChanged = false;
 
 		private readonly SqliteCommand metadataCmd;
 
@@ -31,7 +31,16 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 				"""
 				INSERT INTO download_metadata (normalized_url, download_url, status, type, size)
 				VALUES (:normalized_url, :download_url, :status, :type, :size)
-				ON CONFLICT DO NOTHING
+				ON CONFLICT (normalized_url)
+				DO UPDATE SET
+					download_url = excluded.download_url,
+					type = IFNULL(excluded.type, type),
+					size = IFNULL(excluded.size, size)
+				WHERE status != :success
+				  AND (download_url != excluded.download_url
+				    OR (excluded.type IS NOT NULL AND type IS NOT excluded.type)
+				    OR (excluded.size IS NOT NULL AND size IS NOT excluded.size)
+				  )
 				"""
 			);
 			metadataCmd.Add(":normalized_url", SqliteType.Text);
@@ -39,6 +48,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 			metadataCmd.Add(":status", SqliteType.Integer);
 			metadataCmd.Add(":type", SqliteType.Text);
 			metadataCmd.Add(":size", SqliteType.Integer);
+			metadataCmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 		}
 
 		public async Task Add(Data.Download download) {
@@ -47,11 +57,11 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 			metadataCmd.Set(":status", (int) download.Status);
 			metadataCmd.Set(":type", download.Type);
 			metadataCmd.Set(":size", download.Size);
-			hasAdded |= await metadataCmd.ExecuteNonQueryAsync() > 0;
+			hasChanged |= await metadataCmd.ExecuteNonQueryAsync() > 0;
 		}
 
 		public void OnCommitted() {
-			if (hasAdded) {
+			if (hasChanged) {
 				repository.UpdateTotalCount();
 			}
 		}
@@ -90,7 +100,8 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 					"""
 					INSERT INTO download_blobs (normalized_url, blob)
 					VALUES (:normalized_url, ZEROBLOB(:blob_length))
-					ON CONFLICT (normalized_url) DO UPDATE SET blob = excluded.blob
+					ON CONFLICT (normalized_url)
+					DO UPDATE SET blob = excluded.blob
 					RETURNING rowid
 					"""
 				);

app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs

@@ -29,6 +29,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 			await using var cmd = conn.Upsert("users", [
 				("id", SqliteType.Integer),
 				("name", SqliteType.Text),
+				("display_name", SqliteType.Text),
 				("avatar_url", SqliteType.Text),
 				("discriminator", SqliteType.Text)
 			]);
@@ -38,6 +39,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 			foreach (var user in users) {
 				cmd.Set(":id", user.Id);
 				cmd.Set(":name", user.Name);
+				cmd.Set(":display_name", user.DisplayName);
 				cmd.Set(":avatar_url", user.AvatarUrl);
 				cmd.Set(":discriminator", user.Discriminator);
 				await cmd.ExecuteNonQueryAsync();
@@ -62,15 +64,16 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 	public async IAsyncEnumerable<User> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
-		await using var cmd = conn.Command("SELECT id, name, avatar_url, discriminator FROM users");
+		await using var cmd = conn.Command("SELECT id, name, display_name, avatar_url, discriminator FROM users");
 		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
 		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new User {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),
-				AvatarUrl = reader.IsDBNull(2) ? null : reader.GetString(2),
-				Discriminator = reader.IsDBNull(3) ? null : reader.GetString(3),
+				DisplayName = reader.IsDBNull(2) ? null : reader.GetString(2),
+				AvatarUrl = reader.IsDBNull(3) ? null : reader.GetString(3),
+				Discriminator = reader.IsDBNull(4) ? null : reader.GetString(4),
 			};
 		}
 	}

app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo8.cs

@@ -0,0 +1,11 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo8 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+		await conn.ExecuteAsync("ALTER TABLE users ADD display_name TEXT");
+	}
+}

app/Server/Database/Sqlite/SqliteSchema.cs

@@ -8,7 +8,7 @@ using DHT.Utils.Logging;
 namespace DHT.Server.Database.Sqlite;
 
 sealed class SqliteSchema {
-	internal const int Version = 7;
+	internal const int Version = 8;
 
 	private static readonly Log Log = Log.ForType<SqliteSchema>();
 
@@ -48,6 +48,7 @@ sealed class SqliteSchema {
 		                        CREATE TABLE users (
 		                        	id            INTEGER PRIMARY KEY NOT NULL,
 		                        	name          TEXT NOT NULL,
+		                        	display_name  TEXT,
 		                        	avatar_url    TEXT,
 		                        	discriminator TEXT
 		                        )
@@ -171,6 +172,7 @@ sealed class SqliteSchema {
 			{ 4, new SqliteSchemaUpgradeTo5() },
 			{ 5, new SqliteSchemaUpgradeTo6() },
 			{ 6, new SqliteSchemaUpgradeTo7() },
+			{ 7, new SqliteSchemaUpgradeTo8() },
 		};
 
 		var perf = Log.Start("from version " + dbVersion);

app/Server/Endpoints/TrackUsersEndpoint.cs

@@ -30,6 +30,7 @@ sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
 	private static User ReadUser(JsonElement json, string path) => new () {
 		Id = json.RequireSnowflake("id", path),
 		Name = json.RequireString("name", path),
+		DisplayName = json.HasKey("displayName") ? json.RequireString("displayName", path) : null,
 		AvatarUrl = json.HasKey("avatar") ? json.RequireString("avatar", path) : null,
 		Discriminator = json.HasKey("discriminator") ? json.RequireString("discriminator", path) : null
 	};

app/Version.cs

@@ -8,5 +8,5 @@ using DHT.Utils;
 namespace DHT.Utils;
 
 static class Version {
-	public const string Tag = "42.0.0.0";
+	public const string Tag = "42.1.0.0";
 }