Release v42.1

Hide user discriminators & fix places that still displayed usernames instead of display names
References #216
2025-09-13 15:32:09 +02:00 · 2024-05-09 00:08:15 +02:00 · 2024-05-09 00:04:31 +02:00 · 2024-05-08 23:39:13 +02:00 · 2024-05-08 21:47:16 +02:00 · 2024-05-08 21:08:24 +02:00
32 changed files with 332 additions and 213 deletions
--- a/app/Desktop/Main/Controls/MessageFilterPanelModel.cs
+++ b/app/Desktop/Main/Controls/MessageFilterPanelModel.cs
@@ -249,11 +249,8 @@ sealed partial class MessageFilterPanelModel : ObservableObject, IDisposable {
 			var checkBoxItems = new List<CheckBoxItem<ulong>>();

 			await foreach (var user in state.Db.Users.Get()) {
-				var name = user.Name;
-				var discriminator = user.Discriminator;
-
 				checkBoxItems.Add(new CheckBoxItem<ulong>(user.Id) {
-					Title = discriminator == null ? name : name + " #" + discriminator,
+					Title = user.DisplayName == null ? user.Name : $"{user.DisplayName} ({user.Name})",
 					IsChecked = IncludedUsers == null || IncludedUsers.Contains(user.Id)
 				});
 			}
--- a/app/Desktop/Main/Pages/DebugPageModel.cs
+++ b/app/Desktop/Main/Pages/DebugPageModel.cs
@@ -73,6 +73,7 @@ sealed class DebugPageModel {
 		var users = Enumerable.Range(0, userCount).Select(_ => new User {
 			Id = RandomId(rand),
 			Name = RandomName("u"),
+			DisplayName = RandomName("u"),
 			AvatarUrl = null,
 			Discriminator = rand.Next(0, 9999).ToString(),
 		}).ToArray();
--- a/app/Resources/Schemas/track-users.yml
+++ b/app/Resources/Schemas/track-users.yml
@@ -9,6 +9,8 @@ items:
          pattern: "^[0-9]+$"
        name:
          type: string
+        displayName:
+          type: string
        avatar:
          type: string
        discriminator:
--- a/app/Resources/Tracker/scripts/state.js
+++ b/app/Resources/Tracker/scripts/state.js
@@ -69,6 +69,7 @@ const STATE = (function() {
 	 * @property {String} id
 	 * @property {String} username
 	 * @property {String} discriminator
+	 * @property {String} [globalName]
 	 * @property {String} [avatar]
 	 * @property {Boolean} [bot]
 	 */
@@ -200,6 +201,10 @@ const STATE = (function() {
 						name: user.username
 					};
 					
+					if (user.globalName) {
+						obj.displayName = user.globalName;
+					}
+					
 					if (user.avatar) {
 						obj.avatar = user.avatar;
 					}
--- a/app/Resources/Viewer/index.html
+++ b/app/Resources/Viewer/index.html
@@ -2,6 +2,8 @@
 <html lang="en">
  <head>
    <meta charset="utf-8">
+    <meta name="referrer" content="no-referrer">
+    
    <title>Discord Offline History</title>
    
    <link rel="icon" href="favicon.ico">
@@ -23,6 +25,8 @@
    <div id="menu">
      <button id="btn-settings">Settings</button>
      
+      <div class="splitter"></div>
+      
      <div> <!-- needed to stop the select from messing up -->
        <select id="opt-messages-per-page">
          <option value="50">50 messages per page&nbsp;</option>
--- a/app/Resources/Viewer/scripts/bootstrap.mjs
+++ b/app/Resources/Viewer/scripts/bootstrap.mjs
@@ -1,6 +1,7 @@
 import discord from "./discord.mjs";
 import gui from "./gui.mjs";
 import state from "./state.mjs";
+import "./polyfills.mjs";

 window.DISCORD = discord;

@@ -36,18 +37,69 @@ document.addEventListener("DOMContentLoaded", () => {
 		gui.scrollMessagesToTop();
 	});
 	
-	async function loadData() {
-		try {
-			const response = await fetch("/get-viewer-data?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), {
+	async function fetchUrl(path, contentType) {
+		const response = await fetch("/" + path + "?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), {
 			method: "GET",
 			headers: {
-					"Content-Type": "application/json",
+				"Content-Type": contentType,
 			},
 			credentials: "omit",
 			redirect: "error",
 		});
 		
-			state.uploadFile(await response.json());
+		if (!response.ok) {
+			throw "Unexpected response status: " + response.statusText;
+		}
+		
+		return response;
+	}
+	
+	async function processLines(response, callback) {
+		let body = "";
+		
+		for await (const chunk of response.body.pipeThrough(new TextDecoderStream("utf-8"))) {
+			body += chunk;
+			
+			let startIndex = 0;
+			
+			while (true) {
+				const endIndex = body.indexOf("\n", startIndex);
+				if (endIndex === -1) {
+					break;
+				}
+				
+				callback(body.substring(startIndex, endIndex));
+				startIndex = endIndex + 1;
+			}
+			
+			body = body.substring(startIndex);
+		}
+		
+		if (body !== "") {
+			callback(body);
+		}
+	}
+	
+	async function loadData() {
+		try {
+			const metadataResponse = await fetchUrl("get-viewer-metadata", "application/json");
+			const metadataJson = await metadataResponse.json();
+			
+			const messagesResponse = await fetchUrl("get-viewer-messages", "application/x-ndjson");
+			const messages = {};
+			
+			await processLines(messagesResponse, line => {
+				const message = JSON.parse(line);
+				const channel = message.c;
+				
+				const channelMessages = messages[channel] || (messages[channel] = {});
+				channelMessages[message.id] = message;
+				
+				delete message.id;
+				delete message.c;
+			});
+			
+			state.uploadFile(metadataJson, messages);
 		} catch (e) {
 			console.error(e);
 			alert("Could not load data, see console for details.");
--- a/app/Resources/Viewer/scripts/discord.mjs
+++ b/app/Resources/Viewer/scripts/discord.mjs
@@ -80,7 +80,7 @@ export default (function() {
 		processed = processed
 			.replace(regex.formatUrl, "<a href='$1' target='_blank' rel='noreferrer'>$1</a>")
 			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + state.getChannelName(match) + "</span>")
-			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (state.getUserTag(match) || "????") + "'>@" + state.getUserName(match) + "</span>")
+			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='" + state.getUserName(match) + "'>@" + state.getUserDisplayName(match) + "</span>")
 			.replace(regex.customEmojiStatic, (full, m1, m2) => getEmoji(m1, m2, "webp"))
 			.replace(regex.customEmojiAnimated, (full, m1, m2) => getEmoji(m1, m2, animatedEmojiExtension));
 		
@@ -129,7 +129,7 @@ export default (function() {
 			templateMessageNoAvatar = new template([
 				"<div>",
 				"<div class='reply-message'>{reply}</div>",
-				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
+				"<h2><strong class='username' title='{user.name}'>{user.displayName}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
 				"<div class='message'>{contents}{embeds}{attachments}</div>",
 				"{reactions}",
 				"</div>"
@@ -141,7 +141,7 @@ export default (function() {
 				"<div class='avatar-wrapper'>",
 				"<div class='avatar'>{avatar}</div>",
 				"<div>",
-				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
+				"<h2><strong class='username' title='{user.name}'>{user.displayName}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>",
 				"<div class='message'>{contents}{embeds}{attachments}</div>",
 				"{reactions}",
 				"</div>",
@@ -227,8 +227,8 @@ export default (function() {
 				if (property === "avatar") {
 					return value ? templateUserAvatar.apply(getAvatarUrlObject(value)) : "";
 				}
-				else if (property === "user.tag") {
-					return value ? value : "????";
+				else if (property === "user.displayName") {
+					return value ? value : message.user.name;
 				}
 				else if (property === "timestamp") {
 					return dom.getHumanReadableTime(value);
@@ -246,10 +246,10 @@ export default (function() {
 							return templateEmbedUnsupported.apply(embed);
 						}
 						else if ("image" in embed && embed.image.url) {
-							return getImageEmbed(fileUrlProcessor(embed.url), embed.image);
+							return getImageEmbed(embed.url, embed.image);
 						}
 						else if ("thumbnail" in embed && embed.thumbnail.url) {
-							return getImageEmbed(fileUrlProcessor(embed.url), embed.thumbnail);
+							return getImageEmbed(embed.url, embed.thumbnail);
 						}
 						else if ("title" in embed && "description" in embed) {
 							return templateEmbedRich.apply(embed);
@@ -292,7 +292,7 @@ export default (function() {
 						return value === null ? "<span class='reply-contents reply-missing'>(replies to an unknown message)</span>" : "";
 					}
 					
-					const user = "<span class='reply-username' title='#" + (value.user.tag ? value.user.tag : "????") + "'>" + value.user.name + "</span>";
+					const user = "<span class='reply-username' title='" + value.user.name + "'>" + (value.user.displayName ?? value.user.name) + "</span>";
 					const avatar = settings.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(getAvatarUrlObject(value.avatar)) + "</span>" : "";
 					const contents = value.contents ? "<span class='reply-contents'>" + processMessageContents(value.contents) + "</span>" : "";
 					
--- a/app/Resources/Viewer/scripts/gui.mjs
+++ b/app/Resources/Viewer/scripts/gui.mjs
@@ -243,10 +243,11 @@ export default (function() {
 			
 			const options = [];
 			
-			for (const key of Object.keys(users)) {
+			for (const id of Object.keys(users)) {
+				const user = users[id];
 				const option = document.createElement("option");
-				option.value = key;
-				option.text = users[key].name;
+				option.value = id;
+				option.text = user.displayName ? `${user.displayName} (${user.name})` : user.name;
 				options.push(option);
 			}
 			
--- a/app/Resources/Viewer/scripts/polyfills.mjs
+++ b/app/Resources/Viewer/scripts/polyfills.mjs
@@ -0,0 +1,35 @@
+// https://gist.github.com/MattiasBuelens/496fc1d37adb50a733edd43853f2f60e/088f061ab79b296f29225467ae9ba86ff990195d
+
+ReadableStream.prototype.values ??= function({ preventCancel = false } = {}) {
+	const reader = this.getReader();
+	return {
+		async next() {
+			try {
+				const result = await reader.read();
+				if (result.done) {
+					reader.releaseLock();
+				}
+				return result;
+			} catch (e) {
+				reader.releaseLock();
+				throw e;
+			}
+		},
+		async return(value) {
+			if (!preventCancel) {
+				const cancelPromise = reader.cancel(value);
+				reader.releaseLock();
+				await cancelPromise;
+			}
+			else {
+				reader.releaseLock();
+			}
+			return { done: true, value };
+		},
+		[Symbol.asyncIterator]() {
+			return this;
+		}
+	};
+};
+
+ReadableStream.prototype[Symbol.asyncIterator] ??= ReadableStream.prototype.values;
--- a/app/Resources/Viewer/scripts/processor.mjs
+++ b/app/Resources/Viewer/scripts/processor.mjs
@@ -5,7 +5,7 @@ import discord from "./discord.mjs";
 // ------------------------

 const filter = {
-	byUser: ((userindex) => message => message.u === userindex),
+	byUser: ((user) => message => message.u === user),
 	byTime: ((timeStart, timeEnd) => message => message.t >= timeStart && message.t <= timeEnd),
 	byContents: ((substr) => message => ("m" in message ? message.m : "").indexOf(substr) !== -1),
 	byRegex: ((regex) => message => regex.test("m" in message ? message.m : "")),
--- a/app/Resources/Viewer/scripts/state.mjs
+++ b/app/Resources/Viewer/scripts/state.mjs
@@ -6,8 +6,7 @@ export default (function() {
 	/**
 	 * @type {{}}
 	 * @property {{}} users
-	 * @property {String[]} userindex
-	 * @property {{}[]} servers
+	 * @property {{}} servers
 	 * @property {{}} channels
 	 */
 	let loadedFileMeta;
@@ -20,20 +19,16 @@ export default (function() {
 	let currentPage;
 	let messagesPerPage;
 	
-	const getUser = function(index) {
-		return loadedFileMeta.users[loadedFileMeta.userindex[index]] || { "name": "&lt;unknown&gt;" };
-	};
-	
-	const getUserId = function(index) {
-		return loadedFileMeta.userindex[index];
+	const getUser = function(id) {
+		return loadedFileMeta.users[id] || { "name": "&lt;unknown&gt;" };
 	};
 	
 	const getUserList = function() {
 		return loadedFileMeta ? loadedFileMeta.users : [];
 	};
 	
-	const getServer = function(index) {
-		return loadedFileMeta.servers[index] || { "name": "&lt;unknown&gt;", "type": "unknown" };
+	const getServer = function(id) {
+		return loadedFileMeta.servers[id] || { "name": "&lt;unknown&gt;", "type": "unknown" };
 	};
 	
 	const generateChannelHierarchy = function() {
@@ -207,7 +202,7 @@ export default (function() {
 			 */
 			const message = messages[key];
 			const user = getUser(message.u);
-			const avatar = user.avatar ? { id: getUserId(message.u), path: user.avatar } : null;
+			const avatar = user.avatar ? { id: message.u, path: user.avatar } : null;
 			
 			const obj = {
 				user,
@@ -235,7 +230,7 @@ export default (function() {
 			if ("r" in message) {
 				const replyMessage = getMessageById(message.r);
 				const replyUser = replyMessage ? getUser(replyMessage.u) : null;
-				const replyAvatar = replyUser && replyUser.avatar ? { id: getUserId(replyMessage.u), path: replyUser.avatar } : null;
+				const replyAvatar = replyUser && replyUser.avatar ? { id: replyMessage.u, path: replyUser.avatar } : null;
 				
 				obj["reply"] = replyMessage ? {
 					"id": message.r,
@@ -293,20 +288,17 @@ export default (function() {
 			eventOnUsersRefreshed = callback;
 		},
 		
-		/**
-		 * @param {{ meta, data }} file
-		 */
-		uploadFile(file) {
+		uploadFile(meta, data) {
 			if (loadedFileMeta != null) {
 				throw "A file is already loaded!";
 			}
 			
-			if (!file || typeof file.meta !== "object" || typeof file.data !== "object") {
+			if (typeof meta !== "object" || typeof data !== "object") {
 				throw "Invalid file format!";
 			}
 			
-			loadedFileMeta = file.meta;
-			loadedFileData = file.data;
+			loadedFileMeta = meta;
+			loadedFileData = data;
 			loadedMessages = null;
 			
 			selectedChannel = null;
@@ -324,16 +316,16 @@ export default (function() {
 			return (channelObj && channelObj.name) || channel;
 		},
 		
-		getUserTag(user) {
-			const userObj = loadedFileMeta.users[user];
-			return (userObj && userObj.tag) || "????";
-		},
-		
 		getUserName(user) {
 			const userObj = loadedFileMeta.users[user];
 			return (userObj && userObj.name) || user;
 		},
 		
+		getUserDisplayName(user) {
+			const userObj = loadedFileMeta.users[user];
+			return (userObj && (userObj.displayName || userObj.name)) || user;
+		},
+		
 		selectChannel(channel) {
 			currentPage = 1;
 			selectedChannel = channel;
@@ -419,7 +411,7 @@ export default (function() {
 		setActiveFilter(filter) {
 			switch (filter ? filter.type : "") {
 				case "user":
-					filterFunction = processor.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value));
+					filterFunction = processor.FILTER.byUser(filter.value);
 					break;
 				
 				case "contents":
--- a/app/Resources/Viewer/styles/menu.css
+++ b/app/Resources/Viewer/styles/menu.css
@@ -1,15 +1,16 @@
 #menu {
-  width: 100%;
-  height: 48px;
  display: flex;
  flex-direction: row;
+  align-items: stretch;
+  gap: 8px;
+  padding: 8px;
  background-color: #17181c;
  border-bottom: 1px dotted #5d626b;
 }

 #menu .splitter {
-  width: 1px;
-  margin: 9px 4px;
+  flex: 0 0 1px;
+  margin: 9px 1px;
  background-color: #5d626b;
 }

@@ -23,7 +24,8 @@
 }

 #menu button, #menu select, #menu input[type="text"] {
-  margin: 8px;
+  height: 31px;
+  padding: 0 10px;
  background-color: #7289da;
  color: #fff;
  text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.75);
@@ -31,28 +33,25 @@

 #menu button {
  font-size: 17px;
-  padding: 0 12px;
  border: 0;
  cursor: pointer;
+  white-space: nowrap;
 }

 #menu select {
  font-size: 14px;
-  padding: 6px;
  border: 0;
  cursor: pointer;
 }

 #menu input[type="text"] {
  font-size: 14px;
-  padding: 7px 12px;
  border: 0;
 }

 #menu .nav {
  display: flex;
  flex-direction: row;
-  margin: 0 8px;
 }

 #menu .nav > button {
@@ -66,7 +65,7 @@
 }

 #menu .nav > button, #menu .nav > p {
-  margin: 8px 1px;
+  margin: 0 1px;
 }

 #opt-filter-list > select, #opt-filter-list > input {
@@ -76,3 +75,7 @@
 #opt-filter-list > .active {
  display: block;
 }
+
+#btn-about {
+  margin-left: auto;
+}
--- a/app/Server/Data/User.cs
+++ b/app/Server/Data/User.cs
@@ -3,6 +3,7 @@ namespace DHT.Server.Data;
 public readonly struct User {
 	public ulong Id { get; init; }
 	public string Name { get; init; }
+	public string? DisplayName { get; init; }
 	public string? AvatarUrl { get; init; }
 	public string? Discriminator { get; init; }
 }
--- a/app/Server/Database/Export/Snowflake.cs
+++ b/app/Server/Database/Export/Snowflake.cs
@@ -1,3 +1,5 @@
 namespace DHT.Server.Database.Export;

-readonly record struct Snowflake(ulong Id);
+readonly record struct Snowflake(ulong Id) {
+	public static implicit operator Snowflake(ulong id) => new (id);
+}
--- a/app/Server/Database/Export/ViewerJson.cs
+++ b/app/Server/Database/Export/ViewerJson.cs
@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;

 namespace DHT.Server.Database.Export;

-sealed class ViewerJson {
-	public required JsonMeta Meta { get; init; }
-	public required Dictionary<Snowflake, Dictionary<Snowflake, JsonMessage>> Data { get; init; }
-
+static class ViewerJson {
 	public sealed class JsonMeta {
 		public required Dictionary<Snowflake, JsonUser> Users { get; init; }
-		public required List<Snowflake> Userindex { get; init; }
-		public required List<JsonServer> Servers { get; init; }
+		public required Dictionary<Snowflake, JsonServer> Servers { get; init; }
 		public required Dictionary<Snowflake, JsonChannel> Channels { get; init; }
 	}
 	
@@ -18,10 +14,10 @@ sealed class ViewerJson {
 		public required string Name { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-		public string? Avatar { get; init; }
+		public string? DisplayName { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-		public string? Tag { get; init; }
+		public string? Avatar { get; init; }
 	}

 	public sealed class JsonServer {
@@ -30,7 +26,7 @@ sealed class ViewerJson {
 	}

 	public sealed class JsonChannel {
-		public required int Server { get; init; }
+		public required Snowflake Server { get; init; }
 		public required string Name { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
@@ -47,7 +43,9 @@ sealed class ViewerJson {
 	}

 	public sealed class JsonMessage {
-		public required int U { get; init; }
+		public required Snowflake Id { get; init; }
+		public required Snowflake C { get; init; }
+		public required Snowflake U { get; init; }
 		public required long T { get; init; }

 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
--- a/app/Server/Database/Export/ViewerJsonExport.cs
+++ b/app/Server/Database/Export/ViewerJsonExport.cs
@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
@@ -14,106 +15,90 @@ namespace DHT.Server.Database.Export;
 static class ViewerJsonExport {
 	private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));

-	public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
+	public static async Task GetMetadata(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
 		var perf = Log.Start();

-		var includedUserIds = new HashSet<ulong>();
-		var includedChannelIds = new HashSet<ulong>();
+		var includedChannels = new List<Channel>();
 		var includedServerIds = new HashSet<ulong>();

-		var includedMessages = await db.Messages.Get(filter, cancellationToken).ToListAsync(cancellationToken);
-		var includedChannels = new List<Channel>();
-
-		foreach (var message in includedMessages) {
-			includedUserIds.Add(message.Sender);
-			includedChannelIds.Add(message.Channel);
-		}
+		var channelIdFilter = filter?.ChannelIds;

 		await foreach (var channel in db.Channels.Get(cancellationToken)) {
-			if (includedChannelIds.Contains(channel.Id)) {
+			if (channelIdFilter == null || channelIdFilter.Contains(channel.Id)) {
 				includedChannels.Add(channel);
 				includedServerIds.Add(channel.Server);
 			}
 		}

-		var (users, userIndex, userIndices) = await GenerateUserList(db, includedUserIds);
-		var (servers, serverIndices) = await GenerateServerList(db, includedServerIds);
-		var channels = GenerateChannelList(includedChannels, serverIndices);
+		var users = await GenerateUserList(db, cancellationToken);
+		var servers = await GenerateServerList(db, includedServerIds, cancellationToken);
+		var channels = GenerateChannelList(includedChannels);
+
+		var meta = new ViewerJson.JsonMeta {
+			Users = users,
+			Servers = servers,
+			Channels = channels
+		};

 		perf.Step("Collect database data");

-		var value = new ViewerJson {
-			Meta = new ViewerJson.JsonMeta {
-				Users = users,
-				Userindex = userIndex,
-				Servers = servers,
-				Channels = channels
-			},
-			Data = GenerateMessageList(includedMessages, userIndices)
-		};
-
-		perf.Step("Generate value object");
-
-		await JsonSerializer.SerializeAsync(stream, value, ViewerJsonContext.Default.ViewerJson, cancellationToken);
+		await JsonSerializer.SerializeAsync(stream, meta, ViewerJsonMetadataContext.Default.JsonMeta, cancellationToken);

 		perf.Step("Serialize to JSON");
 		perf.End();
 	}

-	private static async Task<(Dictionary<Snowflake, ViewerJson.JsonUser> Users, List<Snowflake> UserIndex, Dictionary<ulong, int> UserIndices)> GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds) {
-		var users = new Dictionary<Snowflake, ViewerJson.JsonUser>();
-		var userIndex = new List<Snowflake>();
-		var userIndices = new Dictionary<ulong, int>();
+	public static async Task GetMessages(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
+		var perf = Log.Start();

-		await foreach (var user in db.Users.Get()) {
-			var id = user.Id;
-			if (!userIds.Contains(id)) {
-				continue;
+		ReadOnlyMemory<byte> newLine = "\n"u8.ToArray();
+		
+		await foreach(var message in GenerateMessageList(db, filter, cancellationToken)) {
+			await JsonSerializer.SerializeAsync(stream, message, ViewerJsonMessageContext.Default.JsonMessage, cancellationToken);
+			await stream.WriteAsync(newLine, cancellationToken);
 		}

-			var idSnowflake = new Snowflake(id);
-			userIndices[id] = users.Count;
-			userIndex.Add(idSnowflake);
+		perf.Step("Generate and serialize messages to JSON");
+		perf.End();
+	}

-			users[idSnowflake] = new ViewerJson.JsonUser {
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonUser>> GenerateUserList(IDatabaseFile db, CancellationToken cancellationToken) {
+		var users = new Dictionary<Snowflake, ViewerJson.JsonUser>();
+
+		await foreach (var user in db.Users.Get(cancellationToken)) {
+			users[user.Id] = new ViewerJson.JsonUser {
 				Name = user.Name,
+				DisplayName = user.DisplayName,
 				Avatar = user.AvatarUrl,
-				Tag = user.Discriminator
 			};
 		}

-		return (users, userIndex, userIndices);
+		return users;
 	}

-	private static async Task<(List<ViewerJson.JsonServer> Servers, Dictionary<ulong, int> ServerIndices)> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds) {
-		var servers = new List<ViewerJson.JsonServer>();
-		var serverIndices = new Dictionary<ulong, int>();
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonServer>> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds, CancellationToken cancellationToken) {
+		var servers = new Dictionary<Snowflake, ViewerJson.JsonServer>();

-		await foreach (var server in db.Servers.Get()) {
-			var id = server.Id;
-			if (!serverIds.Contains(id)) {
+		await foreach (var server in db.Servers.Get(cancellationToken)) {
+			if (!serverIds.Contains(server.Id)) {
 				continue;
 			}

-			serverIndices[id] = servers.Count;
-			
-			servers.Add(new ViewerJson.JsonServer {
+			servers[server.Id] = new ViewerJson.JsonServer {
 				Name = server.Name,
 				Type = ServerTypes.ToJsonViewerString(server.Type)
-			});
+			};
 		}

-		return (servers, serverIndices);
+		return servers;
 	}

-	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels, Dictionary<ulong, int> serverIndices) {
+	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels) {
 		var channels = new Dictionary<Snowflake, ViewerJson.JsonChannel>();

 		foreach (var channel in includedChannels) {
-			var channelIdSnowflake = new Snowflake(channel.Id);
-			
-			channels[channelIdSnowflake] = new ViewerJson.JsonChannel {
-				Server = serverIndices[channel.Server],
+			channels[channel.Id] = new ViewerJson.JsonChannel {
+				Server = channel.Server,
 				Name = channel.Name,
 				Parent = channel.ParentId?.ToString(),
 				Position = channel.Position,
@@ -125,18 +110,12 @@ static class ViewerJsonExport {
 		return channels;
 	}

-	private static Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>> GenerateMessageList(List<Message> includedMessages, Dictionary<ulong, int> userIndices) {
-		var data = new Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>>();
-
-		foreach (var grouping in includedMessages.GroupBy(static message => message.Channel)) {
-			var channelIdSnowflake = new Snowflake(grouping.Key);
-			var channelData = new Dictionary<Snowflake, ViewerJson.JsonMessage>();
-
-			foreach (var message in grouping) {
-				var messageIdSnowflake = new Snowflake(message.Id);
-				
-				channelData[messageIdSnowflake] = new ViewerJson.JsonMessage {
-					U = userIndices[message.Sender],
+	private static async IAsyncEnumerable<ViewerJson.JsonMessage> GenerateMessageList(IDatabaseFile db, MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
+		await foreach (var message in db.Messages.Get(filter, cancellationToken)) {
+			yield return new ViewerJson.JsonMessage {
+				Id = message.Id,
+				C = message.Channel,
+				U = message.Sender,
 				T = message.Timestamp,
 				M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
 				Te = message.EditTimestamp,
@@ -166,10 +145,5 @@ static class ViewerJsonExport {
 				}).ToArray()
 			};
 		}
-
-			data[channelIdSnowflake] = channelData;
-		}
-
-		return data;
 	}
 }
--- a/app/Server/Database/Export/ViewerJsonMessageContext.cs
+++ b/app/Server/Database/Export/ViewerJsonMessageContext.cs
@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace DHT.Server.Database.Export;
+
+[JsonSourceGenerationOptions(
+	Converters = [typeof(SnowflakeJsonSerializer)],
+	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
+	GenerationMode = JsonSourceGenerationMode.Default
+)]
+[JsonSerializable(typeof(ViewerJson.JsonMessage))]
+sealed partial class ViewerJsonMessageContext : JsonSerializerContext;
--- a/app/Server/Database/Export/ViewerJsonMetadataContext.cs
+++ b/app/Server/Database/Export/ViewerJsonMetadataContext.cs
@@ -7,5 +7,5 @@ namespace DHT.Server.Database.Export;
 	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
 	GenerationMode = JsonSourceGenerationMode.Default
 )]
-[JsonSerializable(typeof(ViewerJson))]
-sealed partial class ViewerJsonContext : JsonSerializerContext;
+[JsonSerializable(typeof(ViewerJson.JsonMeta))]
+sealed partial class ViewerJsonMetadataContext : JsonSerializerContext;
--- a/app/Server/Database/Repositories/IServerRepository.cs
+++ b/app/Server/Database/Repositories/IServerRepository.cs
@@ -14,7 +14,7 @@ public interface IServerRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Data.Server> Get();
+	IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken = default);

 	internal sealed class Dummy : IServerRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -27,7 +27,7 @@ public interface IServerRepository {
 			return Task.FromResult(0L);
 		}

-		public IAsyncEnumerable<Data.Server> Get() {
+		public IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Data.Server>();
 		}
 	}
--- a/app/Server/Database/Repositories/IUserRepository.cs
+++ b/app/Server/Database/Repositories/IUserRepository.cs
@@ -15,7 +15,7 @@ public interface IUserRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<User> Get();
+	IAsyncEnumerable<User> Get(CancellationToken cancellationToken = default);

 	internal sealed class Dummy : IUserRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -28,7 +28,7 @@ public interface IUserRepository {
 			return Task.FromResult(0L);
 		}

-		public IAsyncEnumerable<User> Get() {
+		public IAsyncEnumerable<User> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<User>();
 		}
 	}
--- a/app/Server/Database/Sqlite/Repositories/SqliteServerRepository.cs
+++ b/app/Server/Database/Sqlite/Repositories/SqliteServerRepository.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -46,13 +47,13 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM servers", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}

-	public async IAsyncEnumerable<Data.Server> Get() {
+	public async IAsyncEnumerable<Data.Server> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();

 		await using var cmd = conn.Command("SELECT id, name, type FROM servers");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);

-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new Data.Server {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),
--- a/app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs
+++ b/app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -28,6 +29,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 			await using var cmd = conn.Upsert("users", [
 				("id", SqliteType.Integer),
 				("name", SqliteType.Text),
+				("display_name", SqliteType.Text),
 				("avatar_url", SqliteType.Text),
 				("discriminator", SqliteType.Text)
 			]);
@@ -37,6 +39,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 			foreach (var user in users) {
 				cmd.Set(":id", user.Id);
 				cmd.Set(":name", user.Name);
+				cmd.Set(":display_name", user.DisplayName);
 				cmd.Set(":avatar_url", user.AvatarUrl);
 				cmd.Set(":discriminator", user.Discriminator);
 				await cmd.ExecuteNonQueryAsync();
@@ -58,18 +61,19 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM users", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}

-	public async IAsyncEnumerable<User> Get() {
+	public async IAsyncEnumerable<User> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();

-		await using var cmd = conn.Command("SELECT id, name, avatar_url, discriminator FROM users");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var cmd = conn.Command("SELECT id, name, display_name, avatar_url, discriminator FROM users");
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);

-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new User {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),
-				AvatarUrl = reader.IsDBNull(2) ? null : reader.GetString(2),
-				Discriminator = reader.IsDBNull(3) ? null : reader.GetString(3),
+				DisplayName = reader.IsDBNull(2) ? null : reader.GetString(2),
+				AvatarUrl = reader.IsDBNull(3) ? null : reader.GetString(3),
+				Discriminator = reader.IsDBNull(4) ? null : reader.GetString(4),
 			};
 		}
 	}
--- a/app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo8.cs
+++ b/app/Server/Database/Sqlite/Schema/SqliteSchemaUpgradeTo8.cs
@@ -0,0 +1,11 @@
+using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
+
+namespace DHT.Server.Database.Sqlite.Schema;
+
+sealed class SqliteSchemaUpgradeTo8 : ISchemaUpgrade {
+	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
+		await reporter.MainWork("Applying schema changes...", 0, 1);
+		await conn.ExecuteAsync("ALTER TABLE users ADD display_name TEXT");
+	}
+}
--- a/app/Server/Database/Sqlite/SqliteSchema.cs
+++ b/app/Server/Database/Sqlite/SqliteSchema.cs
@@ -8,7 +8,7 @@ using DHT.Utils.Logging;
 namespace DHT.Server.Database.Sqlite;

 sealed class SqliteSchema {
-	internal const int Version = 7;
+	internal const int Version = 8;

 	private static readonly Log Log = Log.ForType<SqliteSchema>();

@@ -48,6 +48,7 @@ sealed class SqliteSchema {
 		                        CREATE TABLE users (
 		                        	id            INTEGER PRIMARY KEY NOT NULL,
 		                        	name          TEXT NOT NULL,
+		                        	display_name  TEXT,
 		                        	avatar_url    TEXT,
 		                        	discriminator TEXT
 		                        )
@@ -171,6 +172,7 @@ sealed class SqliteSchema {
 			{ 4, new SqliteSchemaUpgradeTo5() },
 			{ 5, new SqliteSchemaUpgradeTo6() },
 			{ 6, new SqliteSchemaUpgradeTo7() },
+			{ 7, new SqliteSchemaUpgradeTo8() },
 		};

 		var perf = Log.Start("from version " + dbVersion);
--- a/app/Server/Endpoints/BaseEndpoint.cs
+++ b/app/Server/Endpoints/BaseEndpoint.cs
@@ -47,4 +47,13 @@ abstract class BaseEndpoint(IDatabaseFile db) {
 			throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON.");
 		}
 	}
+	
+	protected static Guid GetSessionId(HttpRequest request) {
+		if (request.Query.TryGetValue("session", out var sessionIdValue) && sessionIdValue.Count == 1 && Guid.TryParse(sessionIdValue[0], out Guid sessionId)) {
+			return sessionId;
+		}
+		else {
+			throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID.");
+		}
+	}
 }
--- a/app/Server/Endpoints/GetViewerDataEndpoint.cs
+++ b/app/Server/Endpoints/GetViewerDataEndpoint.cs
@@ -1,25 +0,0 @@
-using System;
-using System.Net;
-using System.Net.Mime;
-using System.Threading;
-using System.Threading.Tasks;
-using DHT.Server.Database;
-using DHT.Server.Database.Export;
-using DHT.Server.Service.Viewer;
-using DHT.Utils.Http;
-using Microsoft.AspNetCore.Http;
-
-namespace DHT.Server.Endpoints; 
-
-sealed class GetViewerDataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
-	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
-		if (!request.Query.TryGetValue("session", out var sessionIdValue) || sessionIdValue.Count != 1 || !Guid.TryParse(sessionIdValue[0], out Guid sessionId)) {
-			throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID.");
-		}
-		
-		response.ContentType = MediaTypeNames.Application.Json;
-		
-		var session = viewerSessions.Get(sessionId);
-		return ViewerJsonExport.Generate(response.Body, Db, session.MessageFilter, cancellationToken);
-	}
-}
--- a/app/Server/Endpoints/GetViewerMessagesEndpoint.cs
+++ b/app/Server/Endpoints/GetViewerMessagesEndpoint.cs
@@ -0,0 +1,18 @@
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMessagesEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = "application/x-ndjson";
+		return ViewerJsonExport.GetMessages(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}
--- a/app/Server/Endpoints/GetViewerMetadataEndpoint.cs
+++ b/app/Server/Endpoints/GetViewerMetadataEndpoint.cs
@@ -0,0 +1,19 @@
+using System.Net.Mime;
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMetadataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = MediaTypeNames.Application.Json;
+		return ViewerJsonExport.GetMetadata(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}
--- a/app/Server/Endpoints/TrackUsersEndpoint.cs
+++ b/app/Server/Endpoints/TrackUsersEndpoint.cs
@@ -30,6 +30,7 @@ sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
 	private static User ReadUser(JsonElement json, string path) => new () {
 		Id = json.RequireSnowflake("id", path),
 		Name = json.RequireString("name", path),
+		DisplayName = json.HasKey("displayName") ? json.RequireString("displayName", path) : null,
 		AvatarUrl = json.HasKey("avatar") ? json.RequireString("avatar", path) : null,
 		Discriminator = json.HasKey("discriminator") ? json.RequireString("discriminator", path) : null
 	};
--- a/app/Server/Service/ServerStartup.cs
+++ b/app/Server/Service/ServerStartup.cs
@@ -51,7 +51,8 @@ sealed class Startup {
 		app.UseRouting();
 		app.UseEndpoints(endpoints => {
 			endpoints.MapGet("/get-tracking-script", new GetTrackingScriptEndpoint(db, parameters, resources).Handle);
-			endpoints.MapGet("/get-viewer-data", new GetViewerDataEndpoint(db, viewerSessions).Handle);
+			endpoints.MapGet("/get-viewer-metadata", new GetViewerMetadataEndpoint(db, viewerSessions).Handle);
+			endpoints.MapGet("/get-viewer-messages", new GetViewerMessagesEndpoint(db, viewerSessions).Handle);
 			endpoints.MapGet("/get-downloaded-file/{url}", new GetDownloadedFileEndpoint(db).Handle);
 			endpoints.MapPost("/track-channel", new TrackChannelEndpoint(db).Handle);
 			endpoints.MapPost("/track-users", new TrackUsersEndpoint(db).Handle);
--- a/app/Version.cs
+++ b/app/Version.cs
@@ -8,5 +8,5 @@ using DHT.Utils;
 namespace DHT.Utils;

 static class Version {
-	public const string Tag = "41.2.0.0";
+	public const string Tag = "42.1.0.0";
 }
--- a/app/empty.dht
+++ b/app/empty.dht
Author	SHA1	Message	Date
chylex	c3d4fa5532	Release v42.1	2024-05-09 00:08:15 +02:00
chylex	a6225b9721	Hide user discriminators & fix places that still displayed usernames instead of display names References #216	2024-05-09 00:04:31 +02:00
chylex	943163473a	Add support for global display names of users Closes #216	2024-05-08 23:39:13 +02:00
chylex	fa00df10d8	Fix viewer menu styling	2024-05-08 21:47:16 +02:00
chylex	f54465e5fe	Fix viewer compatibility with older browsers and (hopefully) Safari Closes #260	2024-05-08 21:08:24 +02:00
chylex	2d55ca4013	Release v42.0	2024-04-23 03:56:27 +02:00
chylex	0072e025f4	Fix broken links to image embeds in viewer	2024-04-21 20:55:06 +02:00
chylex	383ee5c90e	Stream messages to viewer using JSON lines	2024-04-21 20:55:05 +02:00
chylex	2b1c178f49	Handle request cancellation in the integrated HTTP server	2024-04-21 18:48:19 +02:00
chylex	16d7e5a6f8	Serve viewer page and JSON data from the integrated HTTP server	2024-04-21 18:48:19 +02:00