[WIP] Experimental Brotli compressing of raw message data

app/Resources/Tracker/scripts/state.js

@@ -304,6 +304,7 @@ const STATE = (function() {
 					});
 				}
 				
+				obj["raw"] = JSON.stringify(msg);
 				return obj;
 			}));
 			

app/Server/Data/Message.cs

@@ -12,5 +12,6 @@ namespace DHT.Server.Data {
 		public ImmutableArray<Attachment> Attachments { get; internal init; }
 		public ImmutableArray<Embed> Embeds { get; internal init; }
 		public ImmutableArray<Reaction> Reactions { get; internal init; }
+		public string? RawJson { get; internal init; }
 	}
 }

app/Server/Database/Sqlite/Schema.cs

@@ -5,7 +5,7 @@ using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite {
 	sealed class Schema {
-		internal const int Version = 2;
+		internal const int Version = 3;
 
 		private readonly SqliteConnection conn;
 
@@ -97,6 +97,8 @@ namespace DHT.Server.Database.Sqlite {
 					emoji_flags INTEGER NOT NULL,
 					count INTEGER NOT NULL)");
 
+			CreateMessagesRawTable();
+
 			Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
 			Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
 			Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
@@ -110,6 +112,16 @@ namespace DHT.Server.Database.Sqlite {
 			if (dbVersion <= 1) {
 				Execute("ALTER TABLE channels ADD parent_id INTEGER");
 			}
+
+			if (dbVersion <= 2) {
+				CreateMessagesRawTable();
+			}
+		}
+
+		private void CreateMessagesRawTable() {
+			Execute(@"CREATE TABLE messages_raw (
+			        message_id INTEGER PRIMARY KEY NOT NULL,
+			        json BLOB)");
 		}
 	}
 }

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
 using DHT.Utils.Collections;
+using DHT.Utils.Compression;
 using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite {
@@ -157,6 +158,10 @@ namespace DHT.Server.Database.Sqlite {
 				"message_id", "sender_id", "channel_id", "text", "timestamp", "edit_timestamp", "replied_to_id"
 			});
 
+			using var messageRawCmd = conn.Upsert("messages_raw", new[] {
+				"message_id", "json"
+			});
+
 			using var deleteAttachmentsCmd = conn.Command("DELETE FROM attachments WHERE message_id = :message_id");
 			using var attachmentCmd = conn.Insert("attachments", new[] {
 				"message_id", "attachment_id", "name", "type", "url", "size"
@@ -181,6 +186,10 @@ namespace DHT.Server.Database.Sqlite {
 			messageParams.Add(":edit_timestamp", SqliteType.Integer);
 			messageParams.Add(":replied_to_id", SqliteType.Integer);
 
+			var messageRawParams = messageRawCmd.Parameters;
+			messageRawParams.Add(":message_id", SqliteType.Integer);
+			messageRawParams.Add(":json", SqliteType.Blob);
+
 			var deleteAttachmentsParams = deleteAttachmentsCmd.Parameters;
 			deleteAttachmentsParams.Add(":message_id", SqliteType.Integer);
 
@@ -209,6 +218,8 @@ namespace DHT.Server.Database.Sqlite {
 			reactionParams.Add(":emoji_flags", SqliteType.Integer);
 			reactionParams.Add(":count", SqliteType.Integer);
 
+			var brotli = new Brotli(4096);
+
 			foreach (var message in messages) {
 				object messageId = message.Id;
 
@@ -221,6 +232,12 @@ namespace DHT.Server.Database.Sqlite {
 				messageParams.Set(":replied_to_id", message.RepliedToId);
 				messageCmd.ExecuteNonQuery();
 
+				if (message.RawJson is {} json) {
+					messageRawParams.Set(":message_id", messageId);
+					messageRawParams.Set(":json", brotli.Compress(Encoding.UTF8.GetBytes(json)));
+					messageRawCmd.ExecuteNonQuery();
+				}
+
 				deleteAttachmentsParams.Set(":message_id", messageId);
 				deleteAttachmentsCmd.ExecuteNonQuery();
 

app/Server/Endpoints/TrackMessagesEndpoint.cs

@@ -50,7 +50,8 @@ namespace DHT.Server.Endpoints {
 			RepliedToId = json.HasKey("repliedToId") ? json.RequireSnowflake("repliedToId", path) : null,
 			Attachments = json.HasKey("attachments") ? ReadAttachments(json.RequireArray("attachments", path + ".attachments"), path + ".attachments[]").ToImmutableArray() : ImmutableArray<Attachment>.Empty,
 			Embeds = json.HasKey("embeds") ? ReadEmbeds(json.RequireArray("embeds", path + ".embeds"), path + ".embeds[]").ToImmutableArray() : ImmutableArray<Embed>.Empty,
-			Reactions = json.HasKey("reactions") ? ReadReactions(json.RequireArray("reactions", path + ".reactions"), path + ".reactions[]").ToImmutableArray() : ImmutableArray<Reaction>.Empty
+			Reactions = json.HasKey("reactions") ? ReadReactions(json.RequireArray("reactions", path + ".reactions"), path + ".reactions[]").ToImmutableArray() : ImmutableArray<Reaction>.Empty,
+			RawJson = json.HasKey("raw") ? json.RequireString("raw", path) : null
 		};
 
 		private static IEnumerable<Attachment> ReadAttachments(JsonElement.ArrayEnumerator array, string path) => array.Select(ele => new Attachment {

app/Utils/Compression/Brotli.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Buffers;
+using System.IO;
+using System.IO.Compression;
+
+namespace DHT.Utils.Compression {
+	public sealed class Brotli {
+		private readonly byte[] tempBuffer;
+
+		public Brotli(int bufferSize) {
+			tempBuffer = new byte[bufferSize];
+		}
+
+		public byte[] Compress(byte[] input) {
+			var inputBuffer = new ReadOnlySpan<byte>(input);
+			var outputBuffer = new Span<byte>(tempBuffer);
+
+			using var outputStream = new MemoryStream();
+			using var brotliEncoder = new BrotliEncoder(10, 11);
+
+			var result = OperationStatus.DestinationTooSmall;
+
+			while (result == OperationStatus.DestinationTooSmall) {
+				result = brotliEncoder.Compress(inputBuffer, outputBuffer, out int bytesConsumed, out int bytesWritten, isFinalBlock: false);
+
+				if (result == OperationStatus.InvalidData) {
+					throw new InvalidDataException();
+				}
+
+				Write(bytesWritten, outputBuffer, outputStream);
+
+				if (bytesConsumed > 0) {
+					inputBuffer = inputBuffer[bytesConsumed..];
+				}
+			}
+
+			result = OperationStatus.DestinationTooSmall;
+
+			while (result == OperationStatus.DestinationTooSmall) {
+				result = brotliEncoder.Flush(outputBuffer, out var bytesWritten);
+
+				if (result == OperationStatus.InvalidData) {
+					throw new InvalidDataException();
+				}
+
+				Write(bytesWritten, outputBuffer, outputStream);
+			}
+
+			return outputStream.ToArray();
+		}
+
+		private static void Write(int bytes, Span<byte> buffer, MemoryStream outputStream) {
+			if (bytes > 0) {
+				outputStream.Write(buffer[..bytes]);
+			}
+		}
+	}
+}