1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-10-17 09:42:44 +02:00

Compare commits

..

2 Commits

5 changed files with 112 additions and 72 deletions

View File

@ -14,17 +14,9 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories; namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository { sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IMessageRepository {
private static readonly Log Log = Log.ForType<SqliteMessageRepository>(); private static readonly Log Log = Log.ForType<SqliteMessageRepository>();
private readonly SqliteConnectionPool pool;
private readonly SqliteDownloadRepository downloads;
public SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : base(Log) {
this.pool = pool;
this.downloads = downloads;
}
public async Task Add(IReadOnlyList<Message> messages) { public async Task Add(IReadOnlyList<Message> messages) {
if (messages.Count == 0) { if (messages.Count == 0) {
return; return;
@ -50,25 +42,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
("timestamp", SqliteType.Integer) ("timestamp", SqliteType.Integer)
]); ]);
await using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps"); await using var attachmentCmd = conn.Upsert("attachments", [
await using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
await using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
await using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
await using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
await using var editTimestampCmd = conn.Insert("edit_timestamps", [
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
]);
await using var repliedToCmd = conn.Insert("replied_to", [
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
]);
await using var attachmentCmd = conn.Insert("attachments", [
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer), ("attachment_id", SqliteType.Integer),
("name", SqliteType.Text), ("name", SqliteType.Text),
("type", SqliteType.Text), ("type", SqliteType.Text),
@ -79,12 +53,34 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
("height", SqliteType.Integer) ("height", SqliteType.Integer)
]); ]);
await using var embedCmd = conn.Insert("embeds", [ await using var deleteMessageEditTimestampCmd = DeleteByMessageId(conn, "message_edit_timestamps");
await using var deleteMessageRepliedToCmd = DeleteByMessageId(conn, "message_replied_to");
await using var deleteMessageAttachmentsCmd = DeleteByMessageId(conn, "message_attachments");
await using var deleteMessageEmbedsCmd = DeleteByMessageId(conn, "message_embeds");
await using var deleteMessageReactionsCmd = DeleteByMessageId(conn, "message_reactions");
await using var messageEditTimestampCmd = conn.Insert("message_edit_timestamps", [
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
]);
await using var messageRepliedToCmd = conn.Insert("message_replied_to", [
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
]);
await using var messageAttachmentCmd = conn.Insert("message_attachments", [
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer)
]);
await using var messageEmbedCmd = conn.Insert("message_embeds", [
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("json", SqliteType.Text) ("json", SqliteType.Text)
]); ]);
await using var reactionCmd = conn.Insert("reactions", [ await using var messageReactionCmd = conn.Insert("message_reactions", [
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer), ("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text), ("emoji_name", SqliteType.Text),
@ -104,29 +100,30 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
messageCmd.Set(":timestamp", message.Timestamp); messageCmd.Set(":timestamp", message.Timestamp);
await messageCmd.ExecuteNonQueryAsync(); await messageCmd.ExecuteNonQueryAsync();
await ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId); await ExecuteDeleteByMessageId(deleteMessageEditTimestampCmd, messageId);
await ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId); await ExecuteDeleteByMessageId(deleteMessageRepliedToCmd, messageId);
await ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId); await ExecuteDeleteByMessageId(deleteMessageAttachmentsCmd, messageId);
await ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId); await ExecuteDeleteByMessageId(deleteMessageEmbedsCmd, messageId);
await ExecuteDeleteByMessageId(deleteReactionsCmd, messageId); await ExecuteDeleteByMessageId(deleteMessageReactionsCmd, messageId);
if (message.EditTimestamp is {} timestamp) { if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId); messageEditTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp); messageEditTimestampCmd.Set(":edit_timestamp", timestamp);
await editTimestampCmd.ExecuteNonQueryAsync(); await messageEditTimestampCmd.ExecuteNonQueryAsync();
} }
if (message.RepliedToId is {} repliedToId) { if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId); messageRepliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId); messageRepliedToCmd.Set(":replied_to_id", repliedToId);
await repliedToCmd.ExecuteNonQueryAsync(); await messageRepliedToCmd.ExecuteNonQueryAsync();
} }
if (!message.Attachments.IsEmpty) { if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) { foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId); object attachmentId = attachment.Id;
attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":attachment_id", attachmentId);
attachmentCmd.Set(":name", attachment.Name); attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type); attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":normalized_url", attachment.NormalizedUrl); attachmentCmd.Set(":normalized_url", attachment.NormalizedUrl);
@ -136,15 +133,19 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
attachmentCmd.Set(":height", attachment.Height); attachmentCmd.Set(":height", attachment.Height);
await attachmentCmd.ExecuteNonQueryAsync(); await attachmentCmd.ExecuteNonQueryAsync();
messageAttachmentCmd.Set(":message_id", messageId);
messageAttachmentCmd.Set(":attachment_id", attachmentId);
await messageAttachmentCmd.ExecuteNonQueryAsync();
await downloadCollector.Add(DownloadLinkExtractor.FromAttachment(attachment)); await downloadCollector.Add(DownloadLinkExtractor.FromAttachment(attachment));
} }
} }
if (!message.Embeds.IsEmpty) { if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) { foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId); messageEmbedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json); messageEmbedCmd.Set(":json", embed.Json);
await embedCmd.ExecuteNonQueryAsync(); await messageEmbedCmd.ExecuteNonQueryAsync();
if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) { if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) {
await downloadCollector.Add(download); await downloadCollector.Add(download);
@ -154,12 +155,12 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
if (!message.Reactions.IsEmpty) { if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) { foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId); messageReactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId); messageReactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName); messageReactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags); messageReactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count); messageReactionCmd.Set(":count", reaction.Count);
await reactionCmd.ExecuteNonQueryAsync(); await messageReactionCmd.ExecuteNonQueryAsync();
if (reaction.EmojiId is {} emojiId) { if (reaction.EmojiId is {} emojiId) {
await downloadCollector.Add(DownloadLinkExtractor.FromEmoji(emojiId, reaction.EmojiFlags)); await downloadCollector.Add(DownloadLinkExtractor.FromEmoji(emojiId, reaction.EmojiFlags));
@ -221,7 +222,8 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
""" """
SELECT attachment_id, name, type, normalized_url, download_url, size, width, height SELECT attachment_id, name, type, normalized_url, download_url, size, width, height
FROM attachments FROM attachments
WHERE message_id = :message_id JOIN message_attachments USING (attachment_id)
WHERE message_attachments.message_id = :message_id
"""; """;
await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment { await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment {
@ -238,7 +240,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
const string EmbedSql = const string EmbedSql =
""" """
SELECT json SELECT json
FROM embeds FROM message_embeds
WHERE message_id = :message_id WHERE message_id = :message_id
"""; """;
@ -249,7 +251,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
const string ReactionSql = const string ReactionSql =
""" """
SELECT emoji_id, emoji_name, emoji_flags, count SELECT emoji_id, emoji_name, emoji_flags, count
FROM reactions FROM message_reactions
WHERE message_id = :message_id WHERE message_id = :message_id
"""; """;
@ -262,10 +264,10 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
await using var messageCmd = conn.Command( await using var messageCmd = conn.Command(
$""" $"""
SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, et.edit_timestamp, rt.replied_to_id SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, met.edit_timestamp, mrt.replied_to_id
FROM messages m FROM messages m
LEFT JOIN edit_timestamps et ON m.message_id = et.message_id LEFT JOIN message_edit_timestamps met ON m.message_id = met.message_id
LEFT JOIN replied_to rt ON m.message_id = rt.message_id LEFT JOIN message_replied_to mrt ON m.message_id = mrt.message_id
{filter.GenerateConditions("m").BuildWhereClause()} {filter.GenerateConditions("m").BuildWhereClause()}
""" """
); );

View File

@ -8,7 +8,10 @@ sealed class SqliteSchemaUpgradeTo3 : ISchemaUpgrade {
await reporter.MainWork("Applying schema changes...", 0, 1); await reporter.MainWork("Applying schema changes...", 0, 1);
await SqliteSchema.CreateMessageEditTimestampTable(conn); await SqliteSchema.CreateMessageEditTimestampTable(conn);
await conn.ExecuteAsync("ALTER TABLE message_edit_timestamps RENAME TO edit_timestamps");
await SqliteSchema.CreateMessageRepliedToTable(conn); await SqliteSchema.CreateMessageRepliedToTable(conn);
await conn.ExecuteAsync("ALTER TABLE message_replied_to RENAME TO replied_to");
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
INSERT INTO edit_timestamps (message_id, edit_timestamp) INSERT INTO edit_timestamps (message_id, edit_timestamp)

View File

@ -0,0 +1,23 @@
using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite.Schema;
sealed class SqliteSchemaUpgradeTo9 : ISchemaUpgrade {
async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
await reporter.MainWork("Applying schema changes...", 0, 3);
await SqliteSchema.CreateMessageAttachmentsTable(conn);
await reporter.MainWork("Migrating message attachments...", 1, 3);
await conn.ExecuteAsync("INSERT INTO message_attachments (message_id, attachment_id) SELECT message_id, attachment_id FROM attachments");
await reporter.MainWork("Applying schema changes...", 2, 3);
await conn.ExecuteAsync("DROP INDEX attachments_message_ix");
await conn.ExecuteAsync("ALTER TABLE attachments DROP COLUMN message_id");
await conn.ExecuteAsync("ALTER TABLE embeds RENAME TO message_embeds");
await conn.ExecuteAsync("ALTER TABLE edit_timestamps RENAME TO message_edit_timestamps");
await conn.ExecuteAsync("ALTER TABLE replied_to RENAME TO message_replied_to");
await conn.ExecuteAsync("ALTER TABLE reactions RENAME TO message_reactions");
}
}

View File

@ -8,7 +8,7 @@ using DHT.Utils.Logging;
namespace DHT.Server.Database.Sqlite; namespace DHT.Server.Database.Sqlite;
sealed class SqliteSchema { sealed class SqliteSchema {
internal const int Version = 8; internal const int Version = 9;
private static readonly Log Log = Log.ForType<SqliteSchema>(); private static readonly Log Log = Log.ForType<SqliteSchema>();
@ -86,7 +86,6 @@ sealed class SqliteSchema {
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE attachments ( CREATE TABLE attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL, attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
type TEXT, type TEXT,
@ -99,14 +98,14 @@ sealed class SqliteSchema {
"""); """);
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE embeds ( CREATE TABLE message_embeds (
message_id INTEGER NOT NULL, message_id INTEGER NOT NULL,
json TEXT NOT NULL json TEXT NOT NULL
) )
"""); """);
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE reactions ( CREATE TABLE message_reactions (
message_id INTEGER NOT NULL, message_id INTEGER NOT NULL,
emoji_id INTEGER, emoji_id INTEGER,
emoji_name TEXT, emoji_name TEXT,
@ -118,17 +117,17 @@ sealed class SqliteSchema {
await CreateMessageEditTimestampTable(conn); await CreateMessageEditTimestampTable(conn);
await CreateMessageRepliedToTable(conn); await CreateMessageRepliedToTable(conn);
await CreateDownloadTables(conn); await CreateDownloadTables(conn);
await CreateMessageAttachmentsTable(conn);
await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)"); await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON message_embeds(message_id)");
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)"); await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON message_reactions(message_id)");
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")"); await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
} }
internal static async Task CreateMessageEditTimestampTable(ISqliteConnection conn) { internal static async Task CreateMessageEditTimestampTable(ISqliteConnection conn) {
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE edit_timestamps ( CREATE TABLE message_edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL, message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL edit_timestamp INTEGER NOT NULL
) )
@ -137,7 +136,7 @@ sealed class SqliteSchema {
internal static async Task CreateMessageRepliedToTable(ISqliteConnection conn) { internal static async Task CreateMessageRepliedToTable(ISqliteConnection conn) {
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE replied_to ( CREATE TABLE message_replied_to (
message_id INTEGER PRIMARY KEY NOT NULL, message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL replied_to_id INTEGER NOT NULL
) )
@ -164,6 +163,18 @@ sealed class SqliteSchema {
"""); """);
} }
internal static async Task CreateMessageAttachmentsTable(ISqliteConnection conn) {
await conn.ExecuteAsync("""
CREATE TABLE message_attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL,
PRIMARY KEY (message_id, attachment_id),
FOREIGN KEY (message_id) REFERENCES messages (message_id) ON UPDATE CASCADE ON DELETE CASCADE,
FOREIGN KEY (attachment_id) REFERENCES attachments (attachment_id) ON UPDATE CASCADE ON DELETE CASCADE
)
""");
}
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) { private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
var upgrades = new Dictionary<int, ISchemaUpgrade> { var upgrades = new Dictionary<int, ISchemaUpgrade> {
{ 1, new SqliteSchemaUpgradeTo2() }, { 1, new SqliteSchemaUpgradeTo2() },
@ -173,6 +184,7 @@ sealed class SqliteSchema {
{ 5, new SqliteSchemaUpgradeTo6() }, { 5, new SqliteSchemaUpgradeTo6() },
{ 6, new SqliteSchemaUpgradeTo7() }, { 6, new SqliteSchemaUpgradeTo7() },
{ 7, new SqliteSchemaUpgradeTo8() }, { 7, new SqliteSchemaUpgradeTo8() },
{ 8, new SqliteSchemaUpgradeTo9() },
}; };
var perf = Log.Start("from version " + dbVersion); var perf = Log.Start("from version " + dbVersion);

Binary file not shown.