1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-11-24 20:42:46 +01:00

Compare commits

...

2 Commits

5 changed files with 112 additions and 72 deletions

View File

@ -14,16 +14,8 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository {
sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IMessageRepository {
private static readonly Log Log = Log.ForType<SqliteMessageRepository>();
private readonly SqliteConnectionPool pool;
private readonly SqliteDownloadRepository downloads;
public SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : base(Log) {
this.pool = pool;
this.downloads = downloads;
}
public async Task Add(IReadOnlyList<Message> messages) {
if (messages.Count == 0) {
@ -50,25 +42,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
("timestamp", SqliteType.Integer)
]);
await using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
await using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
await using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
await using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
await using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
await using var editTimestampCmd = conn.Insert("edit_timestamps", [
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
]);
await using var repliedToCmd = conn.Insert("replied_to", [
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
]);
await using var attachmentCmd = conn.Insert("attachments", [
("message_id", SqliteType.Integer),
await using var attachmentCmd = conn.Upsert("attachments", [
("attachment_id", SqliteType.Integer),
("name", SqliteType.Text),
("type", SqliteType.Text),
@ -79,19 +53,41 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
("height", SqliteType.Integer)
]);
await using var embedCmd = conn.Insert("embeds", [
await using var deleteMessageEditTimestampCmd = DeleteByMessageId(conn, "message_edit_timestamps");
await using var deleteMessageRepliedToCmd = DeleteByMessageId(conn, "message_replied_to");
await using var deleteMessageAttachmentsCmd = DeleteByMessageId(conn, "message_attachments");
await using var deleteMessageEmbedsCmd = DeleteByMessageId(conn, "message_embeds");
await using var deleteMessageReactionsCmd = DeleteByMessageId(conn, "message_reactions");
await using var messageEditTimestampCmd = conn.Insert("message_edit_timestamps", [
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
]);
await using var messageRepliedToCmd = conn.Insert("message_replied_to", [
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
]);
await using var messageAttachmentCmd = conn.Insert("message_attachments", [
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer)
]);
await using var messageEmbedCmd = conn.Insert("message_embeds", [
("message_id", SqliteType.Integer),
("json", SqliteType.Text)
]);
await using var reactionCmd = conn.Insert("reactions", [
await using var messageReactionCmd = conn.Insert("message_reactions", [
("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer)
]);
await using var downloadCollector = new SqliteDownloadRepository.NewDownloadCollector(downloads, conn);
foreach (var message in messages) {
@ -104,29 +100,30 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
messageCmd.Set(":timestamp", message.Timestamp);
await messageCmd.ExecuteNonQueryAsync();
await ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
await ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
await ExecuteDeleteByMessageId(deleteMessageEditTimestampCmd, messageId);
await ExecuteDeleteByMessageId(deleteMessageRepliedToCmd, messageId);
await ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
await ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
await ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
await ExecuteDeleteByMessageId(deleteMessageAttachmentsCmd, messageId);
await ExecuteDeleteByMessageId(deleteMessageEmbedsCmd, messageId);
await ExecuteDeleteByMessageId(deleteMessageReactionsCmd, messageId);
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
await editTimestampCmd.ExecuteNonQueryAsync();
messageEditTimestampCmd.Set(":message_id", messageId);
messageEditTimestampCmd.Set(":edit_timestamp", timestamp);
await messageEditTimestampCmd.ExecuteNonQueryAsync();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
await repliedToCmd.ExecuteNonQueryAsync();
messageRepliedToCmd.Set(":message_id", messageId);
messageRepliedToCmd.Set(":replied_to_id", repliedToId);
await messageRepliedToCmd.ExecuteNonQueryAsync();
}
if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
object attachmentId = attachment.Id;
attachmentCmd.Set(":attachment_id", attachmentId);
attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":normalized_url", attachment.NormalizedUrl);
@ -135,16 +132,20 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
attachmentCmd.Set(":width", attachment.Width);
attachmentCmd.Set(":height", attachment.Height);
await attachmentCmd.ExecuteNonQueryAsync();
messageAttachmentCmd.Set(":message_id", messageId);
messageAttachmentCmd.Set(":attachment_id", attachmentId);
await messageAttachmentCmd.ExecuteNonQueryAsync();
await downloadCollector.Add(DownloadLinkExtractor.FromAttachment(attachment));
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
await embedCmd.ExecuteNonQueryAsync();
messageEmbedCmd.Set(":message_id", messageId);
messageEmbedCmd.Set(":json", embed.Json);
await messageEmbedCmd.ExecuteNonQueryAsync();
if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) {
await downloadCollector.Add(download);
@ -154,12 +155,12 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
await reactionCmd.ExecuteNonQueryAsync();
messageReactionCmd.Set(":message_id", messageId);
messageReactionCmd.Set(":emoji_id", reaction.EmojiId);
messageReactionCmd.Set(":emoji_name", reaction.EmojiName);
messageReactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
messageReactionCmd.Set(":count", reaction.Count);
await messageReactionCmd.ExecuteNonQueryAsync();
if (reaction.EmojiId is {} emojiId) {
await downloadCollector.Add(DownloadLinkExtractor.FromEmoji(emojiId, reaction.EmojiFlags));
@ -178,7 +179,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
public override Task<long> Count(CancellationToken cancellationToken) {
return Count(filter: null, cancellationToken);
}
public async Task<long> Count(MessageFilter? filter, CancellationToken cancellationToken) {
await using var conn = await pool.Take();
return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
@ -221,7 +222,8 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
"""
SELECT attachment_id, name, type, normalized_url, download_url, size, width, height
FROM attachments
WHERE message_id = :message_id
JOIN message_attachments USING (attachment_id)
WHERE message_attachments.message_id = :message_id
""";
await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment {
@ -238,7 +240,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
const string EmbedSql =
"""
SELECT json
FROM embeds
FROM message_embeds
WHERE message_id = :message_id
""";
@ -249,7 +251,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
const string ReactionSql =
"""
SELECT emoji_id, emoji_name, emoji_flags, count
FROM reactions
FROM message_reactions
WHERE message_id = :message_id
""";
@ -262,10 +264,10 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
await using var messageCmd = conn.Command(
$"""
SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, et.edit_timestamp, rt.replied_to_id
SELECT m.message_id, m.sender_id, m.channel_id, m.text, m.timestamp, met.edit_timestamp, mrt.replied_to_id
FROM messages m
LEFT JOIN edit_timestamps et ON m.message_id = et.message_id
LEFT JOIN replied_to rt ON m.message_id = rt.message_id
LEFT JOIN message_edit_timestamps met ON m.message_id = met.message_id
LEFT JOIN message_replied_to mrt ON m.message_id = mrt.message_id
{filter.GenerateConditions("m").BuildWhereClause()}
"""
);
@ -292,7 +294,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
public async IAsyncEnumerable<ulong> GetIds(MessageFilter? filter) {
await using var conn = await pool.Take();
await using var cmd = conn.Command("SELECT message_id FROM messages" + filter.GenerateConditions().BuildWhereClause());
await using var reader = await cmd.ExecuteReaderAsync();

View File

@ -8,7 +8,10 @@ sealed class SqliteSchemaUpgradeTo3 : ISchemaUpgrade {
await reporter.MainWork("Applying schema changes...", 0, 1);
await SqliteSchema.CreateMessageEditTimestampTable(conn);
await conn.ExecuteAsync("ALTER TABLE message_edit_timestamps RENAME TO edit_timestamps");
await SqliteSchema.CreateMessageRepliedToTable(conn);
await conn.ExecuteAsync("ALTER TABLE message_replied_to RENAME TO replied_to");
await conn.ExecuteAsync("""
INSERT INTO edit_timestamps (message_id, edit_timestamp)

View File

@ -0,0 +1,23 @@
using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite.Schema;
sealed class SqliteSchemaUpgradeTo9 : ISchemaUpgrade {
async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
await reporter.MainWork("Applying schema changes...", 0, 3);
await SqliteSchema.CreateMessageAttachmentsTable(conn);
await reporter.MainWork("Migrating message attachments...", 1, 3);
await conn.ExecuteAsync("INSERT INTO message_attachments (message_id, attachment_id) SELECT message_id, attachment_id FROM attachments");
await reporter.MainWork("Applying schema changes...", 2, 3);
await conn.ExecuteAsync("DROP INDEX attachments_message_ix");
await conn.ExecuteAsync("ALTER TABLE attachments DROP COLUMN message_id");
await conn.ExecuteAsync("ALTER TABLE embeds RENAME TO message_embeds");
await conn.ExecuteAsync("ALTER TABLE edit_timestamps RENAME TO message_edit_timestamps");
await conn.ExecuteAsync("ALTER TABLE replied_to RENAME TO message_replied_to");
await conn.ExecuteAsync("ALTER TABLE reactions RENAME TO message_reactions");
}
}

View File

@ -8,7 +8,7 @@ using DHT.Utils.Logging;
namespace DHT.Server.Database.Sqlite;
sealed class SqliteSchema {
internal const int Version = 8;
internal const int Version = 9;
private static readonly Log Log = Log.ForType<SqliteSchema>();
@ -86,7 +86,6 @@ sealed class SqliteSchema {
await conn.ExecuteAsync("""
CREATE TABLE attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
type TEXT,
@ -99,14 +98,14 @@ sealed class SqliteSchema {
""");
await conn.ExecuteAsync("""
CREATE TABLE embeds (
CREATE TABLE message_embeds (
message_id INTEGER NOT NULL,
json TEXT NOT NULL
)
""");
await conn.ExecuteAsync("""
CREATE TABLE reactions (
CREATE TABLE message_reactions (
message_id INTEGER NOT NULL,
emoji_id INTEGER,
emoji_name TEXT,
@ -118,17 +117,17 @@ sealed class SqliteSchema {
await CreateMessageEditTimestampTable(conn);
await CreateMessageRepliedToTable(conn);
await CreateDownloadTables(conn);
await CreateMessageAttachmentsTable(conn);
await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON message_embeds(message_id)");
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON message_reactions(message_id)");
await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
}
internal static async Task CreateMessageEditTimestampTable(ISqliteConnection conn) {
await conn.ExecuteAsync("""
CREATE TABLE edit_timestamps (
CREATE TABLE message_edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL
)
@ -137,7 +136,7 @@ sealed class SqliteSchema {
internal static async Task CreateMessageRepliedToTable(ISqliteConnection conn) {
await conn.ExecuteAsync("""
CREATE TABLE replied_to (
CREATE TABLE message_replied_to (
message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL
)
@ -163,6 +162,18 @@ sealed class SqliteSchema {
)
""");
}
internal static async Task CreateMessageAttachmentsTable(ISqliteConnection conn) {
await conn.ExecuteAsync("""
CREATE TABLE message_attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL,
PRIMARY KEY (message_id, attachment_id),
FOREIGN KEY (message_id) REFERENCES messages (message_id) ON UPDATE CASCADE ON DELETE CASCADE,
FOREIGN KEY (attachment_id) REFERENCES attachments (attachment_id) ON UPDATE CASCADE ON DELETE CASCADE
)
""");
}
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
var upgrades = new Dictionary<int, ISchemaUpgrade> {
@ -173,13 +184,14 @@ sealed class SqliteSchema {
{ 5, new SqliteSchemaUpgradeTo6() },
{ 6, new SqliteSchemaUpgradeTo7() },
{ 7, new SqliteSchemaUpgradeTo8() },
{ 8, new SqliteSchemaUpgradeTo9() },
};
var perf = Log.Start("from version " + dbVersion);
for (int fromVersion = dbVersion; fromVersion < Version; fromVersion++) {
var toVersion = fromVersion + 1;
if (upgrades.TryGetValue(fromVersion, out var upgrade)) {
await upgrade.Run(conn, reporter);
}

Binary file not shown.