1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-10-19 05:42:50 +02:00

Compare commits

..

1 Commits

Author SHA1 Message Date
11f7d4a49f
Rewrite database interface to be asynchronous and improve UI 2023-12-29 21:26:21 +01:00
6 changed files with 218 additions and 214 deletions

View File

@ -34,7 +34,7 @@ sealed class StatusBarModel : BaseModel, IDisposable {
}
public void Dispose() {
state.Server.StatusChanged -= OnServerStatusChanged;
state.Server.StatusChanged += OnServerStatusChanged;
}
private void OnServerStatusChanged(object? sender, ServerManager.Status e) {

View File

@ -69,12 +69,12 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
public bool HasFailedDownloads => statisticsFailed.Items > 0;
private readonly State state;
private readonly ThrottledTask<int> enqueueDownloadItemsTask;
private readonly ThrottledTask enqueueDownloadItemsTask;
private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
private IDisposable? finishedItemsSubscription;
private int doneItemsCount;
private int totalEnqueuedItemCount;
private int initialFinishedCount;
private int? totalItemsToDownloadCount;
public AttachmentsPageModel() : this(State.Dummy) {}
@ -84,7 +84,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
FilterModel = new AttachmentFilterPanelModel(state);
enqueueDownloadItemsTask = new ThrottledTask<int>(OnItemsEnqueued, TaskScheduler.FromCurrentSynchronizationContext());
enqueueDownloadItemsTask = new ThrottledTask(RecomputeDownloadStatistics, TaskScheduler.FromCurrentSynchronizationContext());
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
RecomputeDownloadStatistics();
@ -93,7 +93,6 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
public void Dispose() {
state.Db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
enqueueDownloadItemsTask.Dispose();
downloadStatisticsTask.Dispose();
finishedItemsSubscription?.Dispose();
FilterModel.Dispose();
@ -114,7 +113,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
}
private async Task EnqueueDownloadItems() {
OnItemsEnqueued(await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter()));
await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter());
}
private void EnqueueDownloadItemsLater() {
@ -122,19 +121,49 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
enqueueDownloadItemsTask.Post(cancellationToken => state.Db.Downloads.EnqueueDownloadItems(filter, cancellationToken));
}
private void OnItemsEnqueued(int itemCount) {
totalEnqueuedItemCount += itemCount;
totalItemsToDownloadCount = totalEnqueuedItemCount;
UpdateDownloadMessage();
RecomputeDownloadStatistics();
}
private AttachmentFilter CreateAttachmentFilter() {
var filter = FilterModel.CreateFilter();
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
return filter;
}
private void RecomputeDownloadStatistics() {
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
var hadFailedDownloads = HasFailedDownloads;
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
statisticsFailed.Items = statusStatistics.FailedCount;
statisticsFailed.Size = statusStatistics.FailedSize;
statisticsSkipped.Items = statusStatistics.SkippedCount;
statisticsSkipped.Size = statusStatistics.SkippedSize;
OnPropertyChanged(nameof(StatisticsRows));
if (hadFailedDownloads != HasFailedDownloads) {
OnPropertyChanged(nameof(HasFailedDownloads));
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
}
totalItemsToDownloadCount = statisticsEnqueued.Items + statisticsDownloaded.Items + statisticsFailed.Items - initialFinishedCount;
UpdateDownloadMessage();
}
private void UpdateDownloadMessage() {
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
OnPropertyChanged(nameof(DownloadMessage));
OnPropertyChanged(nameof(DownloadProgress));
}
public async Task OnClickToggleDownload() {
IsToggleDownloadButtonEnabled = false;
@ -149,13 +178,14 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
await state.Db.Downloads.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
doneItemsCount = 0;
totalEnqueuedItemCount = 0;
initialFinishedCount = 0;
totalItemsToDownloadCount = null;
UpdateDownloadMessage();
}
else {
var finishedItems = await state.Downloader.Start();
initialFinishedCount = statisticsDownloaded.Items + statisticsFailed.Items;
finishedItemsSubscription = finishedItems.Select(static _ => true)
.Buffer(TimeSpan.FromMilliseconds(100))
.Select(static items => items.Count)
@ -201,42 +231,6 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
}
}
private void RecomputeDownloadStatistics() {
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
var hadFailedDownloads = HasFailedDownloads;
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
statisticsFailed.Items = statusStatistics.FailedCount;
statisticsFailed.Size = statusStatistics.FailedSize;
statisticsSkipped.Items = statusStatistics.SkippedCount;
statisticsSkipped.Size = statusStatistics.SkippedSize;
OnPropertyChanged(nameof(StatisticsRows));
if (hadFailedDownloads != HasFailedDownloads) {
OnPropertyChanged(nameof(HasFailedDownloads));
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
}
UpdateDownloadMessage();
}
private void UpdateDownloadMessage() {
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
OnPropertyChanged(nameof(DownloadMessage));
OnPropertyChanged(nameof(DownloadProgress));
}
public sealed class StatisticsRow {
public string State { get; }
public int Items { get; set; }

View File

@ -22,7 +22,7 @@ public interface IDownloadRepository {
Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl);
Task<int> EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
Task EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken = default);
@ -53,8 +53,8 @@ public interface IDownloadRepository {
return Task.FromResult<DownloadedAttachment?>(null);
}
public Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
return Task.FromResult(0);
public Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
return Task.CompletedTask;
}
public IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken) {

View File

@ -164,7 +164,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
};
}
public async Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
public async Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
using var conn = pool.Take();
await using var cmd = conn.Command(
@ -178,7 +178,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
);
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
return await cmd.ExecuteNonQueryAsync(cancellationToken);
await cmd.ExecuteNonQueryAsync(cancellationToken);
}
public async IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, [EnumeratorCancellation] CancellationToken cancellationToken) {

View File

@ -21,14 +21,14 @@ sealed class Schema {
}
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
conn.Execute(@"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
var dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
var dbVersionStr = conn.SelectScalar("SELECT value FROM metadata WHERE key = 'version'");
if (dbVersionStr == null) {
await InitializeSchemas();
InitializeSchemas();
}
else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
throw new InvalidDatabaseVersionException(dbVersionStr);
else if (!int.TryParse(dbVersionStr.ToString(), out int dbVersion) || dbVersion < 1) {
throw new InvalidDatabaseVersionException(dbVersionStr.ToString() ?? "<null>");
}
else if (dbVersion > Version) {
throw new DatabaseTooNewException(dbVersion);
@ -45,113 +45,113 @@ sealed class Schema {
return true;
}
private async Task InitializeSchemas() {
await conn.ExecuteAsync("""
CREATE TABLE users (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
avatar_url TEXT,
discriminator TEXT
)
""");
private void InitializeSchemas() {
conn.Execute("""
CREATE TABLE users (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
avatar_url TEXT,
discriminator TEXT
)
""");
await conn.ExecuteAsync("""
CREATE TABLE servers (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL
)
""");
conn.Execute("""
CREATE TABLE servers (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL
)
""");
await conn.ExecuteAsync("""
CREATE TABLE channels (
id INTEGER PRIMARY KEY NOT NULL,
server INTEGER NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER,
position INTEGER,
topic TEXT,
nsfw INTEGER
)
""");
conn.Execute("""
CREATE TABLE channels (
id INTEGER PRIMARY KEY NOT NULL,
server INTEGER NOT NULL,
name TEXT NOT NULL,
parent_id INTEGER,
position INTEGER,
topic TEXT,
nsfw INTEGER
)
""");
await conn.ExecuteAsync("""
CREATE TABLE messages (
message_id INTEGER PRIMARY KEY NOT NULL,
sender_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL,
text TEXT NOT NULL,
timestamp INTEGER NOT NULL
)
""");
conn.Execute("""
CREATE TABLE messages (
message_id INTEGER PRIMARY KEY NOT NULL,
sender_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL,
text TEXT NOT NULL,
timestamp INTEGER NOT NULL
)
""");
await conn.ExecuteAsync("""
CREATE TABLE attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
type TEXT,
normalized_url TEXT NOT NULL,
download_url TEXT,
size INTEGER NOT NULL,
width INTEGER,
height INTEGER
)
""");
conn.Execute("""
CREATE TABLE attachments (
message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
type TEXT,
normalized_url TEXT NOT NULL,
download_url TEXT,
size INTEGER NOT NULL,
width INTEGER,
height INTEGER
)
""");
await conn.ExecuteAsync("""
CREATE TABLE embeds (
message_id INTEGER NOT NULL,
json TEXT NOT NULL
)
""");
conn.Execute("""
CREATE TABLE embeds (
message_id INTEGER NOT NULL,
json TEXT NOT NULL
)
""");
await conn.ExecuteAsync("""
CREATE TABLE downloads (
normalized_url TEXT NOT NULL PRIMARY KEY,
download_url TEXT,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB
)
""");
conn.Execute("""
CREATE TABLE downloads (
normalized_url TEXT NOT NULL PRIMARY KEY,
download_url TEXT,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB
)
""");
await conn.ExecuteAsync("""
CREATE TABLE reactions (
message_id INTEGER NOT NULL,
emoji_id INTEGER,
emoji_name TEXT,
emoji_flags INTEGER NOT NULL,
count INTEGER NOT NULL
)
""");
conn.Execute("""
CREATE TABLE reactions (
message_id INTEGER NOT NULL,
emoji_id INTEGER,
emoji_name TEXT,
emoji_flags INTEGER NOT NULL,
count INTEGER NOT NULL
)
""");
await CreateMessageEditTimestampTable();
await CreateMessageRepliedToTable();
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
conn.Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
conn.Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
conn.Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
conn.Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
}
private async Task CreateMessageEditTimestampTable() {
await conn.ExecuteAsync("""
CREATE TABLE edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL
)
""");
private void CreateMessageEditTimestampTable() {
conn.Execute("""
CREATE TABLE edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL
)
""");
}
private async Task CreateMessageRepliedToTable() {
await conn.ExecuteAsync("""
CREATE TABLE replied_to (
message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL
)
""");
private void CreateMessageRepliedToTable() {
conn.Execute("""
CREATE TABLE replied_to (
message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL
)
""");
}
private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
@ -213,7 +213,7 @@ sealed class Schema {
}
}
await conn.ExecuteAsync("PRAGMA cache_size = -20000");
conn.Execute("PRAGMA cache_size = -20000");
DbTransaction tx;
@ -263,17 +263,17 @@ sealed class Schema {
await tx.CommitAsync();
await tx.DisposeAsync();
await conn.ExecuteAsync("PRAGMA cache_size = -2000");
conn.Execute("PRAGMA cache_size = -2000");
}
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
var perf = Log.Start("from version " + dbVersion);
await conn.ExecuteAsync("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
conn.Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
if (dbVersion <= 1) {
await reporter.MainWork("Applying schema changes...", 0, 1);
await conn.ExecuteAsync("ALTER TABLE channels ADD parent_id INTEGER");
conn.Execute("ALTER TABLE channels ADD parent_id INTEGER");
perf.Step("Upgrade to version 2");
await reporter.NextVersion();
@ -282,44 +282,44 @@ sealed class Schema {
if (dbVersion <= 2) {
await reporter.MainWork("Applying schema changes...", 0, 1);
await CreateMessageEditTimestampTable();
await CreateMessageRepliedToTable();
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
await conn.ExecuteAsync("""
INSERT INTO edit_timestamps (message_id, edit_timestamp)
SELECT message_id, edit_timestamp
FROM messages
WHERE edit_timestamp IS NOT NULL
""");
conn.Execute("""
INSERT INTO edit_timestamps (message_id, edit_timestamp)
SELECT message_id, edit_timestamp
FROM messages
WHERE edit_timestamp IS NOT NULL
""");
await conn.ExecuteAsync("""
INSERT INTO replied_to (message_id, replied_to_id)
SELECT message_id, replied_to_id
FROM messages
WHERE replied_to_id IS NOT NULL
""");
conn.Execute("""
INSERT INTO replied_to (message_id, replied_to_id)
SELECT message_id, replied_to_id
FROM messages
WHERE replied_to_id IS NOT NULL
""");
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN replied_to_id");
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN edit_timestamp");
conn.Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
conn.Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
perf.Step("Upgrade to version 3");
await reporter.MainWork("Vacuuming the database...", 1, 1);
await conn.ExecuteAsync("VACUUM");
conn.Execute("VACUUM");
perf.Step("Vacuum");
await reporter.NextVersion();
}
if (dbVersion <= 3) {
await conn.ExecuteAsync("""
CREATE TABLE downloads (
url TEXT NOT NULL PRIMARY KEY,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB
)
""");
conn.Execute("""
CREATE TABLE downloads (
url TEXT NOT NULL PRIMARY KEY,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB
)
""");
perf.Step("Upgrade to version 4");
await reporter.NextVersion();
@ -327,8 +327,8 @@ sealed class Schema {
if (dbVersion <= 4) {
await reporter.MainWork("Applying schema changes...", 0, 1);
await conn.ExecuteAsync("ALTER TABLE attachments ADD width INTEGER");
await conn.ExecuteAsync("ALTER TABLE attachments ADD height INTEGER");
conn.Execute("ALTER TABLE attachments ADD width INTEGER");
conn.Execute("ALTER TABLE attachments ADD height INTEGER");
perf.Step("Upgrade to version 5");
await reporter.NextVersion();
@ -336,8 +336,8 @@ sealed class Schema {
if (dbVersion <= 5) {
await reporter.MainWork("Applying schema changes...", 0, 3);
await conn.ExecuteAsync("ALTER TABLE attachments ADD download_url TEXT");
await conn.ExecuteAsync("ALTER TABLE downloads ADD download_url TEXT");
conn.Execute("ALTER TABLE attachments ADD download_url TEXT");
conn.Execute("ALTER TABLE downloads ADD download_url TEXT");
await reporter.MainWork("Updating attachments...", 1, 3);
await NormalizeAttachmentUrls(reporter);
@ -346,8 +346,8 @@ sealed class Schema {
await NormalizeDownloadUrls(reporter);
await reporter.MainWork("Applying schema changes...", 3, 3);
await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
conn.Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
conn.Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
perf.Step("Upgrade to version 6");
await reporter.NextVersion();

View File

@ -19,6 +19,11 @@ static class SqliteExtensions {
return cmd;
}
public static void Execute(this ISqliteConnection conn, string sql) {
using var cmd = conn.Command(sql);
cmd.ExecuteNonQuery();
}
public static async Task<int> ExecuteAsync(this ISqliteConnection conn, [LanguageInjection("sql")] string sql, CancellationToken cancellationToken = default) {
await using var cmd = conn.Command(sql);
return await cmd.ExecuteNonQueryAsync(cancellationToken);
@ -31,6 +36,11 @@ static class SqliteExtensions {
return reader.Read() ? readFunction(reader) : readFunction(null);
}
public static object? SelectScalar(this ISqliteConnection conn, string sql) {
using var cmd = conn.Command(sql);
return cmd.ExecuteScalar();
}
public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
string columnNames = string.Join(',', columns.Select(static c => c.Name));
string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name));