mirror of
https://github.com/chylex/Discord-History-Tracker.git
synced 2024-11-25 14:42:44 +01:00
Compare commits
3 Commits
11f7d4a49f
...
e0f359c15b
Author | SHA1 | Date | |
---|---|---|---|
e0f359c15b | |||
935f11d736 | |||
f64141e768 |
@ -34,7 +34,7 @@ sealed class StatusBarModel : BaseModel, IDisposable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose() {
|
public void Dispose() {
|
||||||
state.Server.StatusChanged += OnServerStatusChanged;
|
state.Server.StatusChanged -= OnServerStatusChanged;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void OnServerStatusChanged(object? sender, ServerManager.Status e) {
|
private void OnServerStatusChanged(object? sender, ServerManager.Status e) {
|
||||||
|
@ -69,12 +69,12 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
public bool HasFailedDownloads => statisticsFailed.Items > 0;
|
public bool HasFailedDownloads => statisticsFailed.Items > 0;
|
||||||
|
|
||||||
private readonly State state;
|
private readonly State state;
|
||||||
private readonly ThrottledTask enqueueDownloadItemsTask;
|
private readonly ThrottledTask<int> enqueueDownloadItemsTask;
|
||||||
private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
|
private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
|
||||||
|
|
||||||
private IDisposable? finishedItemsSubscription;
|
private IDisposable? finishedItemsSubscription;
|
||||||
private int doneItemsCount;
|
private int doneItemsCount;
|
||||||
private int initialFinishedCount;
|
private int totalEnqueuedItemCount;
|
||||||
private int? totalItemsToDownloadCount;
|
private int? totalItemsToDownloadCount;
|
||||||
|
|
||||||
public AttachmentsPageModel() : this(State.Dummy) {}
|
public AttachmentsPageModel() : this(State.Dummy) {}
|
||||||
@ -84,7 +84,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
|
|
||||||
FilterModel = new AttachmentFilterPanelModel(state);
|
FilterModel = new AttachmentFilterPanelModel(state);
|
||||||
|
|
||||||
enqueueDownloadItemsTask = new ThrottledTask(RecomputeDownloadStatistics, TaskScheduler.FromCurrentSynchronizationContext());
|
enqueueDownloadItemsTask = new ThrottledTask<int>(OnItemsEnqueued, TaskScheduler.FromCurrentSynchronizationContext());
|
||||||
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
|
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
|
||||||
RecomputeDownloadStatistics();
|
RecomputeDownloadStatistics();
|
||||||
|
|
||||||
@ -93,6 +93,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
|
|
||||||
public void Dispose() {
|
public void Dispose() {
|
||||||
state.Db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
|
state.Db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
|
||||||
|
enqueueDownloadItemsTask.Dispose();
|
||||||
downloadStatisticsTask.Dispose();
|
downloadStatisticsTask.Dispose();
|
||||||
finishedItemsSubscription?.Dispose();
|
finishedItemsSubscription?.Dispose();
|
||||||
FilterModel.Dispose();
|
FilterModel.Dispose();
|
||||||
@ -113,7 +114,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async Task EnqueueDownloadItems() {
|
private async Task EnqueueDownloadItems() {
|
||||||
await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter());
|
OnItemsEnqueued(await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnqueueDownloadItemsLater() {
|
private void EnqueueDownloadItemsLater() {
|
||||||
@ -121,49 +122,19 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
enqueueDownloadItemsTask.Post(cancellationToken => state.Db.Downloads.EnqueueDownloadItems(filter, cancellationToken));
|
enqueueDownloadItemsTask.Post(cancellationToken => state.Db.Downloads.EnqueueDownloadItems(filter, cancellationToken));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void OnItemsEnqueued(int itemCount) {
|
||||||
|
totalEnqueuedItemCount += itemCount;
|
||||||
|
totalItemsToDownloadCount = totalEnqueuedItemCount;
|
||||||
|
UpdateDownloadMessage();
|
||||||
|
RecomputeDownloadStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
private AttachmentFilter CreateAttachmentFilter() {
|
private AttachmentFilter CreateAttachmentFilter() {
|
||||||
var filter = FilterModel.CreateFilter();
|
var filter = FilterModel.CreateFilter();
|
||||||
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
|
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
|
||||||
return filter;
|
return filter;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void RecomputeDownloadStatistics() {
|
|
||||||
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
|
|
||||||
var hadFailedDownloads = HasFailedDownloads;
|
|
||||||
|
|
||||||
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
|
|
||||||
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
|
|
||||||
|
|
||||||
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
|
|
||||||
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
|
|
||||||
|
|
||||||
statisticsFailed.Items = statusStatistics.FailedCount;
|
|
||||||
statisticsFailed.Size = statusStatistics.FailedSize;
|
|
||||||
|
|
||||||
statisticsSkipped.Items = statusStatistics.SkippedCount;
|
|
||||||
statisticsSkipped.Size = statusStatistics.SkippedSize;
|
|
||||||
|
|
||||||
OnPropertyChanged(nameof(StatisticsRows));
|
|
||||||
|
|
||||||
if (hadFailedDownloads != HasFailedDownloads) {
|
|
||||||
OnPropertyChanged(nameof(HasFailedDownloads));
|
|
||||||
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
|
|
||||||
}
|
|
||||||
|
|
||||||
totalItemsToDownloadCount = statisticsEnqueued.Items + statisticsDownloaded.Items + statisticsFailed.Items - initialFinishedCount;
|
|
||||||
UpdateDownloadMessage();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateDownloadMessage() {
|
|
||||||
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
|
|
||||||
|
|
||||||
OnPropertyChanged(nameof(DownloadMessage));
|
|
||||||
OnPropertyChanged(nameof(DownloadProgress));
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task OnClickToggleDownload() {
|
public async Task OnClickToggleDownload() {
|
||||||
IsToggleDownloadButtonEnabled = false;
|
IsToggleDownloadButtonEnabled = false;
|
||||||
|
|
||||||
@ -178,14 +149,13 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
await state.Db.Downloads.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
|
await state.Db.Downloads.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
|
||||||
|
|
||||||
doneItemsCount = 0;
|
doneItemsCount = 0;
|
||||||
initialFinishedCount = 0;
|
totalEnqueuedItemCount = 0;
|
||||||
totalItemsToDownloadCount = null;
|
totalItemsToDownloadCount = null;
|
||||||
UpdateDownloadMessage();
|
UpdateDownloadMessage();
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
var finishedItems = await state.Downloader.Start();
|
var finishedItems = await state.Downloader.Start();
|
||||||
|
|
||||||
initialFinishedCount = statisticsDownloaded.Items + statisticsFailed.Items;
|
|
||||||
finishedItemsSubscription = finishedItems.Select(static _ => true)
|
finishedItemsSubscription = finishedItems.Select(static _ => true)
|
||||||
.Buffer(TimeSpan.FromMilliseconds(100))
|
.Buffer(TimeSpan.FromMilliseconds(100))
|
||||||
.Select(static items => items.Count)
|
.Select(static items => items.Count)
|
||||||
@ -231,6 +201,42 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void RecomputeDownloadStatistics() {
|
||||||
|
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
|
||||||
|
var hadFailedDownloads = HasFailedDownloads;
|
||||||
|
|
||||||
|
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
|
||||||
|
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
|
||||||
|
|
||||||
|
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
|
||||||
|
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
|
||||||
|
|
||||||
|
statisticsFailed.Items = statusStatistics.FailedCount;
|
||||||
|
statisticsFailed.Size = statusStatistics.FailedSize;
|
||||||
|
|
||||||
|
statisticsSkipped.Items = statusStatistics.SkippedCount;
|
||||||
|
statisticsSkipped.Size = statusStatistics.SkippedSize;
|
||||||
|
|
||||||
|
OnPropertyChanged(nameof(StatisticsRows));
|
||||||
|
|
||||||
|
if (hadFailedDownloads != HasFailedDownloads) {
|
||||||
|
OnPropertyChanged(nameof(HasFailedDownloads));
|
||||||
|
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
|
||||||
|
}
|
||||||
|
|
||||||
|
UpdateDownloadMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateDownloadMessage() {
|
||||||
|
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
|
||||||
|
|
||||||
|
OnPropertyChanged(nameof(DownloadMessage));
|
||||||
|
OnPropertyChanged(nameof(DownloadProgress));
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class StatisticsRow {
|
public sealed class StatisticsRow {
|
||||||
public string State { get; }
|
public string State { get; }
|
||||||
public int Items { get; set; }
|
public int Items { get; set; }
|
||||||
|
@ -22,7 +22,7 @@ public interface IDownloadRepository {
|
|||||||
|
|
||||||
Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl);
|
Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl);
|
||||||
|
|
||||||
Task EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
|
Task<int> EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken = default);
|
IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
@ -53,8 +53,8 @@ public interface IDownloadRepository {
|
|||||||
return Task.FromResult<DownloadedAttachment?>(null);
|
return Task.FromResult<DownloadedAttachment?>(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
public Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||||
return Task.CompletedTask;
|
return Task.FromResult(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken) {
|
public IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken) {
|
||||||
|
@ -164,7 +164,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
public async Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||||
using var conn = pool.Take();
|
using var conn = pool.Take();
|
||||||
|
|
||||||
await using var cmd = conn.Command(
|
await using var cmd = conn.Command(
|
||||||
@ -178,7 +178,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
|
|||||||
);
|
);
|
||||||
|
|
||||||
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
|
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
|
||||||
await cmd.ExecuteNonQueryAsync(cancellationToken);
|
return await cmd.ExecuteNonQueryAsync(cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, [EnumeratorCancellation] CancellationToken cancellationToken) {
|
public async IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, [EnumeratorCancellation] CancellationToken cancellationToken) {
|
||||||
|
@ -21,14 +21,14 @@ sealed class Schema {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
|
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
|
||||||
conn.Execute(@"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
|
await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
|
||||||
|
|
||||||
var dbVersionStr = conn.SelectScalar("SELECT value FROM metadata WHERE key = 'version'");
|
var dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
|
||||||
if (dbVersionStr == null) {
|
if (dbVersionStr == null) {
|
||||||
InitializeSchemas();
|
await InitializeSchemas();
|
||||||
}
|
}
|
||||||
else if (!int.TryParse(dbVersionStr.ToString(), out int dbVersion) || dbVersion < 1) {
|
else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
|
||||||
throw new InvalidDatabaseVersionException(dbVersionStr.ToString() ?? "<null>");
|
throw new InvalidDatabaseVersionException(dbVersionStr);
|
||||||
}
|
}
|
||||||
else if (dbVersion > Version) {
|
else if (dbVersion > Version) {
|
||||||
throw new DatabaseTooNewException(dbVersion);
|
throw new DatabaseTooNewException(dbVersion);
|
||||||
@ -45,123 +45,123 @@ sealed class Schema {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void InitializeSchemas() {
|
private async Task InitializeSchemas() {
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE users (
|
CREATE TABLE users (
|
||||||
id INTEGER PRIMARY KEY NOT NULL,
|
id INTEGER PRIMARY KEY NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
avatar_url TEXT,
|
avatar_url TEXT,
|
||||||
discriminator TEXT
|
discriminator TEXT
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE servers (
|
CREATE TABLE servers (
|
||||||
id INTEGER PRIMARY KEY NOT NULL,
|
id INTEGER PRIMARY KEY NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
type TEXT NOT NULL
|
type TEXT NOT NULL
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE channels (
|
CREATE TABLE channels (
|
||||||
id INTEGER PRIMARY KEY NOT NULL,
|
id INTEGER PRIMARY KEY NOT NULL,
|
||||||
server INTEGER NOT NULL,
|
server INTEGER NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
parent_id INTEGER,
|
parent_id INTEGER,
|
||||||
position INTEGER,
|
position INTEGER,
|
||||||
topic TEXT,
|
topic TEXT,
|
||||||
nsfw INTEGER
|
nsfw INTEGER
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE messages (
|
CREATE TABLE messages (
|
||||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||||
sender_id INTEGER NOT NULL,
|
sender_id INTEGER NOT NULL,
|
||||||
channel_id INTEGER NOT NULL,
|
channel_id INTEGER NOT NULL,
|
||||||
text TEXT NOT NULL,
|
text TEXT NOT NULL,
|
||||||
timestamp INTEGER NOT NULL
|
timestamp INTEGER NOT NULL
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE attachments (
|
CREATE TABLE attachments (
|
||||||
message_id INTEGER NOT NULL,
|
message_id INTEGER NOT NULL,
|
||||||
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
|
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
type TEXT,
|
type TEXT,
|
||||||
normalized_url TEXT NOT NULL,
|
normalized_url TEXT NOT NULL,
|
||||||
download_url TEXT,
|
download_url TEXT,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
width INTEGER,
|
width INTEGER,
|
||||||
height INTEGER
|
height INTEGER
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE embeds (
|
CREATE TABLE embeds (
|
||||||
message_id INTEGER NOT NULL,
|
message_id INTEGER NOT NULL,
|
||||||
json TEXT NOT NULL
|
json TEXT NOT NULL
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE downloads (
|
CREATE TABLE downloads (
|
||||||
normalized_url TEXT NOT NULL PRIMARY KEY,
|
normalized_url TEXT NOT NULL PRIMARY KEY,
|
||||||
download_url TEXT,
|
download_url TEXT,
|
||||||
status INTEGER NOT NULL,
|
status INTEGER NOT NULL,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
blob BLOB
|
blob BLOB
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("""
|
|
||||||
CREATE TABLE reactions (
|
|
||||||
message_id INTEGER NOT NULL,
|
|
||||||
emoji_id INTEGER,
|
|
||||||
emoji_name TEXT,
|
|
||||||
emoji_flags INTEGER NOT NULL,
|
|
||||||
count INTEGER NOT NULL
|
|
||||||
)
|
|
||||||
""");
|
|
||||||
|
|
||||||
CreateMessageEditTimestampTable();
|
await conn.ExecuteAsync("""
|
||||||
CreateMessageRepliedToTable();
|
CREATE TABLE reactions (
|
||||||
|
message_id INTEGER NOT NULL,
|
||||||
|
emoji_id INTEGER,
|
||||||
|
emoji_name TEXT,
|
||||||
|
emoji_flags INTEGER NOT NULL,
|
||||||
|
count INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
""");
|
||||||
|
|
||||||
conn.Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
|
await CreateMessageEditTimestampTable();
|
||||||
conn.Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
|
await CreateMessageRepliedToTable();
|
||||||
conn.Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
|
|
||||||
|
|
||||||
conn.Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
|
await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
|
||||||
|
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
|
||||||
|
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
|
||||||
|
|
||||||
|
await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void CreateMessageEditTimestampTable() {
|
private async Task CreateMessageEditTimestampTable() {
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE edit_timestamps (
|
CREATE TABLE edit_timestamps (
|
||||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||||
edit_timestamp INTEGER NOT NULL
|
edit_timestamp INTEGER NOT NULL
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void CreateMessageRepliedToTable() {
|
private async Task CreateMessageRepliedToTable() {
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE replied_to (
|
CREATE TABLE replied_to (
|
||||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||||
replied_to_id INTEGER NOT NULL
|
replied_to_id INTEGER NOT NULL
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||||
await reporter.SubWork("Preparing attachments...", 0, 0);
|
await reporter.SubWork("Preparing attachments...", 0, 0);
|
||||||
|
|
||||||
var normalizedUrls = new Dictionary<long, string>();
|
var normalizedUrls = new Dictionary<long, string>();
|
||||||
|
|
||||||
await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
|
await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
|
||||||
await using var reader = await selectCmd.ExecuteReaderAsync();
|
await using var reader = await selectCmd.ExecuteReaderAsync();
|
||||||
|
|
||||||
while (reader.Read()) {
|
while (reader.Read()) {
|
||||||
var attachmentId = reader.GetInt64(0);
|
var attachmentId = reader.GetInt64(0);
|
||||||
var originalUrl = reader.GetString(1);
|
var originalUrl = reader.GetString(1);
|
||||||
@ -177,7 +177,7 @@ sealed class Schema {
|
|||||||
await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
|
await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
|
||||||
updateCmd.Add(":attachment_id", SqliteType.Integer);
|
updateCmd.Add(":attachment_id", SqliteType.Integer);
|
||||||
updateCmd.Add(":normalized_url", SqliteType.Text);
|
updateCmd.Add(":normalized_url", SqliteType.Text);
|
||||||
|
|
||||||
foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
|
foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
|
||||||
if (++processedUrls % 1000 == 0) {
|
if (++processedUrls % 1000 == 0) {
|
||||||
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
||||||
@ -188,15 +188,15 @@ sealed class Schema {
|
|||||||
updateCmd.ExecuteNonQuery();
|
updateCmd.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
||||||
|
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task NormalizeDownloadUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
private async Task NormalizeDownloadUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||||
await reporter.SubWork("Preparing downloads...", 0, 0);
|
await reporter.SubWork("Preparing downloads...", 0, 0);
|
||||||
|
|
||||||
var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
|
var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
|
||||||
var duplicateUrlsToDelete = new HashSet<string>();
|
var duplicateUrlsToDelete = new HashSet<string>();
|
||||||
|
|
||||||
@ -213,10 +213,10 @@ sealed class Schema {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
conn.Execute("PRAGMA cache_size = -20000");
|
await conn.ExecuteAsync("PRAGMA cache_size = -20000");
|
||||||
|
|
||||||
DbTransaction tx;
|
DbTransaction tx;
|
||||||
|
|
||||||
await using (tx = await conn.BeginTransactionAsync()) {
|
await using (tx = await conn.BeginTransactionAsync()) {
|
||||||
await reporter.SubWork("Deleting duplicates...", 0, 0);
|
await reporter.SubWork("Deleting duplicates...", 0, 0);
|
||||||
|
|
||||||
@ -226,7 +226,7 @@ sealed class Schema {
|
|||||||
deleteCmd.ExecuteNonQuery();
|
deleteCmd.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -234,20 +234,20 @@ sealed class Schema {
|
|||||||
int processedUrls = -1;
|
int processedUrls = -1;
|
||||||
|
|
||||||
tx = await conn.BeginTransactionAsync();
|
tx = await conn.BeginTransactionAsync();
|
||||||
|
|
||||||
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
|
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
|
||||||
updateCmd.Add(":normalized_url", SqliteType.Text);
|
updateCmd.Add(":normalized_url", SqliteType.Text);
|
||||||
updateCmd.Add(":download_url", SqliteType.Text);
|
updateCmd.Add(":download_url", SqliteType.Text);
|
||||||
|
|
||||||
foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
|
foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
|
||||||
if (++processedUrls % 100 == 0) {
|
if (++processedUrls % 100 == 0) {
|
||||||
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
||||||
|
|
||||||
// Not proper way of dealing with transactions, but it avoids a long commit at the end.
|
// Not proper way of dealing with transactions, but it avoids a long commit at the end.
|
||||||
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
|
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
await tx.DisposeAsync();
|
await tx.DisposeAsync();
|
||||||
|
|
||||||
tx = await conn.BeginTransactionAsync();
|
tx = await conn.BeginTransactionAsync();
|
||||||
updateCmd.Transaction = (SqliteTransaction) tx;
|
updateCmd.Transaction = (SqliteTransaction) tx;
|
||||||
}
|
}
|
||||||
@ -257,98 +257,98 @@ sealed class Schema {
|
|||||||
updateCmd.ExecuteNonQuery();
|
updateCmd.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
||||||
|
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
await tx.DisposeAsync();
|
await tx.DisposeAsync();
|
||||||
|
|
||||||
conn.Execute("PRAGMA cache_size = -2000");
|
await conn.ExecuteAsync("PRAGMA cache_size = -2000");
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||||
var perf = Log.Start("from version " + dbVersion);
|
var perf = Log.Start("from version " + dbVersion);
|
||||||
|
|
||||||
conn.Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
|
await conn.ExecuteAsync("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
|
||||||
|
|
||||||
if (dbVersion <= 1) {
|
if (dbVersion <= 1) {
|
||||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||||
conn.Execute("ALTER TABLE channels ADD parent_id INTEGER");
|
await conn.ExecuteAsync("ALTER TABLE channels ADD parent_id INTEGER");
|
||||||
|
|
||||||
perf.Step("Upgrade to version 2");
|
perf.Step("Upgrade to version 2");
|
||||||
await reporter.NextVersion();
|
await reporter.NextVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbVersion <= 2) {
|
if (dbVersion <= 2) {
|
||||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||||
|
|
||||||
CreateMessageEditTimestampTable();
|
|
||||||
CreateMessageRepliedToTable();
|
|
||||||
|
|
||||||
conn.Execute("""
|
await CreateMessageEditTimestampTable();
|
||||||
INSERT INTO edit_timestamps (message_id, edit_timestamp)
|
await CreateMessageRepliedToTable();
|
||||||
SELECT message_id, edit_timestamp
|
|
||||||
FROM messages
|
|
||||||
WHERE edit_timestamp IS NOT NULL
|
|
||||||
""");
|
|
||||||
|
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
INSERT INTO replied_to (message_id, replied_to_id)
|
INSERT INTO edit_timestamps (message_id, edit_timestamp)
|
||||||
SELECT message_id, replied_to_id
|
SELECT message_id, edit_timestamp
|
||||||
FROM messages
|
FROM messages
|
||||||
WHERE replied_to_id IS NOT NULL
|
WHERE edit_timestamp IS NOT NULL
|
||||||
""");
|
""");
|
||||||
|
|
||||||
conn.Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
|
await conn.ExecuteAsync("""
|
||||||
conn.Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
|
INSERT INTO replied_to (message_id, replied_to_id)
|
||||||
|
SELECT message_id, replied_to_id
|
||||||
|
FROM messages
|
||||||
|
WHERE replied_to_id IS NOT NULL
|
||||||
|
""");
|
||||||
|
|
||||||
|
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN replied_to_id");
|
||||||
|
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN edit_timestamp");
|
||||||
|
|
||||||
perf.Step("Upgrade to version 3");
|
perf.Step("Upgrade to version 3");
|
||||||
|
|
||||||
await reporter.MainWork("Vacuuming the database...", 1, 1);
|
await reporter.MainWork("Vacuuming the database...", 1, 1);
|
||||||
conn.Execute("VACUUM");
|
await conn.ExecuteAsync("VACUUM");
|
||||||
perf.Step("Vacuum");
|
perf.Step("Vacuum");
|
||||||
|
|
||||||
await reporter.NextVersion();
|
await reporter.NextVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbVersion <= 3) {
|
if (dbVersion <= 3) {
|
||||||
conn.Execute("""
|
await conn.ExecuteAsync("""
|
||||||
CREATE TABLE downloads (
|
CREATE TABLE downloads (
|
||||||
url TEXT NOT NULL PRIMARY KEY,
|
url TEXT NOT NULL PRIMARY KEY,
|
||||||
status INTEGER NOT NULL,
|
status INTEGER NOT NULL,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
blob BLOB
|
blob BLOB
|
||||||
)
|
)
|
||||||
""");
|
""");
|
||||||
|
|
||||||
perf.Step("Upgrade to version 4");
|
perf.Step("Upgrade to version 4");
|
||||||
await reporter.NextVersion();
|
await reporter.NextVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbVersion <= 4) {
|
if (dbVersion <= 4) {
|
||||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||||
conn.Execute("ALTER TABLE attachments ADD width INTEGER");
|
await conn.ExecuteAsync("ALTER TABLE attachments ADD width INTEGER");
|
||||||
conn.Execute("ALTER TABLE attachments ADD height INTEGER");
|
await conn.ExecuteAsync("ALTER TABLE attachments ADD height INTEGER");
|
||||||
|
|
||||||
perf.Step("Upgrade to version 5");
|
perf.Step("Upgrade to version 5");
|
||||||
await reporter.NextVersion();
|
await reporter.NextVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbVersion <= 5) {
|
if (dbVersion <= 5) {
|
||||||
await reporter.MainWork("Applying schema changes...", 0, 3);
|
await reporter.MainWork("Applying schema changes...", 0, 3);
|
||||||
conn.Execute("ALTER TABLE attachments ADD download_url TEXT");
|
await conn.ExecuteAsync("ALTER TABLE attachments ADD download_url TEXT");
|
||||||
conn.Execute("ALTER TABLE downloads ADD download_url TEXT");
|
await conn.ExecuteAsync("ALTER TABLE downloads ADD download_url TEXT");
|
||||||
|
|
||||||
await reporter.MainWork("Updating attachments...", 1, 3);
|
await reporter.MainWork("Updating attachments...", 1, 3);
|
||||||
await NormalizeAttachmentUrls(reporter);
|
await NormalizeAttachmentUrls(reporter);
|
||||||
|
|
||||||
await reporter.MainWork("Updating downloads...", 2, 3);
|
await reporter.MainWork("Updating downloads...", 2, 3);
|
||||||
await NormalizeDownloadUrls(reporter);
|
await NormalizeDownloadUrls(reporter);
|
||||||
|
|
||||||
await reporter.MainWork("Applying schema changes...", 3, 3);
|
await reporter.MainWork("Applying schema changes...", 3, 3);
|
||||||
conn.Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
|
await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
|
||||||
conn.Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
|
await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
|
||||||
|
|
||||||
perf.Step("Upgrade to version 6");
|
perf.Step("Upgrade to version 6");
|
||||||
await reporter.NextVersion();
|
await reporter.NextVersion();
|
||||||
}
|
}
|
||||||
|
@ -19,11 +19,6 @@ static class SqliteExtensions {
|
|||||||
return cmd;
|
return cmd;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void Execute(this ISqliteConnection conn, string sql) {
|
|
||||||
using var cmd = conn.Command(sql);
|
|
||||||
cmd.ExecuteNonQuery();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Task<int> ExecuteAsync(this ISqliteConnection conn, [LanguageInjection("sql")] string sql, CancellationToken cancellationToken = default) {
|
public static async Task<int> ExecuteAsync(this ISqliteConnection conn, [LanguageInjection("sql")] string sql, CancellationToken cancellationToken = default) {
|
||||||
await using var cmd = conn.Command(sql);
|
await using var cmd = conn.Command(sql);
|
||||||
return await cmd.ExecuteNonQueryAsync(cancellationToken);
|
return await cmd.ExecuteNonQueryAsync(cancellationToken);
|
||||||
@ -36,11 +31,6 @@ static class SqliteExtensions {
|
|||||||
return reader.Read() ? readFunction(reader) : readFunction(null);
|
return reader.Read() ? readFunction(reader) : readFunction(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static object? SelectScalar(this ISqliteConnection conn, string sql) {
|
|
||||||
using var cmd = conn.Command(sql);
|
|
||||||
return cmd.ExecuteScalar();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
|
public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
|
||||||
string columnNames = string.Join(',', columns.Select(static c => c.Name));
|
string columnNames = string.Join(',', columns.Select(static c => c.Name));
|
||||||
string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name));
|
string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name));
|
||||||
|
Loading…
Reference in New Issue
Block a user