mirror of
https://github.com/chylex/Discord-History-Tracker.git
synced 2024-11-25 14:42:44 +01:00
Compare commits
1 Commits
e0f359c15b
...
11f7d4a49f
Author | SHA1 | Date | |
---|---|---|---|
11f7d4a49f |
@ -34,7 +34,7 @@ sealed class StatusBarModel : BaseModel, IDisposable {
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
state.Server.StatusChanged -= OnServerStatusChanged;
|
||||
state.Server.StatusChanged += OnServerStatusChanged;
|
||||
}
|
||||
|
||||
private void OnServerStatusChanged(object? sender, ServerManager.Status e) {
|
||||
|
@ -69,12 +69,12 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
public bool HasFailedDownloads => statisticsFailed.Items > 0;
|
||||
|
||||
private readonly State state;
|
||||
private readonly ThrottledTask<int> enqueueDownloadItemsTask;
|
||||
private readonly ThrottledTask enqueueDownloadItemsTask;
|
||||
private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
|
||||
|
||||
private IDisposable? finishedItemsSubscription;
|
||||
private int doneItemsCount;
|
||||
private int totalEnqueuedItemCount;
|
||||
private int initialFinishedCount;
|
||||
private int? totalItemsToDownloadCount;
|
||||
|
||||
public AttachmentsPageModel() : this(State.Dummy) {}
|
||||
@ -84,7 +84,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
|
||||
FilterModel = new AttachmentFilterPanelModel(state);
|
||||
|
||||
enqueueDownloadItemsTask = new ThrottledTask<int>(OnItemsEnqueued, TaskScheduler.FromCurrentSynchronizationContext());
|
||||
enqueueDownloadItemsTask = new ThrottledTask(RecomputeDownloadStatistics, TaskScheduler.FromCurrentSynchronizationContext());
|
||||
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
|
||||
RecomputeDownloadStatistics();
|
||||
|
||||
@ -93,7 +93,6 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
|
||||
public void Dispose() {
|
||||
state.Db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
|
||||
enqueueDownloadItemsTask.Dispose();
|
||||
downloadStatisticsTask.Dispose();
|
||||
finishedItemsSubscription?.Dispose();
|
||||
FilterModel.Dispose();
|
||||
@ -114,7 +113,7 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
}
|
||||
|
||||
private async Task EnqueueDownloadItems() {
|
||||
OnItemsEnqueued(await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter()));
|
||||
await state.Db.Downloads.EnqueueDownloadItems(CreateAttachmentFilter());
|
||||
}
|
||||
|
||||
private void EnqueueDownloadItemsLater() {
|
||||
@ -122,19 +121,49 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
enqueueDownloadItemsTask.Post(cancellationToken => state.Db.Downloads.EnqueueDownloadItems(filter, cancellationToken));
|
||||
}
|
||||
|
||||
private void OnItemsEnqueued(int itemCount) {
|
||||
totalEnqueuedItemCount += itemCount;
|
||||
totalItemsToDownloadCount = totalEnqueuedItemCount;
|
||||
UpdateDownloadMessage();
|
||||
RecomputeDownloadStatistics();
|
||||
}
|
||||
|
||||
private AttachmentFilter CreateAttachmentFilter() {
|
||||
var filter = FilterModel.CreateFilter();
|
||||
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
|
||||
return filter;
|
||||
}
|
||||
|
||||
private void RecomputeDownloadStatistics() {
|
||||
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
|
||||
}
|
||||
|
||||
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
|
||||
var hadFailedDownloads = HasFailedDownloads;
|
||||
|
||||
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
|
||||
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
|
||||
|
||||
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
|
||||
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
|
||||
|
||||
statisticsFailed.Items = statusStatistics.FailedCount;
|
||||
statisticsFailed.Size = statusStatistics.FailedSize;
|
||||
|
||||
statisticsSkipped.Items = statusStatistics.SkippedCount;
|
||||
statisticsSkipped.Size = statusStatistics.SkippedSize;
|
||||
|
||||
OnPropertyChanged(nameof(StatisticsRows));
|
||||
|
||||
if (hadFailedDownloads != HasFailedDownloads) {
|
||||
OnPropertyChanged(nameof(HasFailedDownloads));
|
||||
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
|
||||
}
|
||||
|
||||
totalItemsToDownloadCount = statisticsEnqueued.Items + statisticsDownloaded.Items + statisticsFailed.Items - initialFinishedCount;
|
||||
UpdateDownloadMessage();
|
||||
}
|
||||
|
||||
private void UpdateDownloadMessage() {
|
||||
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
|
||||
|
||||
OnPropertyChanged(nameof(DownloadMessage));
|
||||
OnPropertyChanged(nameof(DownloadProgress));
|
||||
}
|
||||
|
||||
public async Task OnClickToggleDownload() {
|
||||
IsToggleDownloadButtonEnabled = false;
|
||||
|
||||
@ -149,13 +178,14 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
await state.Db.Downloads.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
|
||||
|
||||
doneItemsCount = 0;
|
||||
totalEnqueuedItemCount = 0;
|
||||
initialFinishedCount = 0;
|
||||
totalItemsToDownloadCount = null;
|
||||
UpdateDownloadMessage();
|
||||
}
|
||||
else {
|
||||
var finishedItems = await state.Downloader.Start();
|
||||
|
||||
initialFinishedCount = statisticsDownloaded.Items + statisticsFailed.Items;
|
||||
finishedItemsSubscription = finishedItems.Select(static _ => true)
|
||||
.Buffer(TimeSpan.FromMilliseconds(100))
|
||||
.Select(static items => items.Count)
|
||||
@ -201,42 +231,6 @@ sealed class AttachmentsPageModel : BaseModel, IDisposable {
|
||||
}
|
||||
}
|
||||
|
||||
private void RecomputeDownloadStatistics() {
|
||||
downloadStatisticsTask.Post(state.Db.Downloads.GetStatistics);
|
||||
}
|
||||
|
||||
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
|
||||
var hadFailedDownloads = HasFailedDownloads;
|
||||
|
||||
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
|
||||
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
|
||||
|
||||
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
|
||||
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
|
||||
|
||||
statisticsFailed.Items = statusStatistics.FailedCount;
|
||||
statisticsFailed.Size = statusStatistics.FailedSize;
|
||||
|
||||
statisticsSkipped.Items = statusStatistics.SkippedCount;
|
||||
statisticsSkipped.Size = statusStatistics.SkippedSize;
|
||||
|
||||
OnPropertyChanged(nameof(StatisticsRows));
|
||||
|
||||
if (hadFailedDownloads != HasFailedDownloads) {
|
||||
OnPropertyChanged(nameof(HasFailedDownloads));
|
||||
OnPropertyChanged(nameof(IsRetryFailedOnDownloadsButtonEnabled));
|
||||
}
|
||||
|
||||
UpdateDownloadMessage();
|
||||
}
|
||||
|
||||
private void UpdateDownloadMessage() {
|
||||
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : "";
|
||||
|
||||
OnPropertyChanged(nameof(DownloadMessage));
|
||||
OnPropertyChanged(nameof(DownloadProgress));
|
||||
}
|
||||
|
||||
public sealed class StatisticsRow {
|
||||
public string State { get; }
|
||||
public int Items { get; set; }
|
||||
|
@ -22,7 +22,7 @@ public interface IDownloadRepository {
|
||||
|
||||
Task<DownloadedAttachment?> GetDownloadedAttachment(string normalizedUrl);
|
||||
|
||||
Task<int> EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
|
||||
Task EnqueueDownloadItems(AttachmentFilter? filter = null, CancellationToken cancellationToken = default);
|
||||
|
||||
IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken = default);
|
||||
|
||||
@ -53,8 +53,8 @@ public interface IDownloadRepository {
|
||||
return Task.FromResult<DownloadedAttachment?>(null);
|
||||
}
|
||||
|
||||
public Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||
return Task.FromResult(0);
|
||||
public Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, CancellationToken cancellationToken) {
|
||||
|
@ -164,7 +164,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<int> EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||
public async Task EnqueueDownloadItems(AttachmentFilter? filter, CancellationToken cancellationToken) {
|
||||
using var conn = pool.Take();
|
||||
|
||||
await using var cmd = conn.Command(
|
||||
@ -178,7 +178,7 @@ sealed class SqliteDownloadRepository : IDownloadRepository {
|
||||
);
|
||||
|
||||
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
|
||||
return await cmd.ExecuteNonQueryAsync(cancellationToken);
|
||||
await cmd.ExecuteNonQueryAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<DownloadItem> PullEnqueuedDownloadItems(int count, [EnumeratorCancellation] CancellationToken cancellationToken) {
|
||||
|
@ -21,14 +21,14 @@ sealed class Schema {
|
||||
}
|
||||
|
||||
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
|
||||
await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
|
||||
conn.Execute(@"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
|
||||
|
||||
var dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
|
||||
var dbVersionStr = conn.SelectScalar("SELECT value FROM metadata WHERE key = 'version'");
|
||||
if (dbVersionStr == null) {
|
||||
await InitializeSchemas();
|
||||
InitializeSchemas();
|
||||
}
|
||||
else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
|
||||
throw new InvalidDatabaseVersionException(dbVersionStr);
|
||||
else if (!int.TryParse(dbVersionStr.ToString(), out int dbVersion) || dbVersion < 1) {
|
||||
throw new InvalidDatabaseVersionException(dbVersionStr.ToString() ?? "<null>");
|
||||
}
|
||||
else if (dbVersion > Version) {
|
||||
throw new DatabaseTooNewException(dbVersion);
|
||||
@ -45,123 +45,123 @@ sealed class Schema {
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task InitializeSchemas() {
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
avatar_url TEXT,
|
||||
discriminator TEXT
|
||||
)
|
||||
""");
|
||||
private void InitializeSchemas() {
|
||||
conn.Execute("""
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
avatar_url TEXT,
|
||||
discriminator TEXT
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE servers (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE servers (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE channels (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
server INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER,
|
||||
position INTEGER,
|
||||
topic TEXT,
|
||||
nsfw INTEGER
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE channels (
|
||||
id INTEGER PRIMARY KEY NOT NULL,
|
||||
server INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
parent_id INTEGER,
|
||||
position INTEGER,
|
||||
topic TEXT,
|
||||
nsfw INTEGER
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE messages (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
sender_id INTEGER NOT NULL,
|
||||
channel_id INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
timestamp INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE messages (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
sender_id INTEGER NOT NULL,
|
||||
channel_id INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
timestamp INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE attachments (
|
||||
message_id INTEGER NOT NULL,
|
||||
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT,
|
||||
normalized_url TEXT NOT NULL,
|
||||
download_url TEXT,
|
||||
size INTEGER NOT NULL,
|
||||
width INTEGER,
|
||||
height INTEGER
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE attachments (
|
||||
message_id INTEGER NOT NULL,
|
||||
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT,
|
||||
normalized_url TEXT NOT NULL,
|
||||
download_url TEXT,
|
||||
size INTEGER NOT NULL,
|
||||
width INTEGER,
|
||||
height INTEGER
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE embeds (
|
||||
message_id INTEGER NOT NULL,
|
||||
json TEXT NOT NULL
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE embeds (
|
||||
message_id INTEGER NOT NULL,
|
||||
json TEXT NOT NULL
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE downloads (
|
||||
normalized_url TEXT NOT NULL PRIMARY KEY,
|
||||
download_url TEXT,
|
||||
status INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
blob BLOB
|
||||
)
|
||||
""");
|
||||
conn.Execute("""
|
||||
CREATE TABLE downloads (
|
||||
normalized_url TEXT NOT NULL PRIMARY KEY,
|
||||
download_url TEXT,
|
||||
status INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
blob BLOB
|
||||
)
|
||||
""");
|
||||
|
||||
conn.Execute("""
|
||||
CREATE TABLE reactions (
|
||||
message_id INTEGER NOT NULL,
|
||||
emoji_id INTEGER,
|
||||
emoji_name TEXT,
|
||||
emoji_flags INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE reactions (
|
||||
message_id INTEGER NOT NULL,
|
||||
emoji_id INTEGER,
|
||||
emoji_name TEXT,
|
||||
emoji_flags INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
CreateMessageEditTimestampTable();
|
||||
CreateMessageRepliedToTable();
|
||||
|
||||
await CreateMessageEditTimestampTable();
|
||||
await CreateMessageRepliedToTable();
|
||||
conn.Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
|
||||
conn.Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
|
||||
conn.Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
|
||||
|
||||
await conn.ExecuteAsync("CREATE INDEX attachments_message_ix ON attachments(message_id)");
|
||||
await conn.ExecuteAsync("CREATE INDEX embeds_message_ix ON embeds(message_id)");
|
||||
await conn.ExecuteAsync("CREATE INDEX reactions_message_ix ON reactions(message_id)");
|
||||
|
||||
await conn.ExecuteAsync("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
|
||||
conn.Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
|
||||
}
|
||||
|
||||
private async Task CreateMessageEditTimestampTable() {
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE edit_timestamps (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
edit_timestamp INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
private void CreateMessageEditTimestampTable() {
|
||||
conn.Execute("""
|
||||
CREATE TABLE edit_timestamps (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
edit_timestamp INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
}
|
||||
|
||||
private async Task CreateMessageRepliedToTable() {
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE replied_to (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
replied_to_id INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
private void CreateMessageRepliedToTable() {
|
||||
conn.Execute("""
|
||||
CREATE TABLE replied_to (
|
||||
message_id INTEGER PRIMARY KEY NOT NULL,
|
||||
replied_to_id INTEGER NOT NULL
|
||||
)
|
||||
""");
|
||||
}
|
||||
|
||||
private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||
await reporter.SubWork("Preparing attachments...", 0, 0);
|
||||
|
||||
|
||||
var normalizedUrls = new Dictionary<long, string>();
|
||||
|
||||
await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
|
||||
await using var reader = await selectCmd.ExecuteReaderAsync();
|
||||
|
||||
|
||||
while (reader.Read()) {
|
||||
var attachmentId = reader.GetInt64(0);
|
||||
var originalUrl = reader.GetString(1);
|
||||
@ -177,7 +177,7 @@ sealed class Schema {
|
||||
await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
|
||||
updateCmd.Add(":attachment_id", SqliteType.Integer);
|
||||
updateCmd.Add(":normalized_url", SqliteType.Text);
|
||||
|
||||
|
||||
foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
|
||||
if (++processedUrls % 1000 == 0) {
|
||||
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
||||
@ -188,15 +188,15 @@ sealed class Schema {
|
||||
updateCmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
||||
|
||||
|
||||
await tx.CommitAsync();
|
||||
}
|
||||
|
||||
private async Task NormalizeDownloadUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||
await reporter.SubWork("Preparing downloads...", 0, 0);
|
||||
|
||||
|
||||
var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
|
||||
var duplicateUrlsToDelete = new HashSet<string>();
|
||||
|
||||
@ -213,10 +213,10 @@ sealed class Schema {
|
||||
}
|
||||
}
|
||||
|
||||
await conn.ExecuteAsync("PRAGMA cache_size = -20000");
|
||||
conn.Execute("PRAGMA cache_size = -20000");
|
||||
|
||||
DbTransaction tx;
|
||||
|
||||
|
||||
await using (tx = await conn.BeginTransactionAsync()) {
|
||||
await reporter.SubWork("Deleting duplicates...", 0, 0);
|
||||
|
||||
@ -226,7 +226,7 @@ sealed class Schema {
|
||||
deleteCmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
await tx.CommitAsync();
|
||||
}
|
||||
|
||||
@ -234,20 +234,20 @@ sealed class Schema {
|
||||
int processedUrls = -1;
|
||||
|
||||
tx = await conn.BeginTransactionAsync();
|
||||
|
||||
|
||||
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
|
||||
updateCmd.Add(":normalized_url", SqliteType.Text);
|
||||
updateCmd.Add(":download_url", SqliteType.Text);
|
||||
|
||||
|
||||
foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
|
||||
if (++processedUrls % 100 == 0) {
|
||||
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
|
||||
|
||||
|
||||
// Not proper way of dealing with transactions, but it avoids a long commit at the end.
|
||||
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
|
||||
await tx.CommitAsync();
|
||||
await tx.DisposeAsync();
|
||||
|
||||
|
||||
tx = await conn.BeginTransactionAsync();
|
||||
updateCmd.Transaction = (SqliteTransaction) tx;
|
||||
}
|
||||
@ -257,98 +257,98 @@ sealed class Schema {
|
||||
updateCmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
|
||||
|
||||
|
||||
await tx.CommitAsync();
|
||||
await tx.DisposeAsync();
|
||||
|
||||
await conn.ExecuteAsync("PRAGMA cache_size = -2000");
|
||||
|
||||
conn.Execute("PRAGMA cache_size = -2000");
|
||||
}
|
||||
|
||||
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
|
||||
var perf = Log.Start("from version " + dbVersion);
|
||||
|
||||
await conn.ExecuteAsync("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
|
||||
conn.Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
|
||||
|
||||
if (dbVersion <= 1) {
|
||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||
await conn.ExecuteAsync("ALTER TABLE channels ADD parent_id INTEGER");
|
||||
|
||||
conn.Execute("ALTER TABLE channels ADD parent_id INTEGER");
|
||||
|
||||
perf.Step("Upgrade to version 2");
|
||||
await reporter.NextVersion();
|
||||
}
|
||||
|
||||
if (dbVersion <= 2) {
|
||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||
|
||||
CreateMessageEditTimestampTable();
|
||||
CreateMessageRepliedToTable();
|
||||
|
||||
await CreateMessageEditTimestampTable();
|
||||
await CreateMessageRepliedToTable();
|
||||
conn.Execute("""
|
||||
INSERT INTO edit_timestamps (message_id, edit_timestamp)
|
||||
SELECT message_id, edit_timestamp
|
||||
FROM messages
|
||||
WHERE edit_timestamp IS NOT NULL
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
INSERT INTO edit_timestamps (message_id, edit_timestamp)
|
||||
SELECT message_id, edit_timestamp
|
||||
FROM messages
|
||||
WHERE edit_timestamp IS NOT NULL
|
||||
""");
|
||||
conn.Execute("""
|
||||
INSERT INTO replied_to (message_id, replied_to_id)
|
||||
SELECT message_id, replied_to_id
|
||||
FROM messages
|
||||
WHERE replied_to_id IS NOT NULL
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("""
|
||||
INSERT INTO replied_to (message_id, replied_to_id)
|
||||
SELECT message_id, replied_to_id
|
||||
FROM messages
|
||||
WHERE replied_to_id IS NOT NULL
|
||||
""");
|
||||
|
||||
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN replied_to_id");
|
||||
await conn.ExecuteAsync("ALTER TABLE messages DROP COLUMN edit_timestamp");
|
||||
conn.Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
|
||||
conn.Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
|
||||
|
||||
perf.Step("Upgrade to version 3");
|
||||
|
||||
|
||||
await reporter.MainWork("Vacuuming the database...", 1, 1);
|
||||
await conn.ExecuteAsync("VACUUM");
|
||||
conn.Execute("VACUUM");
|
||||
perf.Step("Vacuum");
|
||||
|
||||
|
||||
await reporter.NextVersion();
|
||||
}
|
||||
|
||||
if (dbVersion <= 3) {
|
||||
await conn.ExecuteAsync("""
|
||||
CREATE TABLE downloads (
|
||||
url TEXT NOT NULL PRIMARY KEY,
|
||||
status INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
blob BLOB
|
||||
)
|
||||
""");
|
||||
|
||||
conn.Execute("""
|
||||
CREATE TABLE downloads (
|
||||
url TEXT NOT NULL PRIMARY KEY,
|
||||
status INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
blob BLOB
|
||||
)
|
||||
""");
|
||||
|
||||
perf.Step("Upgrade to version 4");
|
||||
await reporter.NextVersion();
|
||||
}
|
||||
|
||||
if (dbVersion <= 4) {
|
||||
await reporter.MainWork("Applying schema changes...", 0, 1);
|
||||
await conn.ExecuteAsync("ALTER TABLE attachments ADD width INTEGER");
|
||||
await conn.ExecuteAsync("ALTER TABLE attachments ADD height INTEGER");
|
||||
|
||||
conn.Execute("ALTER TABLE attachments ADD width INTEGER");
|
||||
conn.Execute("ALTER TABLE attachments ADD height INTEGER");
|
||||
|
||||
perf.Step("Upgrade to version 5");
|
||||
await reporter.NextVersion();
|
||||
}
|
||||
|
||||
if (dbVersion <= 5) {
|
||||
await reporter.MainWork("Applying schema changes...", 0, 3);
|
||||
await conn.ExecuteAsync("ALTER TABLE attachments ADD download_url TEXT");
|
||||
await conn.ExecuteAsync("ALTER TABLE downloads ADD download_url TEXT");
|
||||
|
||||
conn.Execute("ALTER TABLE attachments ADD download_url TEXT");
|
||||
conn.Execute("ALTER TABLE downloads ADD download_url TEXT");
|
||||
|
||||
await reporter.MainWork("Updating attachments...", 1, 3);
|
||||
await NormalizeAttachmentUrls(reporter);
|
||||
|
||||
|
||||
await reporter.MainWork("Updating downloads...", 2, 3);
|
||||
await NormalizeDownloadUrls(reporter);
|
||||
|
||||
|
||||
await reporter.MainWork("Applying schema changes...", 3, 3);
|
||||
await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
|
||||
await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
|
||||
|
||||
conn.Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
|
||||
conn.Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
|
||||
|
||||
perf.Step("Upgrade to version 6");
|
||||
await reporter.NextVersion();
|
||||
}
|
||||
|
@ -19,6 +19,11 @@ static class SqliteExtensions {
|
||||
return cmd;
|
||||
}
|
||||
|
||||
public static void Execute(this ISqliteConnection conn, string sql) {
|
||||
using var cmd = conn.Command(sql);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public static async Task<int> ExecuteAsync(this ISqliteConnection conn, [LanguageInjection("sql")] string sql, CancellationToken cancellationToken = default) {
|
||||
await using var cmd = conn.Command(sql);
|
||||
return await cmd.ExecuteNonQueryAsync(cancellationToken);
|
||||
@ -31,6 +36,11 @@ static class SqliteExtensions {
|
||||
return reader.Read() ? readFunction(reader) : readFunction(null);
|
||||
}
|
||||
|
||||
public static object? SelectScalar(this ISqliteConnection conn, string sql) {
|
||||
using var cmd = conn.Command(sql);
|
||||
return cmd.ExecuteScalar();
|
||||
}
|
||||
|
||||
public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
|
||||
string columnNames = string.Join(',', columns.Select(static c => c.Name));
|
||||
string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name));
|
||||
|
Loading…
Reference in New Issue
Block a user