1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-10-19 05:42:50 +02:00

Compare commits

..

No commits in common. "1024a58a47712d53c01914166e341fc1226882b6" and "069ab97196be42305620f74a8fd842e690c2a68f" have entirely different histories.

14 changed files with 219 additions and 428 deletions

View File

@ -41,11 +41,11 @@ static class DatabaseGui {
}); });
} }
public static async Task<IDatabaseFile?> TryOpenOrCreateDatabaseFromPath(string path, Window window, ISchemaUpgradeCallbacks schemaUpgradeCallbacks) { public static async Task<IDatabaseFile?> TryOpenOrCreateDatabaseFromPath(string path, Window window, Func<Task<bool>> checkCanUpgradeDatabase) {
IDatabaseFile? file = null; IDatabaseFile? file = null;
try { try {
file = await SqliteDatabaseFile.OpenOrCreate(path, schemaUpgradeCallbacks); file = await SqliteDatabaseFile.OpenOrCreate(path, checkCanUpgradeDatabase);
} catch (InvalidDatabaseVersionException ex) { } catch (InvalidDatabaseVersionException ex) {
await Dialog.ShowOk(window, "Database Error", "Database '" + Path.GetFileName(path) + "' appears to be corrupted (invalid version: " + ex.Version + ")."); await Dialog.ShowOk(window, "Database Error", "Database '" + Path.GetFileName(path) + "' appears to be corrupted (invalid version: " + ex.Version + ").");
} catch (DatabaseTooNewException ex) { } catch (DatabaseTooNewException ex) {

View File

@ -4,5 +4,4 @@ namespace DHT.Desktop.Dialogs.Progress;
interface IProgressCallback { interface IProgressCallback {
Task Update(string message, int finishedItems, int totalItems); Task Update(string message, int finishedItems, int totalItems);
Task Hide();
} }

View File

@ -32,18 +32,12 @@
</Style> </Style>
</Window.Styles> </Window.Styles>
<ItemsRepeater ItemsSource="{Binding Items}" Margin="0 10"> <StackPanel Margin="20">
<ItemsRepeater.ItemTemplate> <DockPanel>
<DataTemplate> <TextBlock DockPanel.Dock="Right" Text="{Binding Items}" Classes="items" />
<StackPanel Margin="20 10" IsHitTestVisible="{Binding IsVisible}" Opacity="{Binding Opacity}"> <TextBlock DockPanel.Dock="Left" Text="{Binding Message}" />
<DockPanel> </DockPanel>
<TextBlock DockPanel.Dock="Right" Text="{Binding Items}" Classes="items" /> <ProgressBar Value="{Binding Progress}" />
<TextBlock DockPanel.Dock="Left" Text="{Binding Message}" /> </StackPanel>
</DockPanel>
<ProgressBar Value="{Binding Progress}" />
</StackPanel>
</DataTemplate>
</ItemsRepeater.ItemTemplate>
</ItemsRepeater>
</Window> </Window>

View File

@ -8,7 +8,6 @@ namespace DHT.Desktop.Dialogs.Progress;
[SuppressMessage("ReSharper", "MemberCanBeInternal")] [SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed partial class ProgressDialog : Window { public sealed partial class ProgressDialog : Window {
private bool isFinished = false; private bool isFinished = false;
private Task progressTask = Task.CompletedTask;
public ProgressDialog() { public ProgressDialog() {
InitializeComponent(); InitializeComponent();
@ -16,8 +15,7 @@ public sealed partial class ProgressDialog : Window {
public void OnOpened(object? sender, EventArgs e) { public void OnOpened(object? sender, EventArgs e) {
if (DataContext is ProgressDialogModel model) { if (DataContext is ProgressDialogModel model) {
progressTask = Task.Run(model.StartTask); Task.Run(model.StartTask).ContinueWith(OnFinished, TaskScheduler.FromCurrentSynchronizationContext());
progressTask.ContinueWith(OnFinished, TaskScheduler.FromCurrentSynchronizationContext());
} }
} }
@ -29,9 +27,4 @@ public sealed partial class ProgressDialog : Window {
isFinished = true; isFinished = true;
Close(); Close();
} }
public async Task ShowProgressDialog(Window owner) {
await ShowDialog(owner);
await progressTask;
}
} }

View File

@ -1,6 +1,4 @@
using System; using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Avalonia.Threading; using Avalonia.Threading;
using DHT.Desktop.Common; using DHT.Desktop.Common;
@ -11,43 +9,57 @@ namespace DHT.Desktop.Dialogs.Progress;
sealed class ProgressDialogModel : BaseModel { sealed class ProgressDialogModel : BaseModel {
public string Title { get; init; } = ""; public string Title { get; init; } = "";
public IReadOnlyList<ProgressItem> Items { get; } = Array.Empty<ProgressItem>(); private string message = "";
public string Message {
get => message;
private set => Change(ref message, value);
}
private string items = "";
public string Items {
get => items;
private set => Change(ref items, value);
}
private int progress = 0;
public int Progress {
get => progress;
private set => Change(ref progress, value);
}
private readonly TaskRunner? task; private readonly TaskRunner? task;
[Obsolete("Designer")] [Obsolete("Designer")]
public ProgressDialogModel() {} public ProgressDialogModel() {}
public ProgressDialogModel(TaskRunner task, int progressItems = 1) { public ProgressDialogModel(TaskRunner task) {
this.Items = Enumerable.Range(0, progressItems).Select(static _ => new ProgressItem()).ToArray();
this.task = task; this.task = task;
} }
internal async Task StartTask() { internal async Task StartTask() {
if (task != null) { if (task != null) {
await task(Items.Select(static item => new Callback(item)).ToArray()); await task(new Callback(this));
} }
} }
public delegate Task TaskRunner(IReadOnlyList<IProgressCallback> callbacks); public delegate Task TaskRunner(IProgressCallback callback);
private sealed class Callback : IProgressCallback { private sealed class Callback : IProgressCallback {
private readonly ProgressItem item; private readonly ProgressDialogModel model;
public Callback(ProgressItem item) { public Callback(ProgressDialogModel model) {
this.item = item; this.model = model;
} }
public async Task Update(string message, int finishedItems, int totalItems) { async Task IProgressCallback.Update(string message, int finishedItems, int totalItems) {
await Dispatcher.UIThread.InvokeAsync(() => { await Dispatcher.UIThread.InvokeAsync(() => {
item.Message = message; model.Message = message;
item.Items = totalItems == 0 ? string.Empty : finishedItems.Format() + " / " + totalItems.Format(); model.Items = finishedItems.Format() + " / " + totalItems.Format();
item.Progress = totalItems == 0 ? 0 : 100 * finishedItems / totalItems; model.Progress = 100 * finishedItems / totalItems;
}); });
} }
public Task Hide() {
return Update(string.Empty, 0, 0);
}
} }
} }

View File

@ -1,41 +0,0 @@
using DHT.Utils.Models;
namespace DHT.Desktop.Dialogs.Progress;
sealed class ProgressItem : BaseModel {
private bool isVisible = false;
public bool IsVisible {
get => isVisible;
private set {
Change(ref isVisible, value);
OnPropertyChanged(nameof(Opacity));
}
}
public double Opacity => IsVisible ? 1.0 : 0.0;
private string message = "";
public string Message {
get => message;
set {
Change(ref message, value);
IsVisible = !string.IsNullOrEmpty(value);
}
}
private string items = "";
public string Items {
get => items;
set => Change(ref items, value);
}
private int progress = 0;
public int Progress {
get => progress;
set => Change(ref progress, value);
}
}

View File

@ -17,7 +17,6 @@ using DHT.Desktop.Dialogs.TextBox;
using DHT.Server.Data; using DHT.Server.Data;
using DHT.Server.Database; using DHT.Server.Database;
using DHT.Server.Database.Import; using DHT.Server.Database.Import;
using DHT.Server.Database.Sqlite;
using DHT.Utils.Logging; using DHT.Utils.Logging;
using DHT.Utils.Models; using DHT.Utils.Models;
@ -78,20 +77,30 @@ sealed class DatabasePageModel : BaseModel {
} }
ProgressDialog progressDialog = new ProgressDialog(); ProgressDialog progressDialog = new ProgressDialog();
progressDialog.DataContext = new ProgressDialogModel(async callbacks => await MergeWithDatabaseFromPaths(Db, paths, progressDialog, callbacks[0])) { progressDialog.DataContext = new ProgressDialogModel(async callback => await MergeWithDatabaseFromPaths(Db, paths, progressDialog, callback)) {
Title = "Database Merge" Title = "Database Merge"
}; };
await progressDialog.ShowProgressDialog(window); await progressDialog.ShowDialog(window);
} }
private static async Task MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) { private static async Task MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
int total = paths.Length; int total = paths.Length;
DialogResult.YesNo? upgradeResult = null;
async Task<bool> CheckCanUpgradeDatabase() {
upgradeResult ??= total > 1
? await DatabaseGui.ShowCanUpgradeMultipleDatabaseDialog(dialog)
: await DatabaseGui.ShowCanUpgradeDatabaseDialog(dialog);
return DialogResult.YesNo.Yes == upgradeResult;
}
await PerformImport(target, paths, dialog, callback, "Database Merge", "Database Error", "database file", async path => { await PerformImport(target, paths, dialog, callback, "Database Merge", "Database Error", "database file", async path => {
SynchronizationContext? prevSyncContext = SynchronizationContext.Current; SynchronizationContext? prevSyncContext = SynchronizationContext.Current;
SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext()); SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext());
IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, new SchemaUpgradeCallbacks(dialog, total)); IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, CheckCanUpgradeDatabase);
SynchronizationContext.SetSynchronizationContext(prevSyncContext); SynchronizationContext.SetSynchronizationContext(prevSyncContext);
if (db == null) { if (db == null) {
@ -107,41 +116,6 @@ sealed class DatabasePageModel : BaseModel {
}); });
} }
private sealed class SchemaUpgradeCallbacks : ISchemaUpgradeCallbacks {
private readonly ProgressDialog dialog;
private readonly int total;
private bool? decision;
public SchemaUpgradeCallbacks(ProgressDialog dialog, int total) {
this.total = total;
this.dialog = dialog;
}
public async Task<bool> CanUpgrade() {
return decision ??= (total > 1
? await DatabaseGui.ShowCanUpgradeMultipleDatabaseDialog(dialog)
: await DatabaseGui.ShowCanUpgradeDatabaseDialog(dialog)) == DialogResult.YesNo.Yes;
}
public Task Start(int versionSteps, Func<ISchemaUpgradeCallbacks.IProgressReporter, Task> doUpgrade) {
return doUpgrade(new NullReporter());
}
private sealed class NullReporter : ISchemaUpgradeCallbacks.IProgressReporter {
public Task NextVersion() {
return Task.CompletedTask;
}
public Task MainWork(string message, int finishedItems, int totalItems) {
return Task.CompletedTask;
}
public Task SubWork(string message, int finishedItems, int totalItems) {
return Task.CompletedTask;
}
}
}
public async void ImportLegacyArchive() { public async void ImportLegacyArchive() {
var paths = await window.StorageProvider.OpenFiles(new FilePickerOpenOptions { var paths = await window.StorageProvider.OpenFiles(new FilePickerOpenOptions {
Title = "Open Legacy DHT Archive", Title = "Open Legacy DHT Archive",
@ -154,11 +128,11 @@ sealed class DatabasePageModel : BaseModel {
} }
ProgressDialog progressDialog = new ProgressDialog(); ProgressDialog progressDialog = new ProgressDialog();
progressDialog.DataContext = new ProgressDialogModel(async callbacks => await ImportLegacyArchiveFromPaths(Db, paths, progressDialog, callbacks[0])) { progressDialog.DataContext = new ProgressDialogModel(async callback => await ImportLegacyArchiveFromPaths(Db, paths, progressDialog, callback)) {
Title = "Legacy Archive Import" Title = "Legacy Archive Import"
}; };
await progressDialog.ShowProgressDialog(window); await progressDialog.ShowDialog(window);
} }
private static async Task ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) { private static async Task ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {

View File

@ -45,12 +45,12 @@ namespace DHT.Desktop.Main.Pages {
} }
ProgressDialog progressDialog = new ProgressDialog { ProgressDialog progressDialog = new ProgressDialog {
DataContext = new ProgressDialogModel(async callbacks => await GenerateRandomData(channels, users, messages, callbacks[0])) { DataContext = new ProgressDialogModel(async callback => await GenerateRandomData(channels, users, messages, callback)) {
Title = "Generating Random Data" Title = "Generating Random Data"
} }
}; };
await progressDialog.ShowProgressDialog(window); await progressDialog.ShowDialog(window);
} }
private const int BatchSize = 500; private const int BatchSize = 500;

View File

@ -1,13 +1,10 @@
using System; using System;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using Avalonia.Controls; using Avalonia.Controls;
using DHT.Desktop.Common; using DHT.Desktop.Common;
using DHT.Desktop.Dialogs.Message; using DHT.Desktop.Dialogs.Message;
using DHT.Desktop.Dialogs.Progress;
using DHT.Server.Database; using DHT.Server.Database;
using DHT.Server.Database.Sqlite;
using DHT.Utils.Models; using DHT.Utils.Models;
namespace DHT.Desktop.Main.Screens; namespace DHT.Desktop.Main.Screens;
@ -42,71 +39,14 @@ sealed class WelcomeScreenModel : BaseModel, IDisposable {
} }
dbFilePath = path; dbFilePath = path;
Db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, window, new SchemaUpgradeCallbacks(window)); Db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, window, CheckCanUpgradeDatabase);
OnPropertyChanged(nameof(Db)); OnPropertyChanged(nameof(Db));
OnPropertyChanged(nameof(HasDatabase)); OnPropertyChanged(nameof(HasDatabase));
} }
private sealed class SchemaUpgradeCallbacks : ISchemaUpgradeCallbacks { private async Task<bool> CheckCanUpgradeDatabase() {
private readonly Window window; return DialogResult.YesNo.Yes == await DatabaseGui.ShowCanUpgradeDatabaseDialog(window);
public SchemaUpgradeCallbacks(Window window) {
this.window = window;
}
public async Task<bool> CanUpgrade() {
return DialogResult.YesNo.Yes == await DatabaseGui.ShowCanUpgradeDatabaseDialog(window);
}
public async Task Start(int versionSteps, Func<ISchemaUpgradeCallbacks.IProgressReporter, Task> doUpgrade) {
async Task StartUpgrade(IReadOnlyList<IProgressCallback> callbacks) {
var reporter = new ProgressReporter(versionSteps, callbacks);
await reporter.NextVersion();
await Task.Delay(TimeSpan.FromMilliseconds(800));
await doUpgrade(reporter);
await Task.Delay(TimeSpan.FromMilliseconds(600));
}
await new ProgressDialog {
DataContext = new ProgressDialogModel(StartUpgrade, progressItems: 3) {
Title = "Upgrading Database"
}
}.ShowProgressDialog(window);
}
private sealed class ProgressReporter : ISchemaUpgradeCallbacks.IProgressReporter {
private readonly IReadOnlyList<IProgressCallback> callbacks;
private readonly int versionSteps;
private int versionProgress = 0;
public ProgressReporter(int versionSteps, IReadOnlyList<IProgressCallback> callbacks) {
this.callbacks = callbacks;
this.versionSteps = versionSteps;
}
public async Task NextVersion() {
await callbacks[0].Update("Upgrading schema version...", versionProgress++, versionSteps);
await HideChildren(0);
}
public async Task MainWork(string message, int finishedItems, int totalItems) {
await callbacks[1].Update(message, finishedItems, totalItems);
await HideChildren(1);
}
public async Task SubWork(string message, int finishedItems, int totalItems) {
await callbacks[2].Update(message, finishedItems, totalItems);
await HideChildren(2);
}
private async Task HideChildren(int parentIndex) {
for (int i = parentIndex + 1; i < callbacks.Count; i++) {
await callbacks[i].Hide();
}
}
}
} }
public void CloseDatabase() { public void CloseDatabase() {

View File

@ -1,15 +0,0 @@
using System;
using System.Threading.Tasks;
namespace DHT.Server.Database.Sqlite;
public interface ISchemaUpgradeCallbacks {
Task<bool> CanUpgrade();
Task Start(int versionSteps, Func<IProgressReporter, Task> doUpgrade);
public interface IProgressReporter {
Task NextVersion();
Task MainWork(string message, int finishedItems, int totalItems);
Task SubWork(string message, int finishedItems, int totalItems);
}
}

View File

@ -1,3 +1,4 @@
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Database.Exceptions; using DHT.Server.Database.Exceptions;
@ -19,8 +20,12 @@ sealed class Schema {
this.conn = conn; this.conn = conn;
} }
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) { private void Execute(string sql) {
conn.Execute(@"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)"); conn.Command(sql).ExecuteNonQuery();
}
public async Task<bool> Setup(Func<Task<bool>> checkCanUpgradeSchemas) {
Execute(@"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
var dbVersionStr = conn.SelectScalar("SELECT value FROM metadata WHERE key = 'version'"); var dbVersionStr = conn.SelectScalar("SELECT value FROM metadata WHERE key = 'version'");
if (dbVersionStr == null) { if (dbVersionStr == null) {
@ -33,133 +38,131 @@ sealed class Schema {
throw new DatabaseTooNewException(dbVersion); throw new DatabaseTooNewException(dbVersion);
} }
else if (dbVersion < Version) { else if (dbVersion < Version) {
var proceed = await callbacks.CanUpgrade(); var proceed = await checkCanUpgradeSchemas();
if (!proceed) { if (!proceed) {
return false; return false;
} }
await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter)); UpgradeSchemas(dbVersion);
} }
return true; return true;
} }
private void InitializeSchemas() { private void InitializeSchemas() {
conn.Execute(""" Execute("""
CREATE TABLE users ( CREATE TABLE users (
id INTEGER PRIMARY KEY NOT NULL, id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
avatar_url TEXT, avatar_url TEXT,
discriminator TEXT discriminator TEXT
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE servers ( CREATE TABLE servers (
id INTEGER PRIMARY KEY NOT NULL, id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
type TEXT NOT NULL type TEXT NOT NULL
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE channels ( CREATE TABLE channels (
id INTEGER PRIMARY KEY NOT NULL, id INTEGER PRIMARY KEY NOT NULL,
server INTEGER NOT NULL, server INTEGER NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
parent_id INTEGER, parent_id INTEGER,
position INTEGER, position INTEGER,
topic TEXT, topic TEXT,
nsfw INTEGER nsfw INTEGER
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE messages ( CREATE TABLE messages (
message_id INTEGER PRIMARY KEY NOT NULL, message_id INTEGER PRIMARY KEY NOT NULL,
sender_id INTEGER NOT NULL, sender_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL, channel_id INTEGER NOT NULL,
text TEXT NOT NULL, text TEXT NOT NULL,
timestamp INTEGER NOT NULL timestamp INTEGER NOT NULL
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE attachments ( CREATE TABLE attachments (
message_id INTEGER NOT NULL, message_id INTEGER NOT NULL,
attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL, attachment_id INTEGER NOT NULL PRIMARY KEY NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
type TEXT, type TEXT,
normalized_url TEXT NOT NULL, normalized_url TEXT NOT NULL,
download_url TEXT, download_url TEXT,
size INTEGER NOT NULL, size INTEGER NOT NULL,
width INTEGER, width INTEGER,
height INTEGER height INTEGER
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE embeds ( CREATE TABLE embeds (
message_id INTEGER NOT NULL, message_id INTEGER NOT NULL,
json TEXT NOT NULL json TEXT NOT NULL
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE downloads ( CREATE TABLE downloads (
normalized_url TEXT NOT NULL PRIMARY KEY, normalized_url TEXT NOT NULL PRIMARY KEY,
download_url TEXT, download_url TEXT,
status INTEGER NOT NULL, status INTEGER NOT NULL,
size INTEGER NOT NULL, size INTEGER NOT NULL,
blob BLOB blob BLOB
) )
"""); """);
conn.Execute(""" Execute("""
CREATE TABLE reactions ( CREATE TABLE reactions (
message_id INTEGER NOT NULL, message_id INTEGER NOT NULL,
emoji_id INTEGER, emoji_id INTEGER,
emoji_name TEXT, emoji_name TEXT,
emoji_flags INTEGER NOT NULL, emoji_flags INTEGER NOT NULL,
count INTEGER NOT NULL count INTEGER NOT NULL
) )
"""); """);
CreateMessageEditTimestampTable(); CreateMessageEditTimestampTable();
CreateMessageRepliedToTable(); CreateMessageRepliedToTable();
conn.Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)"); Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
conn.Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)"); Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
conn.Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)"); Execute("CREATE INDEX reactions_message_ix ON reactions(message_id)");
conn.Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")"); Execute("INSERT INTO metadata (key, value) VALUES ('version', " + Version + ")");
} }
private void CreateMessageEditTimestampTable() { private void CreateMessageEditTimestampTable() {
conn.Execute(""" Execute("""
CREATE TABLE edit_timestamps ( CREATE TABLE edit_timestamps (
message_id INTEGER PRIMARY KEY NOT NULL, message_id INTEGER PRIMARY KEY NOT NULL,
edit_timestamp INTEGER NOT NULL edit_timestamp INTEGER NOT NULL
) )
"""); """);
} }
private void CreateMessageRepliedToTable() { private void CreateMessageRepliedToTable() {
conn.Execute(""" Execute("""
CREATE TABLE replied_to ( CREATE TABLE replied_to (
message_id INTEGER PRIMARY KEY NOT NULL, message_id INTEGER PRIMARY KEY NOT NULL,
replied_to_id INTEGER NOT NULL replied_to_id INTEGER NOT NULL
) )
"""); """);
} }
private async Task NormalizeAttachmentUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) { private void NormalizeAttachmentUrls() {
await reporter.SubWork("Preparing attachments...", 0, 0);
var normalizedUrls = new Dictionary<long, string>(); var normalizedUrls = new Dictionary<long, string>();
await using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) { using (var selectCmd = conn.Command("SELECT attachment_id, url FROM attachments")) {
await using var reader = await selectCmd.ExecuteReaderAsync(); using var reader = selectCmd.ExecuteReader();
while (reader.Read()) { while (reader.Read()) {
var attachmentId = reader.GetInt64(0); var attachmentId = reader.GetInt64(0);
@ -168,39 +171,28 @@ sealed class Schema {
} }
} }
await using var tx = conn.BeginTransaction(); using var tx = conn.BeginTransaction();
int totalUrls = normalizedUrls.Count; using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
int processedUrls = -1;
await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
updateCmd.Parameters.Add(":attachment_id", SqliteType.Integer); updateCmd.Parameters.Add(":attachment_id", SqliteType.Integer);
updateCmd.Parameters.Add(":normalized_url", SqliteType.Text); updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
foreach (var (attachmentId, normalizedUrl) in normalizedUrls) { foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
if (++processedUrls % 1000 == 0) {
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
}
updateCmd.Set(":attachment_id", attachmentId); updateCmd.Set(":attachment_id", attachmentId);
updateCmd.Set(":normalized_url", normalizedUrl); updateCmd.Set(":normalized_url", normalizedUrl);
updateCmd.ExecuteNonQuery(); updateCmd.ExecuteNonQuery();
} }
} }
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); tx.Commit();
await tx.CommitAsync();
} }
private async Task NormalizeDownloadUrls(ISchemaUpgradeCallbacks.IProgressReporter reporter) { private void NormalizeDownloadUrls() {
await reporter.SubWork("Preparing downloads...", 0, 0);
var normalizedUrlsToOriginalUrls = new Dictionary<string, string>(); var normalizedUrlsToOriginalUrls = new Dictionary<string, string>();
var duplicateUrlsToDelete = new HashSet<string>(); var duplicateUrlsToDelete = new HashSet<string>();
await using (var selectCmd = conn.Command("SELECT url FROM downloads ORDER BY CASE WHEN status = 200 THEN 0 ELSE 1 END")) { using (var selectCmd = conn.Command("SELECT url FROM downloads ORDER BY CASE WHEN status = 200 THEN 0 ELSE 1 END")) {
await using var reader = await selectCmd.ExecuteReaderAsync(); using var reader = selectCmd.ExecuteReader();
while (reader.Read()) { while (reader.Read()) {
var originalUrl = reader.GetString(0); var originalUrl = reader.GetString(0);
@ -212,144 +204,96 @@ sealed class Schema {
} }
} }
conn.Execute("PRAGMA cache_size = -20000"); using var tx = conn.BeginTransaction();
SqliteTransaction tx;
await using (tx = conn.BeginTransaction()) { using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) {
await reporter.SubWork("Deleting duplicates...", 0, 0); foreach (var duplicateUrl in duplicateUrlsToDelete) {
deleteCmd.Set(":url", duplicateUrl);
await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) { deleteCmd.ExecuteNonQuery();
foreach (var duplicateUrl in duplicateUrlsToDelete) {
deleteCmd.Set(":url", duplicateUrl);
deleteCmd.ExecuteNonQuery();
}
} }
await tx.CommitAsync();
} }
int totalUrls = normalizedUrlsToOriginalUrls.Count; using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
int processedUrls = -1;
tx = conn.BeginTransaction();
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
updateCmd.Parameters.Add(":normalized_url", SqliteType.Text); updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
updateCmd.Parameters.Add(":download_url", SqliteType.Text); updateCmd.Parameters.Add(":download_url", SqliteType.Text);
foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) { foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
if (++processedUrls % 100 == 0) {
await reporter.SubWork("Updating URLs...", processedUrls, totalUrls);
// Not proper way of dealing with transactions, but it avoids a long commit at the end.
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
await tx.CommitAsync();
await tx.DisposeAsync();
tx = conn.BeginTransaction();
updateCmd.Transaction = tx;
}
updateCmd.Set(":normalized_url", normalizedUrl); updateCmd.Set(":normalized_url", normalizedUrl);
updateCmd.Set(":download_url", downloadUrl); updateCmd.Set(":download_url", downloadUrl);
updateCmd.ExecuteNonQuery(); updateCmd.ExecuteNonQuery();
} }
} }
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); tx.Commit();
await tx.CommitAsync();
await tx.DisposeAsync();
conn.Execute("PRAGMA cache_size = -2000");
} }
private async Task UpgradeSchemas(int dbVersion, ISchemaUpgradeCallbacks.IProgressReporter reporter) { private void UpgradeSchemas(int dbVersion) {
var perf = Log.Start("from version " + dbVersion); var perf = Log.Start("from version " + dbVersion);
conn.Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'"); Execute("UPDATE metadata SET value = " + Version + " WHERE key = 'version'");
if (dbVersion <= 1) { if (dbVersion <= 1) {
await reporter.MainWork("Applying schema changes...", 0, 1); Execute("ALTER TABLE channels ADD parent_id INTEGER");
conn.Execute("ALTER TABLE channels ADD parent_id INTEGER");
perf.Step("Upgrade to version 2"); perf.Step("Upgrade to version 2");
await reporter.NextVersion();
} }
if (dbVersion <= 2) { if (dbVersion <= 2) {
await reporter.MainWork("Applying schema changes...", 0, 1);
CreateMessageEditTimestampTable(); CreateMessageEditTimestampTable();
CreateMessageRepliedToTable(); CreateMessageRepliedToTable();
conn.Execute(""" Execute("""
INSERT INTO edit_timestamps (message_id, edit_timestamp) INSERT INTO edit_timestamps (message_id, edit_timestamp)
SELECT message_id, edit_timestamp SELECT message_id, edit_timestamp
FROM messages FROM messages
WHERE edit_timestamp IS NOT NULL WHERE edit_timestamp IS NOT NULL
"""); """);
conn.Execute(""" Execute("""
INSERT INTO replied_to (message_id, replied_to_id) INSERT INTO replied_to (message_id, replied_to_id)
SELECT message_id, replied_to_id SELECT message_id, replied_to_id
FROM messages FROM messages
WHERE replied_to_id IS NOT NULL WHERE replied_to_id IS NOT NULL
"""); """);
conn.Execute("ALTER TABLE messages DROP COLUMN replied_to_id"); Execute("ALTER TABLE messages DROP COLUMN replied_to_id");
conn.Execute("ALTER TABLE messages DROP COLUMN edit_timestamp"); Execute("ALTER TABLE messages DROP COLUMN edit_timestamp");
perf.Step("Upgrade to version 3"); perf.Step("Upgrade to version 3");
await reporter.MainWork("Vacuuming the database...", 1, 1); Execute("VACUUM");
conn.Execute("VACUUM");
perf.Step("Vacuum"); perf.Step("Vacuum");
await reporter.NextVersion();
} }
if (dbVersion <= 3) { if (dbVersion <= 3) {
conn.Execute(""" Execute("""
CREATE TABLE downloads ( CREATE TABLE downloads (
url TEXT NOT NULL PRIMARY KEY, url TEXT NOT NULL PRIMARY KEY,
status INTEGER NOT NULL, status INTEGER NOT NULL,
size INTEGER NOT NULL, size INTEGER NOT NULL,
blob BLOB blob BLOB
) )
"""); """);
perf.Step("Upgrade to version 4"); perf.Step("Upgrade to version 4");
await reporter.NextVersion();
} }
if (dbVersion <= 4) { if (dbVersion <= 4) {
await reporter.MainWork("Applying schema changes...", 0, 1); Execute("ALTER TABLE attachments ADD width INTEGER");
conn.Execute("ALTER TABLE attachments ADD width INTEGER"); Execute("ALTER TABLE attachments ADD height INTEGER");
conn.Execute("ALTER TABLE attachments ADD height INTEGER");
perf.Step("Upgrade to version 5"); perf.Step("Upgrade to version 5");
await reporter.NextVersion();
} }
if (dbVersion <= 5) { if (dbVersion <= 5) {
await reporter.MainWork("Applying schema changes...", 0, 3); Execute("ALTER TABLE attachments ADD download_url TEXT");
conn.Execute("ALTER TABLE attachments ADD download_url TEXT"); Execute("ALTER TABLE downloads ADD download_url TEXT");
conn.Execute("ALTER TABLE downloads ADD download_url TEXT");
await reporter.MainWork("Updating attachments...", 1, 3); NormalizeAttachmentUrls();
await NormalizeAttachmentUrls(reporter); NormalizeDownloadUrls();
await reporter.MainWork("Updating downloads...", 2, 3); Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
await NormalizeDownloadUrls(reporter); Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
await reporter.MainWork("Applying schema changes...", 3, 3);
conn.Execute("ALTER TABLE attachments RENAME COLUMN url TO normalized_url");
conn.Execute("ALTER TABLE downloads RENAME COLUMN url TO normalized_url");
perf.Step("Upgrade to version 6"); perf.Step("Upgrade to version 6");
await reporter.NextVersion();
} }
perf.End(); perf.End();

View File

@ -18,7 +18,7 @@ namespace DHT.Server.Database.Sqlite;
public sealed class SqliteDatabaseFile : IDatabaseFile { public sealed class SqliteDatabaseFile : IDatabaseFile {
private const int DefaultPoolSize = 5; private const int DefaultPoolSize = 5;
public static async Task<SqliteDatabaseFile?> OpenOrCreate(string path, ISchemaUpgradeCallbacks schemaUpgradeCallbacks) { public static async Task<SqliteDatabaseFile?> OpenOrCreate(string path, Func<Task<bool>> checkCanUpgradeSchemas) {
var connectionString = new SqliteConnectionStringBuilder { var connectionString = new SqliteConnectionStringBuilder {
DataSource = path, DataSource = path,
Mode = SqliteOpenMode.ReadWriteCreate, Mode = SqliteOpenMode.ReadWriteCreate,
@ -27,12 +27,8 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
var pool = new SqliteConnectionPool(connectionString, DefaultPoolSize); var pool = new SqliteConnectionPool(connectionString, DefaultPoolSize);
bool wasOpened; bool wasOpened;
try { using (var conn = pool.Take()) {
using var conn = pool.Take(); wasOpened = await new Schema(conn).Setup(checkCanUpgradeSchemas);
wasOpened = await new Schema(conn).Setup(schemaUpgradeCallbacks);
} catch (Exception) {
pool.Dispose();
throw;
} }
if (wasOpened) { if (wasOpened) {

View File

@ -5,19 +5,14 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils; namespace DHT.Server.Database.Sqlite.Utils;
static class SqliteExtensions { static class SqliteExtensions {
public static SqliteTransaction BeginTransaction(this ISqliteConnection conn) {
return conn.InnerConnection.BeginTransaction();
}
public static SqliteCommand Command(this ISqliteConnection conn, string sql) { public static SqliteCommand Command(this ISqliteConnection conn, string sql) {
var cmd = conn.InnerConnection.CreateCommand(); var cmd = conn.InnerConnection.CreateCommand();
cmd.CommandText = sql; cmd.CommandText = sql;
return cmd; return cmd;
} }
public static void Execute(this ISqliteConnection conn, string sql) { public static SqliteTransaction BeginTransaction(this ISqliteConnection conn) {
using var cmd = conn.Command(sql); return conn.InnerConnection.BeginTransaction();
cmd.ExecuteNonQuery();
} }
public static object? SelectScalar(this ISqliteConnection conn, string sql) { public static object? SelectScalar(this ISqliteConnection conn, string sql) {

View File

@ -8,5 +8,5 @@ using DHT.Utils;
namespace DHT.Utils; namespace DHT.Utils;
static class Version { static class Version {
public const string Tag = "39.1.0.0"; public const string Tag = "39.0.0.0";
} }