1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-19 13:15:44 +02:00

Compare commits

..

No commits in common. "5741fad528e66ef953df5e350bfbc49004adb05a" and "50d5bdba97eed69990594c9627384e5cf4d4963e" have entirely different histories.

28 changed files with 230 additions and 465 deletions

View File

@ -13,7 +13,6 @@
<entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Dialogs/NewDatabaseSettingsDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" /> <entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />

View File

@ -54,22 +54,6 @@
<Setter Property="HorizontalAlignment" Value="Stretch" /> <Setter Property="HorizontalAlignment" Value="Stretch" />
</Style> </Style>
<Style Selector="Panel.buttons">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.right">
<Setter Property="HorizontalAlignment" Value="Right" />
</Style>
<Style Selector="Panel.buttons Button">
<Setter Property="MinWidth" Value="80" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.left Button">
<Setter Property="Margin" Value="0 0 8 0" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.right Button">
<Setter Property="Margin" Value="8 0 0 0" />
</Style>
</Application.Styles> </Application.Styles>
<Application.Resources> <Application.Resources>

View File

@ -15,6 +15,24 @@
<namespace:CheckBoxDialogModel /> <namespace:CheckBoxDialogModel />
</Window.DataContext> </Window.DataContext>
<Window.Styles>
<Style Selector="Panel.buttons">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.right">
<Setter Property="HorizontalAlignment" Value="Right" />
</Style>
<Style Selector="Panel.buttons Button">
<Setter Property="MinWidth" Value="80" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.left Button">
<Setter Property="Margin" Value="0 0 8 0" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.right Button">
<Setter Property="Margin" Value="8 0 0 0" />
</Style>
</Window.Styles>
<StackPanel Margin="20"> <StackPanel Margin="20">
<ScrollViewer MaxHeight="400"> <ScrollViewer MaxHeight="400">
<ItemsRepeater ItemsSource="{Binding Items}"> <ItemsRepeater ItemsSource="{Binding Items}">

View File

@ -20,18 +20,24 @@
<Setter Property="FontSize" Value="15" /> <Setter Property="FontSize" Value="15" />
<Setter Property="TextWrapping" Value="Wrap" /> <Setter Property="TextWrapping" Value="Wrap" />
</Style> </Style>
<Style Selector="WrapPanel">
<Setter Property="HorizontalAlignment" Value="Right" />
<Setter Property="Margin" Value="0 8 0 0" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="8 0 0 0" />
<Setter Property="MinWidth" Value="80" />
</Style>
</Window.Styles> </Window.Styles>
<StackPanel Margin="20"> <StackPanel Margin="20">
<TextBlock Text="{Binding Message}" /> <TextBlock Text="{Binding Message}" />
<Panel Classes="buttons"> <WrapPanel>
<WrapPanel Classes="right">
<Button Click="ClickOk" IsVisible="{Binding IsOkVisible}">OK</Button> <Button Click="ClickOk" IsVisible="{Binding IsOkVisible}">OK</Button>
<Button Click="ClickYes" IsVisible="{Binding IsYesVisible}">Yes</Button> <Button Click="ClickYes" IsVisible="{Binding IsYesVisible}">Yes</Button>
<Button Click="ClickNo" IsVisible="{Binding IsNoVisible}">No</Button> <Button Click="ClickNo" IsVisible="{Binding IsNoVisible}">No</Button>
<Button Click="ClickCancel" IsVisible="{Binding IsCancelVisible}">Cancel</Button> <Button Click="ClickCancel" IsVisible="{Binding IsCancelVisible}">Cancel</Button>
</WrapPanel> </WrapPanel>
</Panel>
</StackPanel> </StackPanel>
</Window> </Window>

View File

@ -15,6 +15,19 @@
<namespace:TextBoxDialogModel /> <namespace:TextBoxDialogModel />
</Window.DataContext> </Window.DataContext>
<Window.Styles>
<Style Selector="Panel.buttons">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Panel.buttons > WrapPanel.right">
<Setter Property="HorizontalAlignment" Value="Right" />
</Style>
<Style Selector="Panel.buttons Button">
<Setter Property="MinWidth" Value="80" />
<Setter Property="Margin" Value="8 0 0 0" />
</Style>
</Window.Styles>
<StackPanel Margin="20"> <StackPanel Margin="20">
<ScrollViewer MaxHeight="400"> <ScrollViewer MaxHeight="400">
<StackPanel Spacing="10"> <StackPanel Spacing="10">

View File

@ -1,57 +0,0 @@
<Window xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:namespace="clr-namespace:DHT.Desktop.Main.Dialogs"
mc:Ignorable="d" d:DesignWidth="450"
x:Class="DHT.Desktop.Main.Dialogs.NewDatabaseSettingsDialog"
x:DataType="namespace:NewDatabaseSettingsDialogModel"
Title="New Database Settings"
Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
Closing="OnClosing"
Width="450" SizeToContent="Height" CanResize="False"
WindowStartupLocation="CenterOwner">
<Window.DataContext>
<namespace:NewDatabaseSettingsDialogModel />
</Window.DataContext>
<Window.Styles>
<Style Selector="Label > StackPanel">
<Setter Property="Margin" Value="0 4 0 0" />
<Setter Property="Orientation" Value="Vertical" />
<Setter Property="Spacing" Value="5" />
</Style>
<Style Selector="TextBlock.dim">
<Setter Property="Foreground" Value="DimGray" />
<Setter Property="FontSize" Value="13" />
</Style>
</Window.Styles>
<StackPanel Orientation="Vertical" Margin="20">
<StackPanel Orientation="Vertical" Spacing="15">
<CheckBox IsChecked="{Binding SeparateFileForDownloads}">
<Label>
<StackPanel>
<TextBlock>Use a Separate Database File for Downloads</TextBlock>
<TextBlock Classes="dim">Downloaded files will be stored separately from messages. This cannot be changed later.</TextBlock>
</StackPanel>
</Label>
</CheckBox>
<CheckBox IsChecked="{Binding DownloadsAutoStart}">
<Label>
<StackPanel>
<TextBlock>Download Files Automatically</TextBlock>
<TextBlock Classes="dim">Enables automatic downloads of files hosted on Discord. This can be changed later in the Downloads tab.</TextBlock>
</StackPanel>
</Label>
</CheckBox>
</StackPanel>
<Panel Classes="buttons">
<WrapPanel Classes="right">
<Button Click="ClickOk">OK</Button>
</WrapPanel>
</Panel>
</StackPanel>
</Window>

View File

@ -1,22 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia.Controls;
using Avalonia.Interactivity;
namespace DHT.Desktop.Main.Dialogs;
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed partial class NewDatabaseSettingsDialog : Window {
public NewDatabaseSettingsDialog() {
InitializeComponent();
}
private void OnClosing(object? sender, WindowClosingEventArgs e) {
if (!e.IsProgrammatic) {
e.Cancel = true;
}
}
private void ClickOk(object? sender, RoutedEventArgs e) {
Close();
}
}

View File

@ -1,11 +0,0 @@
using CommunityToolkit.Mvvm.ComponentModel;
namespace DHT.Desktop.Main.Dialogs;
sealed partial class NewDatabaseSettingsDialogModel : ObservableObject {
[ObservableProperty]
private bool separateFileForDownloads = true;
[ObservableProperty]
private bool downloadsAutoStart = true;
}

View File

@ -16,5 +16,11 @@
<Expander Header="Internal Server Configuration" IsExpanded="True"> <Expander Header="Internal Server Configuration" IsExpanded="True">
<controls:ServerConfigurationPanel DataContext="{Binding ServerConfigurationModel}" /> <controls:ServerConfigurationPanel DataContext="{Binding ServerConfigurationModel}" />
</Expander> </Expander>
<Expander Header="Database Tools" IsExpanded="True">
<StackPanel Orientation="Vertical" Spacing="10">
<TextBlock TextWrapping="Wrap">Recreates the database to remove any traces of deleted data, which frees up space and prevents forensic recovery.</TextBlock>
<Button Command="{Binding VacuumDatabase}">Vacuum Database</Button>
</StackPanel>
</Expander>
</StackPanel> </StackPanel>
</UserControl> </UserControl>

View File

@ -1,5 +1,8 @@
using System; using System;
using System.Threading.Tasks;
using Avalonia.Controls; using Avalonia.Controls;
using DHT.Desktop.Dialogs.Message;
using DHT.Desktop.Dialogs.Progress;
using DHT.Desktop.Main.Controls; using DHT.Desktop.Main.Controls;
using DHT.Server; using DHT.Server;
@ -8,14 +11,26 @@ namespace DHT.Desktop.Main.Pages;
sealed class AdvancedPageModel : IDisposable { sealed class AdvancedPageModel : IDisposable {
public ServerConfigurationPanelModel ServerConfigurationModel { get; } public ServerConfigurationPanelModel ServerConfigurationModel { get; }
private readonly Window window;
private readonly State state;
[Obsolete("Designer")] [Obsolete("Designer")]
public AdvancedPageModel() : this(null!, State.Dummy) {} public AdvancedPageModel() : this(null!, State.Dummy) {}
public AdvancedPageModel(Window window, State state) { public AdvancedPageModel(Window window, State state) {
this.window = window;
this.state = state;
ServerConfigurationModel = new ServerConfigurationPanelModel(window, state); ServerConfigurationModel = new ServerConfigurationPanelModel(window, state);
} }
public void Dispose() { public void Dispose() {
ServerConfigurationModel.Dispose(); ServerConfigurationModel.Dispose();
} }
public async Task VacuumDatabase() {
const string Title = "Vacuum Database";
await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => state.Db.Vacuum());
await Dialog.ShowOk(window, Title, "Done.");
}
} }

View File

@ -27,12 +27,6 @@
<Button Command="{Binding MergeWithDatabase}">Merge with Database(s)...</Button> <Button Command="{Binding MergeWithDatabase}">Merge with Database(s)...</Button>
<Button Command="{Binding ImportLegacyArchive}">Import Legacy Archive(s)...</Button> <Button Command="{Binding ImportLegacyArchive}">Import Legacy Archive(s)...</Button>
</WrapPanel> </WrapPanel>
<Expander Header="Advanced Tools" Margin="0 15 0 0">
<StackPanel Orientation="Vertical" Spacing="10">
<TextBlock TextWrapping="Wrap">Recreate the database to free up space after deleting data.</TextBlock>
<Button Command="{Binding VacuumDatabase}">Vacuum Database</Button>
</StackPanel>
</Expander>
</StackPanel> </StackPanel>
</UserControl> </UserControl>

View File

@ -19,7 +19,6 @@ using DHT.Server.Data;
using DHT.Server.Database; using DHT.Server.Database;
using DHT.Server.Database.Import; using DHT.Server.Database.Import;
using DHT.Server.Database.Sqlite.Schema; using DHT.Server.Database.Sqlite.Schema;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Logging; using DHT.Utils.Logging;
namespace DHT.Desktop.Main.Pages; namespace DHT.Desktop.Main.Pages;
@ -55,11 +54,11 @@ sealed class DatabasePageModel {
break; break;
case PlatformID.Unix: case PlatformID.Unix:
Process.Start("xdg-open", [ folder ]); Process.Start("xdg-open", new string[] { folder });
break; break;
case PlatformID.MacOSX: case PlatformID.MacOSX:
Process.Start("open", [ folder ]); Process.Start("open", new string[] { folder });
break; break;
default: default:
@ -98,11 +97,14 @@ sealed class DatabasePageModel {
}); });
} }
private sealed class SchemaUpgradeCallbacks(ProgressDialog dialog, int total) : ISchemaUpgradeCallbacks { private sealed class SchemaUpgradeCallbacks : ISchemaUpgradeCallbacks {
private readonly ProgressDialog dialog;
private readonly int total;
private bool? decision; private bool? decision;
public Task<InitialDatabaseSettings?> GetInitialDatabaseSettings() { public SchemaUpgradeCallbacks(ProgressDialog dialog, int total) {
return Task.FromResult<InitialDatabaseSettings?>(null); this.total = total;
this.dialog = dialog;
} }
public async Task<bool> CanUpgrade() { public async Task<bool> CanUpgrade() {
@ -261,10 +263,4 @@ sealed class DatabasePageModel {
return message.ToString(); return message.ToString();
} }
public async Task VacuumDatabase() {
const string Title = "Vacuum Database";
await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => Db.Vacuum());
await Dialog.ShowOk(window, Title, "Done.");
}
} }

View File

@ -165,10 +165,13 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken)); downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
} }
public async Task OnClickDeleteOrphanedDownloads() { private const string DeleteOrphanedDownloadsTitle = "Delete Orphaned Downloads";
const string Title = "Delete Orphaned Downloads";
await ProgressDialog.Show(window, Title, async (_, callback) => { public async Task OnClickDeleteOrphanedDownloads() {
await ProgressDialog.Show(window, DeleteOrphanedDownloadsTitle, DeleteOrphanedDownloads);
}
private async Task DeleteOrphanedDownloads(ProgressDialog dialog, IProgressCallback callback) {
await callback.UpdateIndeterminate("Searching for orphaned downloads..."); await callback.UpdateIndeterminate("Searching for orphaned downloads...");
HashSet<string> reachableNormalizedUrls = []; HashSet<string> reachableNormalizedUrls = [];
@ -186,11 +189,11 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
} }
if (orphanedNormalizedUrls.Count == 0) { if (orphanedNormalizedUrls.Count == 0) {
await Dialog.ShowOk(window, Title, "No orphaned downloads found."); await Dialog.ShowOk(window, DeleteOrphanedDownloadsTitle, "No orphaned downloads found.");
return; return;
} }
if (await Dialog.ShowYesNo(window, Title, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) { if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
return; return;
} }
@ -198,13 +201,12 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
await state.Db.Downloads.Remove(orphanedNormalizedUrls); await state.Db.Downloads.Remove(orphanedNormalizedUrls);
RecomputeDownloadStatistics(); RecomputeDownloadStatistics();
if (await Dialog.ShowYesNo(window, Title, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) { if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
return; return;
} }
await callback.UpdateIndeterminate("Vacuuming database..."); await callback.UpdateIndeterminate("Vacuuming database...");
await state.Db.Vacuum(); await state.Db.Vacuum();
});
} }
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) { private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {

View File

@ -9,10 +9,9 @@ using CommunityToolkit.Mvvm.ComponentModel;
using DHT.Desktop.Common; using DHT.Desktop.Common;
using DHT.Desktop.Dialogs.Message; using DHT.Desktop.Dialogs.Message;
using DHT.Desktop.Dialogs.Progress; using DHT.Desktop.Dialogs.Progress;
using DHT.Desktop.Main.Dialogs; using DHT.Server.Data.Settings;
using DHT.Server.Database; using DHT.Server.Database;
using DHT.Server.Database.Sqlite.Schema; using DHT.Server.Database.Sqlite.Schema;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Logging; using DHT.Utils.Logging;
namespace DHT.Desktop.Main.Screens; namespace DHT.Desktop.Main.Screens;
@ -53,22 +52,21 @@ sealed partial class WelcomeScreenModel : ObservableObject {
public async Task OpenOrCreateDatabaseFromPath(string path) { public async Task OpenOrCreateDatabaseFromPath(string path) {
dbFilePath = path; dbFilePath = path;
bool isNew = !File.Exists(path);
IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, window, new SchemaUpgradeCallbacks(window)); IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, window, new SchemaUpgradeCallbacks(window));
if (db != null) { if (db == null) {
DatabaseSelected?.Invoke(this, db); return;
} }
if (isNew && await Dialog.ShowYesNo(window, "Automatic Downloads", "Do you want to automatically download files hosted on Discord? You can change this later in the Downloads tab.") == DialogResult.YesNo.Yes) {
await db.Settings.Set(SettingsKey.DownloadsAutoStart, value: true);
}
DatabaseSelected?.Invoke(this, db);
} }
private sealed class SchemaUpgradeCallbacks(Window window) : ISchemaUpgradeCallbacks { private sealed class SchemaUpgradeCallbacks(Window window) : ISchemaUpgradeCallbacks {
public async Task<InitialDatabaseSettings?> GetInitialDatabaseSettings() {
var model = new NewDatabaseSettingsDialogModel();
var dialog = new NewDatabaseSettingsDialog { DataContext = model };
await dialog.ShowDialog(window);
return new InitialDatabaseSettings(model.SeparateFileForDownloads, model.DownloadsAutoStart);
}
public async Task<bool> CanUpgrade() { public async Task<bool> CanUpgrade() {
return DialogResult.YesNo.Yes == await DatabaseGui.ShowCanUpgradeDatabaseDialog(window); return DialogResult.YesNo.Yes == await DatabaseGui.ShowCanUpgradeDatabaseDialog(window);
} }

View File

@ -3,7 +3,6 @@ using System.Diagnostics.CodeAnalysis;
namespace DHT.Server.Data.Settings; namespace DHT.Server.Data.Settings;
public static class SettingsKey { public static class SettingsKey {
public static Bool SeparateFileForDownloads { get; } = new ("separate_file_for_downloads");
public static Bool DownloadsAutoStart { get; } = new ("downloads_auto_start"); public static Bool DownloadsAutoStart { get; } = new ("downloads_auto_start");
public static Bool DownloadsLimitSize { get; } = new ("downloads_limit_size"); public static Bool DownloadsLimitSize { get; } = new ("downloads_limit_size");
public static UnsignedLong DownloadsMaximumSize { get; } = new ("downloads_maximum_size"); public static UnsignedLong DownloadsMaximumSize { get; } = new ("downloads_maximum_size");

View File

@ -18,8 +18,6 @@ namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository { sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository {
private static readonly Log Log = Log.ForType<SqliteDownloadRepository>(); private static readonly Log Log = Log.ForType<SqliteDownloadRepository>();
public const string Schema = "downloads";
internal sealed class NewDownloadCollector : IAsyncDisposable { internal sealed class NewDownloadCollector : IAsyncDisposable {
private readonly SqliteDownloadRepository repository; private readonly SqliteDownloadRepository repository;
private bool hasChanged = false; private bool hasChanged = false;
@ -73,25 +71,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
} }
} }
private static SqliteBlob BlobReference(ISqliteConnection conn, long rowid, bool readOnly) {
string schema = conn.HasAttachedDatabase(Schema) ? Schema : "main";
return new SqliteBlob(conn.InnerConnection, databaseName: schema, tableName: "download_blobs", columnName: "blob", rowid, readOnly);
}
public async Task AddDownload(Data.Download item, Stream? stream) { public async Task AddDownload(Data.Download item, Stream? stream) {
ulong? actualSize;
if (stream is not null) {
actualSize = (ulong) stream.Length;
if (actualSize != item.Size) {
Log.Warn("Download size differs from its metadata - metadata size: " + item.Size + " B, actual size: " + actualSize + " B, url: " + item.NormalizedUrl);
}
}
else {
actualSize = item.Size;
}
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); await conn.BeginTransactionAsync();
@ -107,7 +87,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
metadataCmd.Set(":download_url", item.DownloadUrl); metadataCmd.Set(":download_url", item.DownloadUrl);
metadataCmd.Set(":status", (int) item.Status); metadataCmd.Set(":status", (int) item.Status);
metadataCmd.Set(":type", item.Type); metadataCmd.Set(":type", item.Type);
metadataCmd.Set(":size", actualSize); metadataCmd.Set(":size", item.Size);
await metadataCmd.ExecuteNonQueryAsync(); await metadataCmd.ExecuteNonQueryAsync();
if (stream == null) { if (stream == null) {
@ -127,10 +107,10 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
); );
upsertBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl); upsertBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl);
upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, actualSize); upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, item.Size);
long rowid = await upsertBlobCmd.ExecuteLongScalarAsync(); long rowid = await upsertBlobCmd.ExecuteLongScalarAsync();
await using var blob = BlobReference(conn, rowid, readOnly: false); await using var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid);
await stream.CopyToAsync(blob); await stream.CopyToAsync(blob);
} }
@ -234,7 +214,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
rowid = reader.GetInt64(0); rowid = reader.GetInt64(0);
} }
await using (var blob = BlobReference(conn, rowid, readOnly: true)) { await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
await dataProcessor(blob); await dataProcessor(blob);
} }
@ -269,7 +249,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
rowid = reader.GetInt64(2); rowid = reader.GetInt64(2);
} }
await using (var blob = BlobReference(conn, rowid, readOnly: true)) { await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob, cancellationToken); await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob, cancellationToken);
} }

View File

@ -43,14 +43,10 @@ sealed class SqliteSettingsRepository(SqliteConnectionPool pool) : ISettingsRepo
} }
public async Task<T?> Get<T>(SettingsKey<T> key, T? defaultValue) { public async Task<T?> Get<T>(SettingsKey<T> key, T? defaultValue) {
await using var conn = await pool.Take();
return await Get(conn, key, defaultValue);
}
public static async Task<T?> Get<T>(ISqliteConnection conn, SettingsKey<T> key, T? defaultValue) {
string? value; string? value;
await using (var cmd = conn.Command("SELECT value FROM metadata WHERE key = :key")) { await using (var conn = await pool.Take()) {
await using var cmd = conn.Command("SELECT value FROM metadata WHERE key = :key");
cmd.AddAndSet(":key", SqliteType.Text, key.Key); cmd.AddAndSet(":key", SqliteType.Text, key.Key);
await using var reader = await cmd.ExecuteReaderAsync(); await using var reader = await cmd.ExecuteReaderAsync();

View File

@ -1,11 +1,9 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite.Schema; namespace DHT.Server.Database.Sqlite.Schema;
public interface ISchemaUpgradeCallbacks { public interface ISchemaUpgradeCallbacks {
Task<InitialDatabaseSettings?> GetInitialDatabaseSettings();
Task<bool> CanUpgrade(); Task<bool> CanUpgrade();
Task Start(int versionSteps, Func<IProgressReporter, Task> doUpgrade); Task Start(int versionSteps, Func<IProgressReporter, Task> doUpgrade);

View File

@ -1,46 +1,42 @@
using System.Collections.Generic; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Data.Settings;
using DHT.Server.Database.Repositories; using DHT.Server.Database.Repositories;
using DHT.Server.Database.Sqlite.Repositories; using DHT.Server.Database.Sqlite.Repositories;
using DHT.Server.Database.Sqlite.Schema; using DHT.Server.Database.Sqlite.Schema;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Logging; using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite; namespace DHT.Server.Database.Sqlite;
public sealed class SqliteDatabaseFile : IDatabaseFile { public sealed class SqliteDatabaseFile : IDatabaseFile {
private static readonly Log Log = Log.ForType<SqliteDatabaseFile>();
private const int DefaultPoolSize = 5; private const int DefaultPoolSize = 5;
public static async Task<SqliteDatabaseFile?> OpenOrCreate(string path, ISchemaUpgradeCallbacks schemaUpgradeCallbacks) { public static async Task<SqliteDatabaseFile?> OpenOrCreate(string path, ISchemaUpgradeCallbacks schemaUpgradeCallbacks) {
var connectionString = new SqliteConnectionStringFactory(path); var connectionString = new SqliteConnectionStringBuilder {
var attachedDatabaseCollector = new AttachedDatabaseCollector(path); DataSource = path,
Mode = SqliteOpenMode.ReadWriteCreate,
};
var pool = await SqliteConnectionPool.Create(connectionString, DefaultPoolSize);
bool wasOpened; bool wasOpened;
await using (var conn = await CustomSqliteConnection.OpenUnpooled(connectionString)) {
wasOpened = await new SqliteSchema(conn).Setup(attachedDatabaseCollector, schemaUpgradeCallbacks); try {
await using var conn = await pool.Take();
wasOpened = await new SqliteSchema(conn).Setup(schemaUpgradeCallbacks);
} catch (Exception) {
await pool.DisposeAsync();
throw;
} }
if (wasOpened) { if (wasOpened) {
var pool = await SqliteConnectionPool.Create(connectionString, DefaultPoolSize, attachedDatabaseCollector);
return new SqliteDatabaseFile(path, pool); return new SqliteDatabaseFile(path, pool);
} }
else { else {
await pool.DisposeAsync();
return null; return null;
} }
} }
private sealed class AttachedDatabaseCollector(string path) : ISqliteAttachedDatabaseCollector {
public async IAsyncEnumerable<AttachedDatabase> GetAttachedDatabases(ISqliteConnection conn) {
bool useSeparateFileForDownloads = await SqliteSettingsRepository.Get(conn, SettingsKey.SeparateFileForDownloads, defaultValue: false);
if (useSeparateFileForDownloads) {
yield return new AttachedDatabase(path + "_downloads", SqliteDownloadRepository.Schema);
}
}
}
public string Path { get; } public string Path { get; }
public ISettingsRepository Settings => settings; public ISettingsRepository Settings => settings;
@ -82,21 +78,6 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
public async Task Vacuum() { public async Task Vacuum() {
await using var conn = await pool.Take(); await using var conn = await pool.Take();
Perf perf = Log.Start();
await conn.ExecuteAsync("VACUUM"); await conn.ExecuteAsync("VACUUM");
perf.Step("Vacuum main schema");
await VacuumAttachedDatabase(conn, perf, SqliteDownloadRepository.Schema);
perf.End();
return;
static async Task VacuumAttachedDatabase(ISqliteConnection conn, Perf perf, string schema) {
if (conn.HasAttachedDatabase(schema)) {
await conn.ExecuteAsync("VACUUM " + schema);
perf.Step("Vacuum " + schema + " schema");
}
}
} }
} }

View File

@ -1,32 +1,28 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Data.Settings;
using DHT.Server.Database.Exceptions; using DHT.Server.Database.Exceptions;
using DHT.Server.Database.Sqlite.Repositories;
using DHT.Server.Database.Sqlite.Schema; using DHT.Server.Database.Sqlite.Schema;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Logging; using DHT.Utils.Logging;
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite; namespace DHT.Server.Database.Sqlite;
sealed class SqliteSchema(CustomSqliteConnection conn) { sealed class SqliteSchema {
internal const int Version = 10; internal const int Version = 10;
private static readonly Log Log = Log.ForType<SqliteSchema>(); private static readonly Log Log = Log.ForType<SqliteSchema>();
public async Task<bool> Setup(ISqliteAttachedDatabaseCollector attachedDatabaseCollector, ISchemaUpgradeCallbacks callbacks) { private readonly ISqliteConnection conn;
public SqliteSchema(ISqliteConnection conn) {
this.conn = conn;
}
public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)"); await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
string? dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0)); string? dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
if (dbVersionStr == null) { if (dbVersionStr == null) {
if (await callbacks.GetInitialDatabaseSettings() is {} initialSettings) {
await ApplyInitialSettings(initialSettings);
await AttachAdditionalDatabasesIfNecessary(attachedDatabaseCollector);
}
await InitializeSchemas(); await InitializeSchemas();
} }
else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) { else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
@ -41,44 +37,12 @@ sealed class SqliteSchema(CustomSqliteConnection conn) {
return false; return false;
} }
await AttachAdditionalDatabasesIfNecessary(attachedDatabaseCollector);
await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter)); await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter));
} }
return true; return true;
} }
private async Task ApplyInitialSettings(InitialDatabaseSettings initialSettings) {
await using var cmd = conn.Command("INSERT INTO metadata (key, value) VALUES (:key, :value)");
cmd.Add("key", SqliteType.Text);
cmd.Add("value", SqliteType.Text);
static async Task ApplySetting<T>(SqliteCommand cmd, SettingsKey<T> key, T value) {
cmd.Set("key", key.Key);
cmd.Set("value", key.ToString(value));
await cmd.ExecuteNonQueryAsync();
}
if (initialSettings.SeparateFileForDownloads) {
await ApplySetting(cmd, SettingsKey.SeparateFileForDownloads, value: true);
}
if (initialSettings.DownloadsAutoStart) {
await ApplySetting(cmd, SettingsKey.DownloadsAutoStart, value: true);
}
}
[SuppressMessage("ReSharper", "WithExpressionModifiesAllMembers")]
private async Task AttachAdditionalDatabasesIfNecessary(ISqliteAttachedDatabaseCollector attachedDatabaseCollector) {
await foreach (var attachedDatabase in attachedDatabaseCollector.GetAttachedDatabases(conn)) {
if (!File.Exists(attachedDatabase.Path)) {
await using CustomSqliteConnection _ = await CustomSqliteConnection.OpenUnpooled(conn.ConnectionStringFactory with { Path = attachedDatabase.Path });
}
await conn.AttachDatabase(attachedDatabase);
}
}
private async Task InitializeSchemas() { private async Task InitializeSchemas() {
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE users ( CREATE TABLE users (
@ -178,10 +142,8 @@ sealed class SqliteSchema(CustomSqliteConnection conn) {
} }
internal static async Task CreateDownloadTables(ISqliteConnection conn) { internal static async Task CreateDownloadTables(ISqliteConnection conn) {
string schema = conn.HasAttachedDatabase(SqliteDownloadRepository.Schema) ? SqliteDownloadRepository.Schema : "main"; await conn.ExecuteAsync("""
CREATE TABLE download_metadata (
await conn.ExecuteAsync($"""
CREATE TABLE {schema}.download_metadata (
normalized_url TEXT NOT NULL PRIMARY KEY, normalized_url TEXT NOT NULL PRIMARY KEY,
download_url TEXT NOT NULL, download_url TEXT NOT NULL,
status INTEGER NOT NULL, status INTEGER NOT NULL,
@ -190,8 +152,8 @@ sealed class SqliteSchema(CustomSqliteConnection conn) {
) )
"""); """);
await conn.ExecuteAsync($""" await conn.ExecuteAsync("""
CREATE TABLE {schema}.download_blobs ( CREATE TABLE download_blobs (
normalized_url TEXT NOT NULL PRIMARY KEY, normalized_url TEXT NOT NULL PRIMARY KEY,
blob BLOB NOT NULL, blob BLOB NOT NULL,
FOREIGN KEY (normalized_url) REFERENCES download_metadata (normalized_url) ON UPDATE CASCADE ON DELETE CASCADE FOREIGN KEY (normalized_url) REFERENCES download_metadata (normalized_url) ON UPDATE CASCADE ON DELETE CASCADE

View File

@ -1,3 +0,0 @@
namespace DHT.Server.Database.Sqlite.Utils;
sealed record AttachedDatabase(string Path, string Schema);

View File

@ -1,91 +0,0 @@
using System;
using System.Collections.Immutable;
using System.Data.Common;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils;
class CustomSqliteConnection : ISqliteConnection {
internal static async Task<CustomSqliteConnection> OpenUnpooled(SqliteConnectionStringFactory connectionStringFactory) {
SqliteConnection conn = new SqliteConnection(connectionStringFactory.Create());
await conn.OpenAsync();
CustomSqliteConnection custom = new CustomSqliteConnection(connectionStringFactory, conn);
await custom.Setup(CancellationToken.None);
return custom;
}
internal SqliteConnectionStringFactory ConnectionStringFactory { get; }
public SqliteConnection InnerConnection { get; }
private ImmutableHashSet<string> attachedDatabases = [];
private DbTransaction? activeTransaction;
protected CustomSqliteConnection(SqliteConnectionStringFactory connectionStringFactory, SqliteConnection innerConnection) {
this.ConnectionStringFactory = connectionStringFactory;
this.InnerConnection = innerConnection;
}
internal async Task Setup(CancellationToken cancellationToken) {
await this.ExecuteAsync("PRAGMA journal_mode=WAL", cancellationToken);
await this.ExecuteAsync("PRAGMA foreign_keys=ON", cancellationToken);
}
[SuppressMessage("ReSharper", "WithExpressionModifiesAllMembers")]
internal async Task AttachDatabase(AttachedDatabase attachedDatabase) {
await this.ExecuteAsync("ATTACH DATABASE '" + attachedDatabase.Path.Replace("'", "''") + "' AS " + attachedDatabase.Schema);
ImmutableInterlocked.Update(ref attachedDatabases, static (set, schema) => set.Add(schema), attachedDatabase.Schema);
}
public bool HasAttachedDatabase(string schema) {
return attachedDatabases.Contains(schema);
}
public async Task BeginTransactionAsync() {
if (activeTransaction != null) {
throw new InvalidOperationException("A transaction is already active.");
}
activeTransaction = await InnerConnection.BeginTransactionAsync();
}
public async Task CommitTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to commit.");
}
await activeTransaction.CommitAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public async Task RollbackTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to rollback.");
}
await activeTransaction.RollbackAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public void AssignActiveTransaction(SqliteCommand command) {
command.Transaction = (SqliteTransaction?) activeTransaction;
}
public async ValueTask DisposeAsync() {
if (activeTransaction != null) {
await RollbackTransactionAsync();
}
await DisposeConnection();
}
private protected virtual async ValueTask DisposeConnection() {
await InnerConnection.DisposeAsync();
}
}

View File

@ -1,7 +0,0 @@
using System.Collections.Generic;
namespace DHT.Server.Database.Sqlite.Utils;
interface ISqliteAttachedDatabaseCollector {
IAsyncEnumerable<AttachedDatabase> GetAttachedDatabases(ISqliteConnection conn);
}

View File

@ -7,8 +7,6 @@ namespace DHT.Server.Database.Sqlite.Utils;
interface ISqliteConnection : IAsyncDisposable { interface ISqliteConnection : IAsyncDisposable {
SqliteConnection InnerConnection { get; } SqliteConnection InnerConnection { get; }
bool HasAttachedDatabase(string schema);
Task BeginTransactionAsync(); Task BeginTransactionAsync();
Task CommitTransactionAsync(); Task CommitTransactionAsync();
Task RollbackTransactionAsync(); Task RollbackTransactionAsync();

View File

@ -1,3 +0,0 @@
namespace DHT.Server.Database.Sqlite.Utils;
public sealed record InitialDatabaseSettings(bool SeparateFileForDownloads, bool DownloadsAutoStart);

View File

@ -1,6 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Data.Common;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Utils.Collections; using DHT.Utils.Collections;
@ -9,13 +9,16 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils; namespace DHT.Server.Database.Sqlite.Utils;
sealed class SqliteConnectionPool : IAsyncDisposable { sealed class SqliteConnectionPool : IAsyncDisposable {
public static async Task<SqliteConnectionPool> Create(SqliteConnectionStringFactory connectionStringFactory, int poolSize, ISqliteAttachedDatabaseCollector attachedDatabaseCollector) { public static async Task<SqliteConnectionPool> Create(SqliteConnectionStringBuilder connectionStringBuilder, int poolSize) {
SqliteConnectionPool pool = new SqliteConnectionPool(poolSize, attachedDatabaseCollector); var pool = new SqliteConnectionPool(poolSize);
await pool.InitializePooledConnections(connectionStringFactory); await pool.InitializePooledConnections(connectionStringBuilder);
return pool; return pool;
} }
private readonly ISqliteAttachedDatabaseCollector attachedDatabaseCollector; private static string GetConnectionString(SqliteConnectionStringBuilder connectionStringBuilder) {
connectionStringBuilder.Pooling = false;
return connectionStringBuilder.ToString();
}
private readonly int poolSize; private readonly int poolSize;
private readonly List<PooledConnection> all; private readonly List<PooledConnection> all;
@ -24,35 +27,24 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
private readonly CancellationTokenSource disposalTokenSource = new (); private readonly CancellationTokenSource disposalTokenSource = new ();
private readonly CancellationToken disposalToken; private readonly CancellationToken disposalToken;
private SqliteConnectionPool(int poolSize, ISqliteAttachedDatabaseCollector attachedDatabaseCollector) { private SqliteConnectionPool(int poolSize) {
this.attachedDatabaseCollector = attachedDatabaseCollector;
this.poolSize = poolSize; this.poolSize = poolSize;
this.all = new List<PooledConnection>(poolSize); this.all = new List<PooledConnection>(poolSize);
this.free = new ConcurrentPool<PooledConnection>(poolSize); this.free = new ConcurrentPool<PooledConnection>(poolSize);
this.disposalToken = disposalTokenSource.Token; this.disposalToken = disposalTokenSource.Token;
} }
private async Task InitializePooledConnections(SqliteConnectionStringFactory connectionStringFactory) { private async Task InitializePooledConnections(SqliteConnectionStringBuilder connectionStringBuilder) {
string connectionString = connectionStringFactory.Create(); string connectionString = GetConnectionString(connectionStringBuilder);
List<AttachedDatabase> attachedDatabases = [];
for (int i = 0; i < poolSize; i++) { for (int i = 0; i < poolSize; i++) {
SqliteConnection innerConnection = new SqliteConnection(connectionString); var conn = new SqliteConnection(connectionString);
await innerConnection.OpenAsync(disposalToken); conn.Open();
PooledConnection pooledConnection = new PooledConnection(this, connectionStringFactory, innerConnection); var pooledConnection = new PooledConnection(this, conn);
await pooledConnection.Setup(disposalToken);
if (i == 0) { await pooledConnection.ExecuteAsync("PRAGMA journal_mode=WAL", disposalToken);
attachedDatabases = await attachedDatabaseCollector.GetAttachedDatabases(pooledConnection).ToListAsync(disposalToken); await pooledConnection.ExecuteAsync("PRAGMA foreign_keys=ON", disposalToken);
}
foreach (var attachedDatabase in attachedDatabases) {
await pooledConnection.AttachDatabase(attachedDatabase);
}
all.Add(pooledConnection); all.Add(pooledConnection);
await free.Push(pooledConnection, disposalToken); await free.Push(pooledConnection, disposalToken);
@ -82,8 +74,48 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
disposalTokenSource.Dispose(); disposalTokenSource.Dispose();
} }
private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnectionStringFactory connectionStringFactory, SqliteConnection innerConnection) : CustomSqliteConnection(connectionStringFactory, innerConnection) { private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) : ISqliteConnection {
private protected override async ValueTask DisposeConnection() { public SqliteConnection InnerConnection { get; } = conn;
private DbTransaction? activeTransaction;
public async Task BeginTransactionAsync() {
if (activeTransaction != null) {
throw new InvalidOperationException("A transaction is already active.");
}
activeTransaction = await InnerConnection.BeginTransactionAsync();
}
public async Task CommitTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to commit.");
}
await activeTransaction.CommitAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public async Task RollbackTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to rollback.");
}
await activeTransaction.RollbackAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public void AssignActiveTransaction(SqliteCommand command) {
command.Transaction = (SqliteTransaction?) activeTransaction;
}
public async ValueTask DisposeAsync() {
if (activeTransaction != null) {
await RollbackTransactionAsync();
}
await pool.Return(this); await pool.Return(this);
} }
} }

View File

@ -1,15 +0,0 @@
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils;
sealed record SqliteConnectionStringFactory(string Path) {
public string Create() {
var builder = new SqliteConnectionStringBuilder {
DataSource = Path,
Mode = SqliteOpenMode.ReadWriteCreate,
Pooling = false,
};
return builder.ToString();
}
}

View File

@ -55,10 +55,7 @@ public sealed class DelayedThrottledTask<T> : IDisposable {
} }
public void Dispose() { public void Dispose() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
taskChannel.Writer.Complete(); taskChannel.Writer.Complete();
cancellationTokenSource.Cancel();
} }
} }