Add option to use a separate database file for downloads when creating a new database

app/.idea/.idea.DiscordHistoryTracker/.idea/avalonia.xml

@@ -13,6 +13,7 @@
         <entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
+        <entry key="Desktop/Main/Dialogs/NewDatabaseSettingsDialog.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
         <entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />
@@ -25,4 +26,4 @@
       </map>
     </option>
   </component>
-</project>
+</project>

app/Desktop/App.axaml

@@ -54,6 +54,22 @@
             <Setter Property="HorizontalAlignment" Value="Stretch" />
         </Style>
 
+        <Style Selector="Panel.buttons">
+            <Setter Property="Margin" Value="0 20 0 0" />
+        </Style>
+        <Style Selector="Panel.buttons > WrapPanel.right">
+            <Setter Property="HorizontalAlignment" Value="Right" />
+        </Style>
+        <Style Selector="Panel.buttons Button">
+            <Setter Property="MinWidth" Value="80" />
+        </Style>
+        <Style Selector="Panel.buttons > WrapPanel.left Button">
+            <Setter Property="Margin" Value="0 0 8 0" />
+        </Style>
+        <Style Selector="Panel.buttons > WrapPanel.right Button">
+            <Setter Property="Margin" Value="8 0 0 0" />
+        </Style>
+
     </Application.Styles>
 
     <Application.Resources>

app/Desktop/Dialogs/CheckBox/CheckBoxDialog.axaml

@@ -15,24 +15,6 @@
         <namespace:CheckBoxDialogModel />
     </Window.DataContext>
 
-    <Window.Styles>
-        <Style Selector="Panel.buttons">
-            <Setter Property="Margin" Value="0 20 0 0" />
-        </Style>
-        <Style Selector="Panel.buttons > WrapPanel.right">
-            <Setter Property="HorizontalAlignment" Value="Right" />
-        </Style>
-        <Style Selector="Panel.buttons Button">
-            <Setter Property="MinWidth" Value="80" />
-        </Style>
-        <Style Selector="Panel.buttons > WrapPanel.left Button">
-            <Setter Property="Margin" Value="0 0 8 0" />
-        </Style>
-        <Style Selector="Panel.buttons > WrapPanel.right Button">
-            <Setter Property="Margin" Value="8 0 0 0" />
-        </Style>
-    </Window.Styles>
-
     <StackPanel Margin="20">
         <ScrollViewer MaxHeight="400">
             <ItemsRepeater ItemsSource="{Binding Items}">

app/Desktop/Dialogs/Message/MessageDialog.axaml

@@ -20,24 +20,18 @@
             <Setter Property="FontSize" Value="15" />
             <Setter Property="TextWrapping" Value="Wrap" />
         </Style>
-        <Style Selector="WrapPanel">
-            <Setter Property="HorizontalAlignment" Value="Right" />
-            <Setter Property="Margin" Value="0 8 0 0" />
-        </Style>
-        <Style Selector="Button">
-            <Setter Property="Margin" Value="8 0 0 0" />
-            <Setter Property="MinWidth" Value="80" />
-        </Style>
     </Window.Styles>
 
     <StackPanel Margin="20">
         <TextBlock Text="{Binding Message}" />
-        <WrapPanel>
-            <Button Click="ClickOk" IsVisible="{Binding IsOkVisible}">OK</Button>
-            <Button Click="ClickYes" IsVisible="{Binding IsYesVisible}">Yes</Button>
-            <Button Click="ClickNo" IsVisible="{Binding IsNoVisible}">No</Button>
-            <Button Click="ClickCancel" IsVisible="{Binding IsCancelVisible}">Cancel</Button>
-        </WrapPanel>
+        <Panel Classes="buttons">
+            <WrapPanel Classes="right">
+                <Button Click="ClickOk" IsVisible="{Binding IsOkVisible}">OK</Button>
+                <Button Click="ClickYes" IsVisible="{Binding IsYesVisible}">Yes</Button>
+                <Button Click="ClickNo" IsVisible="{Binding IsNoVisible}">No</Button>
+                <Button Click="ClickCancel" IsVisible="{Binding IsCancelVisible}">Cancel</Button>
+            </WrapPanel>
+        </Panel>
     </StackPanel>
 
 </Window>

app/Desktop/Dialogs/TextBox/TextBoxDialog.axaml

@@ -15,19 +15,6 @@
         <namespace:TextBoxDialogModel />
     </Window.DataContext>
 
-    <Window.Styles>
-        <Style Selector="Panel.buttons">
-            <Setter Property="Margin" Value="0 20 0 0" />
-        </Style>
-        <Style Selector="Panel.buttons > WrapPanel.right">
-            <Setter Property="HorizontalAlignment" Value="Right" />
-        </Style>
-        <Style Selector="Panel.buttons Button">
-            <Setter Property="MinWidth" Value="80" />
-            <Setter Property="Margin" Value="8 0 0 0" />
-        </Style>
-    </Window.Styles>
-
     <StackPanel Margin="20">
         <ScrollViewer MaxHeight="400">
             <StackPanel Spacing="10">

app/Desktop/Main/Dialogs/NewDatabaseSettingsDialog.axaml

@@ -0,0 +1,57 @@
+<Window xmlns="https://github.com/avaloniaui"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:namespace="clr-namespace:DHT.Desktop.Main.Dialogs"
+        mc:Ignorable="d" d:DesignWidth="450"
+        x:Class="DHT.Desktop.Main.Dialogs.NewDatabaseSettingsDialog"
+        x:DataType="namespace:NewDatabaseSettingsDialogModel"
+        Title="New Database Settings"
+        Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
+        Closing="OnClosing"
+        Width="450" SizeToContent="Height" CanResize="False"
+        WindowStartupLocation="CenterOwner">
+
+    <Window.DataContext>
+        <namespace:NewDatabaseSettingsDialogModel />
+    </Window.DataContext>
+
+    <Window.Styles>
+        <Style Selector="Label > StackPanel">
+            <Setter Property="Margin" Value="0 4 0 0" />
+            <Setter Property="Orientation" Value="Vertical" />
+            <Setter Property="Spacing" Value="5" />
+        </Style>
+        <Style Selector="TextBlock.dim">
+            <Setter Property="Foreground" Value="DimGray" />
+            <Setter Property="FontSize" Value="13" />
+        </Style>
+    </Window.Styles>
+
+    <StackPanel Orientation="Vertical" Margin="20">
+        <StackPanel Orientation="Vertical" Spacing="15">
+            <CheckBox IsChecked="{Binding SeparateFileForDownloads}">
+                <Label>
+                    <StackPanel>
+                        <TextBlock>Use a Separate Database File for Downloads</TextBlock>
+                        <TextBlock Classes="dim">Downloaded files will be stored separately from messages. This cannot be changed later.</TextBlock>
+                    </StackPanel>
+                </Label>
+            </CheckBox>
+            <CheckBox IsChecked="{Binding DownloadsAutoStart}">
+                <Label>
+                    <StackPanel>
+                        <TextBlock>Download Files Automatically</TextBlock>
+                        <TextBlock Classes="dim">Enables automatic downloads of files hosted on Discord. This can be changed later in the Downloads tab.</TextBlock>
+                    </StackPanel>
+                </Label>
+            </CheckBox>
+        </StackPanel>
+        <Panel Classes="buttons">
+            <WrapPanel Classes="right">
+                <Button Click="ClickOk">OK</Button>
+            </WrapPanel>
+        </Panel>
+    </StackPanel>
+
+</Window>

app/Desktop/Main/Dialogs/NewDatabaseSettingsDialog.axaml.cs

@@ -0,0 +1,22 @@
+using System.Diagnostics.CodeAnalysis;
+using Avalonia.Controls;
+using Avalonia.Interactivity;
+
+namespace DHT.Desktop.Main.Dialogs;
+
+[SuppressMessage("ReSharper", "MemberCanBeInternal")]
+public sealed partial class NewDatabaseSettingsDialog : Window {
+	public NewDatabaseSettingsDialog() {
+		InitializeComponent();
+	}
+	
+	private void OnClosing(object? sender, WindowClosingEventArgs e) {
+		if (!e.IsProgrammatic) {
+			e.Cancel = true;
+		}
+	}
+	
+	private void ClickOk(object? sender, RoutedEventArgs e) {
+		Close();
+	}
+}

app/Desktop/Main/Dialogs/NewDatabaseSettingsDialogModel.cs

@@ -0,0 +1,11 @@
+using CommunityToolkit.Mvvm.ComponentModel;
+
+namespace DHT.Desktop.Main.Dialogs;
+
+sealed partial class NewDatabaseSettingsDialogModel : ObservableObject {
+	[ObservableProperty]
+	private bool separateFileForDownloads = true;
+	
+	[ObservableProperty]
+	private bool downloadsAutoStart = true;
+}

app/Desktop/Main/Pages/AdvancedPage.axaml

@@ -16,11 +16,5 @@
         <Expander Header="Internal Server Configuration" IsExpanded="True">
             <controls:ServerConfigurationPanel DataContext="{Binding ServerConfigurationModel}" />
         </Expander>
-        <Expander Header="Database Tools" IsExpanded="True">
-            <StackPanel Orientation="Vertical" Spacing="10">
-                <TextBlock TextWrapping="Wrap">Recreates the database to remove any traces of deleted data, which frees up space and prevents forensic recovery.</TextBlock>
-                <Button Command="{Binding VacuumDatabase}">Vacuum Database</Button>
-            </StackPanel>
-        </Expander>
     </StackPanel>
 </UserControl>

app/Desktop/Main/Pages/AdvancedPageModel.cs

@@ -1,8 +1,5 @@
 using System;
-using System.Threading.Tasks;
 using Avalonia.Controls;
-using DHT.Desktop.Dialogs.Message;
-using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
 using DHT.Server;
 
@@ -11,26 +8,14 @@ namespace DHT.Desktop.Main.Pages;
 sealed class AdvancedPageModel : IDisposable {
 	public ServerConfigurationPanelModel ServerConfigurationModel { get; }
 	
-	private readonly Window window;
-	private readonly State state;
-	
 	[Obsolete("Designer")]
 	public AdvancedPageModel() : this(null!, State.Dummy) {}
 	
 	public AdvancedPageModel(Window window, State state) {
-		this.window = window;
-		this.state = state;
-		
 		ServerConfigurationModel = new ServerConfigurationPanelModel(window, state);
 	}
 	
 	public void Dispose() {
 		ServerConfigurationModel.Dispose();
 	}
-	
-	public async Task VacuumDatabase() {
-		const string Title = "Vacuum Database";
-		await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => state.Db.Vacuum());
-		await Dialog.ShowOk(window, Title, "Done.");
-	}
 }

app/Desktop/Main/Pages/DatabasePage.axaml

@@ -27,6 +27,12 @@
             <Button Command="{Binding MergeWithDatabase}">Merge with Database(s)...</Button>
             <Button Command="{Binding ImportLegacyArchive}">Import Legacy Archive(s)...</Button>
         </WrapPanel>
+        <Expander Header="Advanced Tools" Margin="0 15 0 0">
+            <StackPanel Orientation="Vertical" Spacing="10">
+                <TextBlock TextWrapping="Wrap">Recreate the database to free up space after deleting data.</TextBlock>
+                <Button Command="{Binding VacuumDatabase}">Vacuum Database</Button>
+            </StackPanel>
+        </Expander>
     </StackPanel>
 
 </UserControl>

app/Desktop/Main/Pages/DatabasePageModel.cs

@@ -19,6 +19,7 @@ using DHT.Server.Data;
 using DHT.Server.Database;
 using DHT.Server.Database.Import;
 using DHT.Server.Database.Sqlite.Schema;
+using DHT.Server.Database.Sqlite.Utils;
 using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Main.Pages;
@@ -54,11 +55,11 @@ sealed class DatabasePageModel {
 				break;
 			
 			case PlatformID.Unix:
-				Process.Start("xdg-open", new string[] { folder });
+				Process.Start("xdg-open", [ folder ]);
 				break;
 			
 			case PlatformID.MacOSX:
-				Process.Start("open", new string[] { folder });
+				Process.Start("open", [ folder ]);
 				break;
 			
 			default:
@@ -97,14 +98,11 @@ sealed class DatabasePageModel {
 		});
 	}
 	
-	private sealed class SchemaUpgradeCallbacks : ISchemaUpgradeCallbacks {
-		private readonly ProgressDialog dialog;
-		private readonly int total;
+	private sealed class SchemaUpgradeCallbacks(ProgressDialog dialog, int total) : ISchemaUpgradeCallbacks {
 		private bool? decision;
 		
-		public SchemaUpgradeCallbacks(ProgressDialog dialog, int total) {
-			this.total = total;
-			this.dialog = dialog;
+		public Task<InitialDatabaseSettings?> GetInitialDatabaseSettings() {
+			return Task.FromResult<InitialDatabaseSettings?>(null);
 		}
 		
 		public async Task<bool> CanUpgrade() {
@@ -263,4 +261,10 @@ sealed class DatabasePageModel {
 		
 		return message.ToString();
 	}
+	
+	public async Task VacuumDatabase() {
+		const string Title = "Vacuum Database";
+		await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => Db.Vacuum());
+		await Dialog.ShowOk(window, Title, "Done.");
+	}
 }

app/Desktop/Main/Screens/WelcomeScreenModel.cs

@@ -9,9 +9,10 @@ using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
-using DHT.Server.Data.Settings;
+using DHT.Desktop.Main.Dialogs;
 using DHT.Server.Database;
 using DHT.Server.Database.Sqlite.Schema;
+using DHT.Server.Database.Sqlite.Utils;
 using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Main.Screens;
@@ -52,21 +53,22 @@ sealed partial class WelcomeScreenModel : ObservableObject {
 	public async Task OpenOrCreateDatabaseFromPath(string path) {
 		dbFilePath = path;
 		
-		bool isNew = !File.Exists(path);
-		
 		IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, window, new SchemaUpgradeCallbacks(window));
-		if (db == null) {
-			return;
+		if (db != null) {
+			DatabaseSelected?.Invoke(this, db);
 		}
-		
-		if (isNew && await Dialog.ShowYesNo(window, "Automatic Downloads", "Do you want to automatically download files hosted on Discord? You can change this later in the Downloads tab.") == DialogResult.YesNo.Yes) {
-			await db.Settings.Set(SettingsKey.DownloadsAutoStart, value: true);
-		}
-		
-		DatabaseSelected?.Invoke(this, db);
 	}
 	
 	private sealed class SchemaUpgradeCallbacks(Window window) : ISchemaUpgradeCallbacks {
+		public async Task<InitialDatabaseSettings?> GetInitialDatabaseSettings() {
+			var model = new NewDatabaseSettingsDialogModel();
+			
+			var dialog = new NewDatabaseSettingsDialog { DataContext = model };
+			await dialog.ShowDialog(window);
+			
+			return new InitialDatabaseSettings(model.SeparateFileForDownloads, model.DownloadsAutoStart);
+		}
+		
 		public async Task<bool> CanUpgrade() {
 			return DialogResult.YesNo.Yes == await DatabaseGui.ShowCanUpgradeDatabaseDialog(window);
 		}

app/Server/Data/Settings/SettingsKey.cs

@@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis;
 namespace DHT.Server.Data.Settings;
 
 public static class SettingsKey {
+	public static Bool SeparateFileForDownloads { get; } = new ("separate_file_for_downloads");
 	public static Bool DownloadsAutoStart { get; } = new ("downloads_auto_start");
 	public static Bool DownloadsLimitSize { get; } = new ("downloads_limit_size");
 	public static UnsignedLong DownloadsMaximumSize { get; } = new ("downloads_maximum_size");

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -18,6 +18,8 @@ namespace DHT.Server.Database.Sqlite.Repositories;
 sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository {
 	private static readonly Log Log = Log.ForType<SqliteDownloadRepository>();
 	
+	public const string Schema = "downloads";
+	
 	internal sealed class NewDownloadCollector : IAsyncDisposable {
 		private readonly SqliteDownloadRepository repository;
 		private bool hasChanged = false;
@@ -71,6 +73,11 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		}
 	}
 	
+	private static SqliteBlob BlobReference(ISqliteConnection conn, long rowid, bool readOnly) {
+		string schema = conn.HasAttachedDatabase(Schema) ? Schema : "main";
+		return new SqliteBlob(conn.InnerConnection, databaseName: schema, tableName: "download_blobs", columnName: "blob", rowid, readOnly);
+	}
+	
 	public async Task AddDownload(Data.Download item, Stream? stream) {
 		await using (var conn = await pool.Take()) {
 			await conn.BeginTransactionAsync();
@@ -110,7 +117,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 				upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, item.Size);
 				long rowid = await upsertBlobCmd.ExecuteLongScalarAsync();
 				
-				await using var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid);
+				await using var blob = BlobReference(conn, rowid, readOnly: false);
 				await stream.CopyToAsync(blob);
 			}
 			
@@ -214,7 +221,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 			rowid = reader.GetInt64(0);
 		}
 		
-		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
+		await using (var blob = BlobReference(conn, rowid, readOnly: true)) {
 			await dataProcessor(blob);
 		}
 		
@@ -249,7 +256,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 			rowid = reader.GetInt64(2);
 		}
 		
-		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
+		await using (var blob = BlobReference(conn, rowid, readOnly: true)) {
 			await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob, cancellationToken);
 		}
 		

app/Server/Database/Sqlite/Repositories/SqliteSettingsRepository.cs

@@ -43,10 +43,14 @@ sealed class SqliteSettingsRepository(SqliteConnectionPool pool) : ISettingsRepo
 	}
 	
 	public async Task<T?> Get<T>(SettingsKey<T> key, T? defaultValue) {
+		await using var conn = await pool.Take();
+		return await Get(conn, key, defaultValue);
+	}
+	
+	public static async Task<T?> Get<T>(ISqliteConnection conn, SettingsKey<T> key, T? defaultValue) {
 		string? value;
 		
-		await using (var conn = await pool.Take()) {
-			await using var cmd = conn.Command("SELECT value FROM metadata WHERE key = :key");
+		await using (var cmd = conn.Command("SELECT value FROM metadata WHERE key = :key")) {
 			cmd.AddAndSet(":key", SqliteType.Text, key.Key);
 			
 			await using var reader = await cmd.ExecuteReaderAsync();

app/Server/Database/Sqlite/Schema/ISchemaUpgradeCallbacks.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Threading.Tasks;
+using DHT.Server.Database.Sqlite.Utils;
 
 namespace DHT.Server.Database.Sqlite.Schema;
 
 public interface ISchemaUpgradeCallbacks {
+	Task<InitialDatabaseSettings?> GetInitialDatabaseSettings();
 	Task<bool> CanUpgrade();
 	Task Start(int versionSteps, Func<IProgressReporter, Task> doUpgrade);
 	

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -1,10 +1,10 @@
-using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
+using DHT.Server.Data.Settings;
 using DHT.Server.Database.Repositories;
 using DHT.Server.Database.Sqlite.Repositories;
 using DHT.Server.Database.Sqlite.Schema;
 using DHT.Server.Database.Sqlite.Utils;
-using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite;
 
@@ -12,31 +12,32 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 	private const int DefaultPoolSize = 5;
 	
 	public static async Task<SqliteDatabaseFile?> OpenOrCreate(string path, ISchemaUpgradeCallbacks schemaUpgradeCallbacks) {
-		var connectionString = new SqliteConnectionStringBuilder {
-			DataSource = path,
-			Mode = SqliteOpenMode.ReadWriteCreate,
-		};
+		var connectionString = new SqliteConnectionStringFactory(path);
+		var attachedDatabaseCollector = new AttachedDatabaseCollector(path);
 		
-		var pool = await SqliteConnectionPool.Create(connectionString, DefaultPoolSize);
 		bool wasOpened;
-		
-		try {
-			await using var conn = await pool.Take();
-			wasOpened = await new SqliteSchema(conn).Setup(schemaUpgradeCallbacks);
-		} catch (Exception) {
-			await pool.DisposeAsync();
-			throw;
+		await using (var conn = await CustomSqliteConnection.OpenUnpooled(connectionString)) {
+			wasOpened = await new SqliteSchema(conn).Setup(attachedDatabaseCollector, schemaUpgradeCallbacks);
 		}
 		
 		if (wasOpened) {
+			var pool = await SqliteConnectionPool.Create(connectionString, DefaultPoolSize, attachedDatabaseCollector);
 			return new SqliteDatabaseFile(path, pool);
 		}
 		else {
-			await pool.DisposeAsync();
 			return null;
 		}
 	}
 	
+	private sealed class AttachedDatabaseCollector(string path) : ISqliteAttachedDatabaseCollector {
+		public async IAsyncEnumerable<AttachedDatabase> GetAttachedDatabases(ISqliteConnection conn) {
+			bool useSeparateFileForDownloads = await SqliteSettingsRepository.Get(conn, SettingsKey.SeparateFileForDownloads, defaultValue: false);
+			if (useSeparateFileForDownloads) {
+				yield return new AttachedDatabase(path + "_downloads", SqliteDownloadRepository.Schema);
+			}
+		}
+	}
+	
 	public string Path { get; }
 	
 	public ISettingsRepository Settings => settings;
@@ -78,6 +79,11 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
 	
 	public async Task Vacuum() {
 		await using var conn = await pool.Take();
+		
 		await conn.ExecuteAsync("VACUUM");
+		
+		if (conn.HasAttachedDatabase(SqliteDownloadRepository.Schema)) {
+			await conn.ExecuteAsync("VACUUM " + SqliteDownloadRepository.Schema);
+		}
 	}
 }

app/Server/Database/Sqlite/SqliteSchema.cs

@@ -1,28 +1,32 @@
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Threading.Tasks;
+using DHT.Server.Data.Settings;
 using DHT.Server.Database.Exceptions;
+using DHT.Server.Database.Sqlite.Repositories;
 using DHT.Server.Database.Sqlite.Schema;
 using DHT.Server.Database.Sqlite.Utils;
 using DHT.Utils.Logging;
+using Microsoft.Data.Sqlite;
 
 namespace DHT.Server.Database.Sqlite;
 
-sealed class SqliteSchema {
+sealed class SqliteSchema(CustomSqliteConnection conn) {
 	internal const int Version = 10;
 	
 	private static readonly Log Log = Log.ForType<SqliteSchema>();
 	
-	private readonly ISqliteConnection conn;
-	
-	public SqliteSchema(ISqliteConnection conn) {
-		this.conn = conn;
-	}
-	
-	public async Task<bool> Setup(ISchemaUpgradeCallbacks callbacks) {
+	public async Task<bool> Setup(ISqliteAttachedDatabaseCollector attachedDatabaseCollector, ISchemaUpgradeCallbacks callbacks) {
 		await conn.ExecuteAsync("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)");
 		
 		string? dbVersionStr = await conn.ExecuteReaderAsync("SELECT value FROM metadata WHERE key = 'version'", static reader => reader?.GetString(0));
 		if (dbVersionStr == null) {
+			if (await callbacks.GetInitialDatabaseSettings() is {} initialSettings) {
+				await ApplyInitialSettings(initialSettings);
+				await AttachAdditionalDatabasesIfNecessary(attachedDatabaseCollector);
+			}
+			
 			await InitializeSchemas();
 		}
 		else if (!int.TryParse(dbVersionStr, out int dbVersion) || dbVersion < 1) {
@@ -37,12 +41,44 @@ sealed class SqliteSchema {
 				return false;
 			}
 			
+			await AttachAdditionalDatabasesIfNecessary(attachedDatabaseCollector);
 			await callbacks.Start(Version - dbVersion, async reporter => await UpgradeSchemas(dbVersion, reporter));
 		}
 		
 		return true;
 	}
 	
+	private async Task ApplyInitialSettings(InitialDatabaseSettings initialSettings) {
+		await using var cmd = conn.Command("INSERT INTO metadata (key, value) VALUES (:key, :value)");
+		cmd.Add("key", SqliteType.Text);
+		cmd.Add("value", SqliteType.Text);
+		
+		static async Task ApplySetting<T>(SqliteCommand cmd, SettingsKey<T> key, T value) {
+			cmd.Set("key", key.Key);
+			cmd.Set("value", key.ToString(value));
+			await cmd.ExecuteNonQueryAsync();
+		}
+		
+		if (initialSettings.SeparateFileForDownloads) {
+			await ApplySetting(cmd, SettingsKey.SeparateFileForDownloads, value: true);
+		}
+		
+		if (initialSettings.DownloadsAutoStart) {
+			await ApplySetting(cmd, SettingsKey.DownloadsAutoStart, value: true);
+		}
+	}
+	
+	[SuppressMessage("ReSharper", "WithExpressionModifiesAllMembers")]
+	private async Task AttachAdditionalDatabasesIfNecessary(ISqliteAttachedDatabaseCollector attachedDatabaseCollector) {
+		await foreach (var attachedDatabase in attachedDatabaseCollector.GetAttachedDatabases(conn)) {
+			if (!File.Exists(attachedDatabase.Path)) {
+				await using CustomSqliteConnection _ = await CustomSqliteConnection.OpenUnpooled(conn.ConnectionStringFactory with { Path = attachedDatabase.Path });
+			}
+			
+			await conn.AttachDatabase(attachedDatabase);
+		}
+	}
+	
 	private async Task InitializeSchemas() {
 		await conn.ExecuteAsync("""
 		                        CREATE TABLE users (
@@ -142,23 +178,25 @@ sealed class SqliteSchema {
 	}
 	
 	internal static async Task CreateDownloadTables(ISqliteConnection conn) {
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE download_metadata (
-		                        	normalized_url TEXT NOT NULL PRIMARY KEY,
-		                        	download_url   TEXT NOT NULL,
-		                        	status         INTEGER NOT NULL,
-		                        	type           TEXT,
-		                        	size           INTEGER
-		                        )
-		                        """);
+		string schema = conn.HasAttachedDatabase(SqliteDownloadRepository.Schema) ? SqliteDownloadRepository.Schema : "main";
 		
-		await conn.ExecuteAsync("""
-		                        CREATE TABLE download_blobs (
-		                        	normalized_url TEXT NOT NULL PRIMARY KEY,
-		                        	blob           BLOB NOT NULL,
-		                        	FOREIGN KEY (normalized_url) REFERENCES download_metadata (normalized_url) ON UPDATE CASCADE ON DELETE CASCADE
-		                        )
-		                        """);
+		await conn.ExecuteAsync($"""
+		                         CREATE TABLE {schema}.download_metadata (
+		                         	normalized_url TEXT NOT NULL PRIMARY KEY,
+		                         	download_url   TEXT NOT NULL,
+		                         	status         INTEGER NOT NULL,
+		                         	type           TEXT,
+		                         	size           INTEGER
+		                         )
+		                         """);
+		
+		await conn.ExecuteAsync($"""
+		                         CREATE TABLE {schema}.download_blobs (
+		                         	normalized_url TEXT NOT NULL PRIMARY KEY,
+		                         	blob           BLOB NOT NULL,
+		                         	FOREIGN KEY (normalized_url) REFERENCES download_metadata (normalized_url) ON UPDATE CASCADE ON DELETE CASCADE
+		                         )
+		                         """);
 	}
 	
 	internal static async Task CreateMessageAttachmentsTable(ISqliteConnection conn) {

app/Server/Database/Sqlite/Utils/AttachedDatabase.cs

@@ -0,0 +1,3 @@
+namespace DHT.Server.Database.Sqlite.Utils;
+
+sealed record AttachedDatabase(string Path, string Schema);

app/Server/Database/Sqlite/Utils/CustomSqliteConnection.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Immutable;
+using System.Data.Common;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Data.Sqlite;
+
+namespace DHT.Server.Database.Sqlite.Utils;
+
+class CustomSqliteConnection : ISqliteConnection {
+	internal static async Task<CustomSqliteConnection> OpenUnpooled(SqliteConnectionStringFactory connectionStringFactory) {
+		SqliteConnection conn = new SqliteConnection(connectionStringFactory.Create());
+		await conn.OpenAsync();
+		
+		CustomSqliteConnection custom = new CustomSqliteConnection(connectionStringFactory, conn);
+		await custom.Setup(CancellationToken.None);
+		
+		return custom;
+	}
+	
+	internal SqliteConnectionStringFactory ConnectionStringFactory { get; }
+	public SqliteConnection InnerConnection { get; }
+	
+	private ImmutableHashSet<string> attachedDatabases = [];
+	private DbTransaction? activeTransaction;
+	
+	protected CustomSqliteConnection(SqliteConnectionStringFactory connectionStringFactory, SqliteConnection innerConnection) {
+		this.ConnectionStringFactory = connectionStringFactory;
+		this.InnerConnection = innerConnection;
+	}
+	
+	internal async Task Setup(CancellationToken cancellationToken) {
+		await this.ExecuteAsync("PRAGMA journal_mode=WAL", cancellationToken);
+		await this.ExecuteAsync("PRAGMA foreign_keys=ON", cancellationToken);
+	}
+	
+	[SuppressMessage("ReSharper", "WithExpressionModifiesAllMembers")]
+	internal async Task AttachDatabase(AttachedDatabase attachedDatabase) {
+		await this.ExecuteAsync("ATTACH DATABASE '" + attachedDatabase.Path.Replace("'", "''") + "' AS " + attachedDatabase.Schema);
+		ImmutableInterlocked.Update(ref attachedDatabases, static (set, schema) => set.Add(schema), attachedDatabase.Schema);
+	}
+	
+	public bool HasAttachedDatabase(string schema) {
+		return attachedDatabases.Contains(schema);
+	}
+	
+	public async Task BeginTransactionAsync() {
+		if (activeTransaction != null) {
+			throw new InvalidOperationException("A transaction is already active.");
+		}
+		
+		activeTransaction = await InnerConnection.BeginTransactionAsync();
+	}
+	
+	public async Task CommitTransactionAsync() {
+		if (activeTransaction == null) {
+			throw new InvalidOperationException("No active transaction to commit.");
+		}
+		
+		await activeTransaction.CommitAsync();
+		await activeTransaction.DisposeAsync();
+		activeTransaction = null;
+	}
+	
+	public async Task RollbackTransactionAsync() {
+		if (activeTransaction == null) {
+			throw new InvalidOperationException("No active transaction to rollback.");
+		}
+		
+		await activeTransaction.RollbackAsync();
+		await activeTransaction.DisposeAsync();
+		activeTransaction = null;
+	}
+	
+	public void AssignActiveTransaction(SqliteCommand command) {
+		command.Transaction = (SqliteTransaction?) activeTransaction;
+	}
+	
+	public async ValueTask DisposeAsync() {
+		if (activeTransaction != null) {
+			await RollbackTransactionAsync();
+		}
+		
+		await DisposeConnection();
+	}
+	
+	private protected virtual async ValueTask DisposeConnection() {
+		await InnerConnection.DisposeAsync();
+	}
+}

app/Server/Database/Sqlite/Utils/ISqliteAttachedDatabaseCollector.cs

@@ -0,0 +1,7 @@
+using System.Collections.Generic;
+
+namespace DHT.Server.Database.Sqlite.Utils;
+
+interface ISqliteAttachedDatabaseCollector {
+	IAsyncEnumerable<AttachedDatabase> GetAttachedDatabases(ISqliteConnection conn);
+}

app/Server/Database/Sqlite/Utils/ISqliteConnection.cs

@@ -7,6 +7,8 @@ namespace DHT.Server.Database.Sqlite.Utils;
 interface ISqliteConnection : IAsyncDisposable {
 	SqliteConnection InnerConnection { get; }
 	
+	bool HasAttachedDatabase(string schema);
+	
 	Task BeginTransactionAsync();
 	Task CommitTransactionAsync();
 	Task RollbackTransactionAsync();

app/Server/Database/Sqlite/Utils/InitialDatabaseSettings.cs

@@ -0,0 +1,3 @@
+namespace DHT.Server.Database.Sqlite.Utils;
+
+public sealed record InitialDatabaseSettings(bool SeparateFileForDownloads, bool DownloadsAutoStart);

app/Server/Database/Sqlite/Utils/SqliteConnectionPool.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Data.Common;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Utils.Collections;
@@ -9,16 +9,13 @@ using Microsoft.Data.Sqlite;
 namespace DHT.Server.Database.Sqlite.Utils;
 
 sealed class SqliteConnectionPool : IAsyncDisposable {
-	public static async Task<SqliteConnectionPool> Create(SqliteConnectionStringBuilder connectionStringBuilder, int poolSize) {
-		var pool = new SqliteConnectionPool(poolSize);
-		await pool.InitializePooledConnections(connectionStringBuilder);
+	public static async Task<SqliteConnectionPool> Create(SqliteConnectionStringFactory connectionStringFactory, int poolSize, ISqliteAttachedDatabaseCollector attachedDatabaseCollector) {
+		SqliteConnectionPool pool = new SqliteConnectionPool(poolSize, attachedDatabaseCollector);
+		await pool.InitializePooledConnections(connectionStringFactory);
 		return pool;
 	}
 	
-	private static string GetConnectionString(SqliteConnectionStringBuilder connectionStringBuilder) {
-		connectionStringBuilder.Pooling = false;
-		return connectionStringBuilder.ToString();
-	}
+	private readonly ISqliteAttachedDatabaseCollector attachedDatabaseCollector;
 	
 	private readonly int poolSize;
 	private readonly List<PooledConnection> all;
@@ -27,24 +24,35 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
 	private readonly CancellationTokenSource disposalTokenSource = new ();
 	private readonly CancellationToken disposalToken;
 	
-	private SqliteConnectionPool(int poolSize) {
+	private SqliteConnectionPool(int poolSize, ISqliteAttachedDatabaseCollector attachedDatabaseCollector) {
+		this.attachedDatabaseCollector = attachedDatabaseCollector;
+		
 		this.poolSize = poolSize;
 		this.all = new List<PooledConnection>(poolSize);
 		this.free = new ConcurrentPool<PooledConnection>(poolSize);
+		
 		this.disposalToken = disposalTokenSource.Token;
 	}
 	
-	private async Task InitializePooledConnections(SqliteConnectionStringBuilder connectionStringBuilder) {
-		string connectionString = GetConnectionString(connectionStringBuilder);
+	private async Task InitializePooledConnections(SqliteConnectionStringFactory connectionStringFactory) {
+		string connectionString = connectionStringFactory.Create();
+		
+		List<AttachedDatabase> attachedDatabases = [];
 		
 		for (int i = 0; i < poolSize; i++) {
-			var conn = new SqliteConnection(connectionString);
-			conn.Open();
+			SqliteConnection innerConnection = new SqliteConnection(connectionString);
+			await innerConnection.OpenAsync(disposalToken);
 			
-			var pooledConnection = new PooledConnection(this, conn);
+			PooledConnection pooledConnection = new PooledConnection(this, connectionStringFactory, innerConnection);
+			await pooledConnection.Setup(disposalToken);
 			
-			await pooledConnection.ExecuteAsync("PRAGMA journal_mode=WAL", disposalToken);
-			await pooledConnection.ExecuteAsync("PRAGMA foreign_keys=ON", disposalToken);
+			if (i == 0) {
+				attachedDatabases = await attachedDatabaseCollector.GetAttachedDatabases(pooledConnection).ToListAsync(disposalToken);
+			}
+			
+			foreach (var attachedDatabase in attachedDatabases) {
+				await pooledConnection.AttachDatabase(attachedDatabase);
+			}
 			
 			all.Add(pooledConnection);
 			await free.Push(pooledConnection, disposalToken);
@@ -74,48 +82,8 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
 		disposalTokenSource.Dispose();
 	}
 	
-	private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) : ISqliteConnection {
-		public SqliteConnection InnerConnection { get; } = conn;
-		
-		private DbTransaction? activeTransaction;
-		
-		public async Task BeginTransactionAsync() {
-			if (activeTransaction != null) {
-				throw new InvalidOperationException("A transaction is already active.");
-			}
-			
-			activeTransaction = await InnerConnection.BeginTransactionAsync();
-		}
-		
-		public async Task CommitTransactionAsync() {
-			if (activeTransaction == null) {
-				throw new InvalidOperationException("No active transaction to commit.");
-			}
-			
-			await activeTransaction.CommitAsync();
-			await activeTransaction.DisposeAsync();
-			activeTransaction = null;
-		}
-		
-		public async Task RollbackTransactionAsync() {
-			if (activeTransaction == null) {
-				throw new InvalidOperationException("No active transaction to rollback.");
-			}
-			
-			await activeTransaction.RollbackAsync();
-			await activeTransaction.DisposeAsync();
-			activeTransaction = null;
-		}
-		
-		public void AssignActiveTransaction(SqliteCommand command) {
-			command.Transaction = (SqliteTransaction?) activeTransaction;
-		}
-		
-		public async ValueTask DisposeAsync() {
-			if (activeTransaction != null) {
-				await RollbackTransactionAsync();
-			}
-			
+	private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnectionStringFactory connectionStringFactory, SqliteConnection innerConnection) : CustomSqliteConnection(connectionStringFactory, innerConnection) {
+		private protected override async ValueTask DisposeConnection() {
 			await pool.Return(this);
 		}
 	}

app/Server/Database/Sqlite/Utils/SqliteConnectionStringFactory.cs

@@ -0,0 +1,15 @@
+using Microsoft.Data.Sqlite;
+
+namespace DHT.Server.Database.Sqlite.Utils;
+
+sealed record SqliteConnectionStringFactory(string Path) {
+	public string Create() {
+		var builder = new SqliteConnectionStringBuilder {
+			DataSource = Path,
+			Mode = SqliteOpenMode.ReadWriteCreate,
+			Pooling = false,
+		};
+		
+		return builder.ToString();
+	}
+}