Add support for merging downloaded attachments from other databases

app/Desktop/Main/Pages/AttachmentsPageModel.cs

@@ -87,6 +87,9 @@ namespace DHT.Desktop.Main.Pages {
 					downloadStatisticsComputer.Recompute();
 				}
 			}
+			else if (e.PropertyName == nameof(DatabaseStatistics.TotalDownloads)) {
+				downloadStatisticsComputer.Recompute();
+			}
 		}
 
 		private void EnqueueDownloadItems() {

app/Desktop/Main/Pages/DatabasePageModel.cs

@@ -2,8 +2,10 @@ using System;
 using System.Diagnostics;
 using System.IO;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Avalonia.Controls;
+using Avalonia.Threading;
 using DHT.Desktop.Common;
 using DHT.Desktop.Dialogs.Message;
 using DHT.Desktop.Dialogs.Progress;
@@ -56,6 +58,10 @@ namespace DHT.Desktop.Main.Pages {
 			}
 		}
 
+		public void CloseDatabase() {
+			DatabaseClosed?.Invoke(this, EventArgs.Empty);
+		}
+
 		public async void MergeWithDatabase() {
 			var fileDialog = DatabaseGui.NewOpenDatabaseFileDialog();
 			fileDialog.Directory = Path.GetDirectoryName(Db.Path);
@@ -74,10 +80,6 @@ namespace DHT.Desktop.Main.Pages {
 			await progressDialog.ShowDialog(window);
 		}
 
-		public void CloseDatabase() {
-			DatabaseClosed?.Invoke(this, EventArgs.Empty);
-		}
-
 		private static async Task MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
 			int total = paths.Length;
 
@@ -90,9 +92,29 @@ namespace DHT.Desktop.Main.Pages {
 
 				return DialogResult.YesNo.Yes == upgradeResult;
 			}
+			
+			await PerformImport(target, paths, dialog, callback, "Database Merge", "Database Error", "database file", async path => {
+				SynchronizationContext? prevSyncContext = SynchronizationContext.Current;
+				SynchronizationContext.SetSynchronizationContext(new AvaloniaSynchronizationContext());
+				IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, CheckCanUpgradeDatabase);
+				SynchronizationContext.SetSynchronizationContext(prevSyncContext);
+				
+				if (db == null) {
+					return false;
+				}
 
-			var oldStatistics = target.Statistics.Clone();
-			var oldMessageCount = target.CountMessages();
+				try {
+					target.AddFrom(db);
+					return true;
+				} finally {
+					db.Dispose();
+				}
+			});
+		}
+
+		private static async Task PerformImport(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback, string neutralDialogTitle, string errorDialogTitle, string itemName, Func<string, Task<bool>> performImport) {
+			int total = paths.Length;
+			var oldStatistics = target.SnapshotStatistics();
 
 			int successful = 0;
 			int finished = 0;
@@ -102,56 +124,53 @@ namespace DHT.Desktop.Main.Pages {
 				++finished;
 
 				if (!File.Exists(path)) {
-					await Dialog.ShowOk(dialog, "Database Error", "Database '" + Path.GetFileName(path) + "' no longer exists.");
-					continue;
-				}
-
-				IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, CheckCanUpgradeDatabase);
-				if (db == null) {
+					await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' no longer exists.");
 					continue;
 				}
 
 				try {
-					target.AddFrom(db);
+					if (await performImport(path)) {
+						++successful;
+					}
 				} catch (Exception ex) {
 					Log.Error(ex);
-					await Dialog.ShowOk(dialog, "Database Error", "Database '" + Path.GetFileName(path) + "' could not be merged: " + ex.Message);
-					continue;
-				} finally {
-					db.Dispose();
+					await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' could not be imported: " + ex.Message);
 				}
-
-				++successful;
 			}
 
 			await callback.Update("Done", finished, total);
 
 			if (successful == 0) {
-				await Dialog.ShowOk(dialog, "Database Merge", "Nothing was merged.");
+				await Dialog.ShowOk(dialog, neutralDialogTitle, "Nothing was imported.");
 				return;
 			}
 
-			var newStatistics = target.Statistics;
+			await Dialog.ShowOk(dialog, neutralDialogTitle, GetImportDialogMessage(oldStatistics, target.SnapshotStatistics(), successful, total, itemName));
+		}
+
+		private static string GetImportDialogMessage(DatabaseStatisticsSnapshot oldStatistics, DatabaseStatisticsSnapshot newStatistics, int successfulItems, int totalItems, string itemName) {
 			long newServers = newStatistics.TotalServers - oldStatistics.TotalServers;
 			long newChannels = newStatistics.TotalChannels - oldStatistics.TotalChannels;
-			long newMessages = target.CountMessages() - oldMessageCount;
+			long newUsers = newStatistics.TotalUsers - oldStatistics.TotalUsers;
+			long newMessages = newStatistics.TotalMessages - oldStatistics.TotalMessages;
 
 			StringBuilder message = new StringBuilder();
 			message.Append("Processed ");
 
-			if (successful == total) {
-				message.Append(successful.Pluralize("database file"));
+			if (successfulItems == totalItems) {
+				message.Append(successfulItems.Pluralize(itemName));
 			}
 			else {
-				message.Append(successful.Format()).Append(" out of ").Append(total.Pluralize("database file"));
+				message.Append(successfulItems.Format()).Append(" out of ").Append(totalItems.Pluralize(itemName));
 			}
 
 			message.Append(" and added:\n\n  \u2022 ");
 			message.Append(newServers.Pluralize("server")).Append("\n  \u2022 ");
 			message.Append(newChannels.Pluralize("channel")).Append("\n  \u2022 ");
+			message.Append(newUsers.Pluralize("user")).Append("\n  \u2022 ");
 			message.Append(newMessages.Pluralize("message"));
 
-			await Dialog.ShowOk(dialog, "Database Merge", message.ToString());
+			return message.ToString();
 		}
 	}
 }

app/Server/Data/Download.cs

@@ -3,10 +3,6 @@ using System.Net;
 
 namespace DHT.Server.Data {
 	public readonly struct Download {
-		internal static Download NewEnqueued(string url, ulong size) {
-			return new Download(url, DownloadStatus.Enqueued, size);
-		}
-
 		internal static Download NewSuccess(string url, byte[] data) {
 			return new Download(url, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
 		}
@@ -20,11 +16,15 @@ namespace DHT.Server.Data {
 		public ulong Size { get; }
 		public byte[]? Data { get; }
 
-		private Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
+		internal Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
 			Url = url;
 			Status = status;
 			Size = size;
 			Data = data;
 		}
+
+		internal Download WithData(byte[] data) {
+			return new Download(Url, Status, Size, data);
+		}
 	}
 }

app/Server/Database/DatabaseExtensions.cs

@@ -1,3 +1,5 @@
+using DHT.Server.Data;
+
 namespace DHT.Server.Database {
 	public static class DatabaseExtensions {
 		public static void AddFrom(this IDatabaseFile target, IDatabaseFile source) {
@@ -11,6 +13,10 @@ namespace DHT.Server.Database {
 
 			target.AddUsers(source.GetAllUsers().ToArray());
 			target.AddMessages(source.GetMessages().ToArray());
+			
+			foreach (var download in source.GetDownloadsWithoutData()) {
+				target.AddDownload(download.Status == DownloadStatus.Success ? source.GetDownloadWithData(download) : download);
+			}
 		}
 	}
 }

app/Server/Database/DatabaseStatistics.cs

@@ -1,12 +1,17 @@
 using DHT.Utils.Models;
 
 namespace DHT.Server.Database {
+	/// <summary>
+	/// A live view of database statistics.
+	/// Some of the totals are computed asynchronously and may not reflect the most recent version of the database, or may not be available at all until computed for the first time.
+	/// </summary>
 	public sealed class DatabaseStatistics : BaseModel {
 		private long totalServers;
 		private long totalChannels;
 		private long totalUsers;
 		private long? totalMessages;
 		private long? totalAttachments;
+		private long? totalDownloads;
 
 		public long TotalServers {
 			get => totalServers;
@@ -33,14 +38,9 @@ namespace DHT.Server.Database {
 			internal set => Change(ref totalAttachments, value);
 		}
 
-		public DatabaseStatistics Clone() {
-			return new DatabaseStatistics {
-				totalServers = totalServers,
-				totalChannels = totalChannels,
-				totalUsers = TotalUsers,
-				totalMessages = totalMessages,
-				totalAttachments = totalAttachments
-			};
+		public long? TotalDownloads {
+			get => totalDownloads;
+			internal set => Change(ref totalDownloads, value);
 		}
 	}
 }

app/Server/Database/DatabaseStatisticsSnapshot.cs

@@ -0,0 +1,11 @@
+namespace DHT.Server.Database {
+	/// <summary>
+	/// A complete snapshot of database statistics at a particular point in time.
+	/// </summary>
+	public readonly struct DatabaseStatisticsSnapshot {
+		public long TotalServers { get; internal init; }
+		public long TotalChannels { get; internal init; }
+		public long TotalUsers { get; internal init; }
+		public long TotalMessages { get; internal init; }
+	}
+}

app/Server/Database/DummyDatabaseFile.cs

@@ -13,6 +13,10 @@ namespace DHT.Server.Database {
 
 		private DummyDatabaseFile() {}
 
+		public DatabaseStatisticsSnapshot SnapshotStatistics() {
+			return new();
+		}
+
 		public void AddServer(Data.Server server) {}
 
 		public List<Data.Server> GetAllServers() {
@@ -47,7 +51,15 @@ namespace DHT.Server.Database {
 			return new();
 		}
 
-		public void AddDownloads(IEnumerable<Data.Download> downloads) {}
+		public List<Data.Download> GetDownloadsWithoutData() {
+			return new();
+		}
+
+		public Data.Download GetDownloadWithData(Data.Download download) {
+			return download;
+		}
+
+		public void AddDownload(Data.Download download) {}
 
 		public void EnqueueDownloadItems(AttachmentFilter? filter = null) {}
 		

app/Server/Database/IDatabaseFile.cs

@@ -9,6 +9,7 @@ namespace DHT.Server.Database {
 	public interface IDatabaseFile : IDisposable {
 		string Path { get; }
 		DatabaseStatistics Statistics { get; }
+		DatabaseStatisticsSnapshot SnapshotStatistics();
 
 		void AddServer(Data.Server server);
 		List<Data.Server> GetAllServers();
@@ -26,7 +27,10 @@ namespace DHT.Server.Database {
 
 		int CountAttachments(AttachmentFilter? filter = null);
 
-		void AddDownloads(IEnumerable<Data.Download> downloads);
+		void AddDownload(Data.Download download);
+		List<Data.Download> GetDownloadsWithoutData();
+		Data.Download GetDownloadWithData(Data.Download download);
+		
 		void EnqueueDownloadItems(AttachmentFilter? filter = null);
 		List<DownloadItem> GetEnqueuedDownloadItems(int count);
 		void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);

app/Server/Database/Sqlite/SqliteDatabaseFile.cs

@@ -24,14 +24,19 @@ namespace DHT.Server.Database.Sqlite {
 			};
 
 			var pool = new SqliteConnectionPool(connectionString, DefaultPoolSize);
+			bool wasOpened;
 
 			using (var conn = pool.Take()) {
-				if (!await new Schema(conn).Setup(checkCanUpgradeSchemas)) {
-					return null;
-				}
+				wasOpened = await new Schema(conn).Setup(checkCanUpgradeSchemas);
 			}
 
-			return new SqliteDatabaseFile(path, pool);
+			if (wasOpened) {
+				return new SqliteDatabaseFile(path, pool);
+			}
+			else {
+				pool.Dispose();
+				return null;
+			}
 		}
 
 		public string Path { get; }
@@ -41,6 +46,7 @@ namespace DHT.Server.Database.Sqlite {
 		private readonly SqliteConnectionPool pool;
 		private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
 		private readonly AsyncValueComputer<long>.Single totalAttachmentsComputer;
+		private readonly AsyncValueComputer<long>.Single totalDownloadsComputer;
 
 		private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
 			this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
@@ -48,6 +54,7 @@ namespace DHT.Server.Database.Sqlite {
 			
 			this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
 			this.totalAttachmentsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateAttachmentStatistics).WithOutdatedResults().BuildWithComputer(ComputeAttachmentStatistics);
+			this.totalDownloadsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateDownloadStatistics).WithOutdatedResults().BuildWithComputer(ComputeDownloadStatistics);
 
 			this.Path = path;
 			this.Statistics = new DatabaseStatistics();
@@ -60,12 +67,22 @@ namespace DHT.Server.Database.Sqlite {
 
 			totalMessagesComputer.Recompute();
 			totalAttachmentsComputer.Recompute();
+			totalDownloadsComputer.Recompute();
 		}
 
 		public void Dispose() {
 			pool.Dispose();
 		}
 
+		public DatabaseStatisticsSnapshot SnapshotStatistics() {
+			return new DatabaseStatisticsSnapshot {
+				TotalServers = Statistics.TotalServers,
+				TotalChannels = Statistics.TotalChannels,
+				TotalUsers = Statistics.TotalUsers,
+				TotalMessages = ComputeMessageStatistics()
+			};
+		}
+
 		public void AddServer(Data.Server server) {
 			using var conn = pool.Take();
 			using var cmd = conn.Upsert("servers", new[] {
@@ -390,10 +407,8 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 			return reader.Read() ? reader.GetInt32(0) : 0;
 		}
 
-		public void AddDownloads(IEnumerable<Data.Download> downloads) {
+		public void AddDownload(Data.Download download) {
 			using var conn = pool.Take();
-			using var tx = conn.BeginTransaction();
-
 			using var cmd = conn.Upsert("downloads", new[] {
 				("url", SqliteType.Text),
 				("status", SqliteType.Integer),
@@ -401,15 +416,46 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
 				("blob", SqliteType.Blob)
 			});
 
-			foreach (var download in downloads) {
-				cmd.Set(":url", download.Url);
-				cmd.Set(":status", (int) download.Status);
-				cmd.Set(":size", download.Size);
-				cmd.Set(":blob", download.Data);
-				cmd.ExecuteNonQuery();
-			}
+			cmd.Set(":url", download.Url);
+			cmd.Set(":status", (int) download.Status);
+			cmd.Set(":size", download.Size);
+			cmd.Set(":blob", download.Data);
+			cmd.ExecuteNonQuery();
+			
+			totalDownloadsComputer.Recompute();
+		}
 
-			tx.Commit();
+		public List<Data.Download> GetDownloadsWithoutData() {
+			var list = new List<Data.Download>();
+			
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT url, status, size FROM downloads");
+			using var reader = cmd.ExecuteReader();
+			
+			while (reader.Read()) {
+				string url = reader.GetString(0);
+				var status = (DownloadStatus) reader.GetInt32(1);
+				ulong size = reader.GetUint64(2);
+				
+				list.Add(new Data.Download(url, status, size));
+			}
+			
+			return list;
+		}
+
+		public Data.Download GetDownloadWithData(Data.Download download) {
+			using var conn = pool.Take();
+			using var cmd = conn.Command("SELECT blob FROM downloads WHERE url = :url");
+			cmd.AddAndSet(":url", SqliteType.Text, download.Url);
+			
+			using var reader = cmd.ExecuteReader();
+
+			if (reader.Read() && !reader.IsDBNull(0)) {
+				return download.WithData((byte[]) reader["blob"]);
+			}
+			else {
+				return download;
+			}
 		}
 
 		public void EnqueueDownloadItems(AttachmentFilter? filter = null) {
@@ -599,5 +645,14 @@ FROM downloads");
 		private void UpdateAttachmentStatistics(long totalAttachments) {
 			Statistics.TotalAttachments = totalAttachments;
 		}
+
+		private long ComputeDownloadStatistics() {
+			using var conn = pool.Take();
+			return conn.SelectScalar("SELECT COUNT(*) FROM downloads") as long? ?? 0L;
+		}
+
+		private void UpdateDownloadStatistics(long totalDownloads) {
+			Statistics.TotalDownloads = totalDownloads;
+		}
 	}
 }

app/Server/Download/BackgroundDownloadThread.cs

@@ -92,9 +92,9 @@ namespace DHT.Server.Download {
 							Log.Debug("Downloading " + url + "...");
 
 							try {
-								db.AddDownloads(new [] { Data.Download.NewSuccess(url, client.DownloadData(url)) });
+								db.AddDownload(Data.Download.NewSuccess(url, client.DownloadData(url)));
 							} catch (WebException e) {
-								db.AddDownloads(new [] { Data.Download.NewFailure(url, e.Response is HttpWebResponse response ? response.StatusCode : null, item.Size) });
+								db.AddDownload(Data.Download.NewFailure(url, e.Response is HttpWebResponse response ? response.StatusCode : null, item.Size));
 								Log.Error(e);
 							} finally {
 								parameters.FireOnItemFinished(item);