Release v45.0

Closes #273

.github/FUNDING.yml

@@ -1,3 +1 @@
-github: chylex
-patreon: chylex
 ko_fi: chylex

README.md

@@ -18,7 +18,7 @@ Folder organization:
 * `app/` contains a Visual Studio solution for the desktop app
 * `web/` contains source code of the [official website](https://dht.chylex.com), which can be used as a template when making your own website
 
-To start editing source code for the desktop app, install the [.NET 8 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/8.0), and then open `app/DiscordHistoryTracker.sln` in [Visual Studio](https://visualstudio.microsoft.com/downloads/) or [Rider](https://www.jetbrains.com/rider/).
+To start editing source code for the desktop app, install the [.NET 9 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/9.0), and then open `app/DiscordHistoryTracker.sln` in [Visual Studio](https://visualstudio.microsoft.com/downloads/) or [Rider](https://www.jetbrains.com/rider/).
 
 ### Building
 
@@ -28,18 +28,18 @@ To build a `Release` version of the desktop app, follow the instructions for you
 
 #### Release – Windows (64-bit)
 
-1. Install [Powershell 5](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows) or newer (on Windows 10, the included version of Powershell should be enough)
+1. Install Debian in WSL and open a terminal in the project folder.
+2. Run the `app/build.wsl.sh` script.
+3. Read the [Distribution](#distribution) section below.
 
-Run the `app/build.bat` script, and read the [Distribution](#distribution) section below.
+Note: The build script expects `dotnet.exe` to be installed in `C:\Program Files\dotnet`.
 
 #### Release – Other Operating Systems
 
-1. Install the `zip` package from your repository
+1. Install the `zip` package from your repository.
-
+2. Run the `app/build.sh` script.
-Run the `app/build.sh` script, and read the [Distribution](#distribution) section below.
+3. Read the [Distribution](#distribution) section below.
 
 #### Distribution
 
-The mentioned build scripts will prepare `Release` builds ready for distribution. Once the script finishes, the `app/bin` folder will contain self-contained executables for each major operating system, and a portable version that works on all other systems but requires .NET 8 to be installed.
+The mentioned build scripts will prepare `Release` builds ready for distribution. Once the script finishes, the `app/bin` folder will contain self-contained executables for each major operating system, and a portable version that works on all other systems but requires the ASP.NET Core Runtime to be installed.
-
-Note that when building on Windows, the generated `.zip` files for Linux and Mac will not have correct file permissions, so it will not be possible to run them by double-clicking the executable. Since .NET 8 fixed several issues with publishing Windows executables on Linux, I recommend using Linux to build the app for all operating systems.

app/Desktop/Dialogs/Progress/ProgressDialog.axaml.cs

@@ -2,11 +2,15 @@ using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using Avalonia.Controls;
+using DHT.Desktop.Dialogs.Message;
+using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Dialogs.Progress;
 
 [SuppressMessage("ReSharper", "MemberCanBeInternal")]
 public sealed partial class ProgressDialog : Window {
+	private static readonly Log Log = Log.ForType<ProgressDialog>();
+	
 	internal static async Task Show(Window owner, string title, Func<ProgressDialog, IProgressCallback, Task> action) {
 		var taskCompletionSource = new TaskCompletionSource();
 		var dialog = new ProgressDialog();
@@ -84,6 +88,11 @@ public sealed partial class ProgressDialog : Window {
 	
 	public async Task ShowProgressDialog(Window owner) {
 		await ShowDialog(owner);
+		try {
 			await progressTask;
+		} catch (Exception e) {
+			Log.Error(e);
+			await Dialog.ShowOk(owner, "Unexpected Error", e.Message);
+		}
 	}
 }

app/Desktop/Main/Pages/DownloadsPage.axaml

@@ -13,8 +13,8 @@
     </Design.DataContext>
 
     <UserControl.Styles>
-        <Style Selector="Expander">
+        <Style Selector="WrapPanel > Button">
-            <Setter Property="Margin" Value="0 5 0 0" />
+            <Setter Property="Margin" Value="0 0 10 10" />
         </Style>
         <Style Selector="DataGridColumnHeader">
             <Setter Property="FontWeight" Value="Medium" />
@@ -30,16 +30,17 @@
         </Style>
     </UserControl.Styles>
 
-    <StackPanel Orientation="Vertical" Spacing="20">
+    <StackPanel Orientation="Vertical">
-        <StackPanel Orientation="Horizontal" Spacing="10">
+        <WrapPanel Orientation="Horizontal">
             <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
             <Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed Downloads</Button>
-        </StackPanel>
+            <Button Command="{Binding OnClickDeleteOrphanedDownloads}">Delete Orphaned Downloads</Button>
+        </WrapPanel>
+        <StackPanel Orientation="Vertical" Spacing="20" Margin="0 10 0 0">
             <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
             <TextBlock TextWrapping="Wrap">
                 Downloading state and filter settings are remembered per-database.
             </TextBlock>
-        <StackPanel Orientation="Vertical" Spacing="12">
             <Expander Header="Download Status" IsExpanded="True">
                 <DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
                     <DataGrid.Columns>

app/Desktop/Main/Pages/DownloadsPageModel.cs

@@ -1,12 +1,17 @@
 using System;
+using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
+using Avalonia.Controls;
 using Avalonia.ReactiveUI;
 using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
+using DHT.Desktop.Dialogs.Message;
+using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
 using DHT.Server;
+using DHT.Server.Data;
 using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
 using DHT.Server.Data.Settings;
@@ -48,6 +53,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	
 	public bool IsDownloading => state.Downloader.IsDownloading;
 	
+	private readonly Window window;
 	private readonly State state;
 	private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
 	private readonly IDisposable downloadItemCountSubscription;
@@ -55,9 +61,10 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	private IDisposable? finishedItemsSubscription;
 	private DownloadItemFilter? currentDownloadFilter;
 	
-	public DownloadsPageModel() : this(State.Dummy) {}
+	public DownloadsPageModel() : this(null!, State.Dummy) {}
 	
-	public DownloadsPageModel(State state) {
+	public DownloadsPageModel(Window window, State state) {
+		this.window = window;
 		this.state = state;
 		
 		FilterModel = new DownloadItemFilterPanelModel(state);
@@ -158,6 +165,50 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
 	}
 	
+	private const string DeleteOrphanedDownloadsTitle = "Delete Orphaned Downloads";
+	
+	public async Task OnClickDeleteOrphanedDownloads() {
+		await ProgressDialog.Show(window, DeleteOrphanedDownloadsTitle, DeleteOrphanedDownloads);
+	}
+	
+	private async Task DeleteOrphanedDownloads(ProgressDialog dialog, IProgressCallback callback) {
+		await callback.UpdateIndeterminate("Searching for orphaned downloads...");
+		
+		HashSet<string> reachableNormalizedUrls = [];
+		HashSet<string> orphanedNormalizedUrls = [];
+		
+		await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
+			reachableNormalizedUrls.Add(download.NormalizedUrl);
+		}
+		
+		await foreach (Download download in state.Db.Downloads.Get()) {
+			string normalizedUrl = download.NormalizedUrl;
+			if (!reachableNormalizedUrls.Contains(normalizedUrl)) {
+				orphanedNormalizedUrls.Add(normalizedUrl);
+			}
+		}
+		
+		if (orphanedNormalizedUrls.Count == 0) {
+			await Dialog.ShowOk(window, DeleteOrphanedDownloadsTitle, "No orphaned downloads found.");
+			return;
+		}
+		
+		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
+			return;
+		}
+		
+		await callback.UpdateIndeterminate("Deleting orphaned downloads...");
+		await state.Db.Downloads.Remove(orphanedNormalizedUrls);
+		RecomputeDownloadStatistics();
+		
+		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
+			return;
+		}
+		
+		await callback.UpdateIndeterminate("Vacuuming database...");
+		await state.Db.Vacuum();
+	}
+	
 	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
 		statisticsPending.Items = statusStatistics.PendingCount;
 		statisticsPending.Size = statusStatistics.PendingTotalSize;

app/Desktop/Main/Screens/MainContentScreenModel.cs

@@ -49,7 +49,7 @@ sealed class MainContentScreenModel : IAsyncDisposable {
 		TrackingPageModel = new TrackingPageModel(window);
 		TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
 		
-		DownloadsPageModel = new DownloadsPageModel(state);
+		DownloadsPageModel = new DownloadsPageModel(window, state);
 		DownloadsPage = new DownloadsPage { DataContext = DownloadsPageModel };
 		
 		ViewerPageModel = new ViewerPageModel(window, state);

app/Directory.Build.props

@@ -1,8 +1,8 @@
 <Project>
   
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
-    <LangVersion>12</LangVersion>
+    <LangVersion>13</LangVersion>
     <Nullable>enable</Nullable>
   </PropertyGroup>
   
@@ -39,6 +39,7 @@
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
     <DebugSymbols>false</DebugSymbols>
     <DebugType>none</DebugType>
+    <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
   </PropertyGroup>
   
   <PropertyGroup>

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -32,6 +32,10 @@ public interface IDownloadRepository {
 	
 	Task<int> RetryFailed(CancellationToken cancellationToken = default);
 	
+	Task Remove(ICollection<string> normalizedUrls);
+	
+	IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken = default);
+	
 	internal sealed class Dummy : IDownloadRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
 		
@@ -70,5 +74,13 @@ public interface IDownloadRepository {
 		public Task<int> RetryFailed(CancellationToken cancellationToken) {
 			return Task.FromResult(0);
 		}
+		
+		public Task Remove(ICollection<string> normalizedUrls) {
+			return Task.CompletedTask;
+		}
+		
+		public IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken) {
+			return AsyncEnumerable.Empty<Data.Download>();
+		}
 	}
 }

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -321,4 +321,62 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 		return await cmd.ExecuteNonQueryAsync(cancellationToken);
 	}
+	
+	public async Task Remove(ICollection<string> normalizedUrls) {
+		await using (var conn = await pool.Take()) {
+			await conn.BeginTransactionAsync();
+			
+			await using (var cmd = conn.Command("DELETE FROM download_metadata WHERE normalized_url = :normalized_url")) {
+				cmd.Add(":normalized_url", SqliteType.Text);
+				
+				foreach (string normalizedUrl in normalizedUrls) {
+					cmd.Set(":normalized_url", normalizedUrl);
+					await cmd.ExecuteNonQueryAsync();
+				}
+			}
+			
+			await conn.CommitTransactionAsync();
+		}
+		
+		UpdateTotalCount();
+	}
+	
+	public async IAsyncEnumerable<Data.Download> FindAllDownloadableUrls([EnumeratorCancellation] CancellationToken cancellationToken = default) {
+		await using var conn = await pool.Take();
+		
+		await using (var cmd = conn.Command("SELECT normalized_url, download_url, type, size FROM attachments")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromAttachment(reader.GetString(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetUint64(3));
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT json FROM message_embeds")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				var result = await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0));
+				if (result is not null) {
+					yield return result;
+				}
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1));
+			}
+		}
+		
+		await using (var cmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM message_reactions WHERE emoji_id IS NOT NULL")) {
+			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
+			
+			while (await reader.ReadAsync(cancellationToken)) {
+				yield return DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1));
+			}
+		}
+	}
 }

app/Server/Download/DownloadLinkExtractor.cs

@@ -28,7 +28,11 @@ static class DownloadLinkExtractor {
 	}
 	
 	public static Data.Download FromAttachment(Attachment attachment) {
-		return new Data.Download(attachment.NormalizedUrl, attachment.DownloadUrl, DownloadStatus.Pending, attachment.Type, attachment.Size);
+		return FromAttachment(attachment.NormalizedUrl, attachment.DownloadUrl, attachment.Type, attachment.Size);
+	}
+	
+	public static Data.Download FromAttachment(string normalizedUrl, string downloadUrl, string? type, ulong size) {
+		return new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Pending, type, size);
 	}
 	
 	public static async Task<Data.Download?> TryFromEmbedJson(Stream jsonStream) {

app/Version.cs

@@ -8,5 +8,5 @@ using DHT.Utils;
 namespace DHT.Utils;
 
 static class Version {
-	public const string Tag = "44.0.0.0";
+	public const string Tag = "45.0.0.0";
 }

app/build.bat

@@ -1,15 +0,0 @@
-@echo off
-set list=win-x64 linux-x64 osx-x64
-
-rmdir /S /Q bin
-
-(for %%a in (%list%) do (
-  dotnet publish Desktop -c Release -r %%a -o ./bin/%%a --self-contained true
-  powershell "Compress-Archive -Path ./bin/%%a/* -DestinationPath ./bin/%%a.zip -CompressionLevel Optimal"
-))
-
-dotnet publish Desktop -c Release -o ./bin/portable -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
-powershell "Compress-Archive -Path ./bin/portable/* -DestinationPath ./bin/portable.zip -CompressionLevel Optimal"
-
-echo Done
-pause

app/build.sh

@@ -1,25 +1,49 @@
 #!/bin/bash
 set -e
 
+export TZ=UTC
+
 if [ ! -f "DiscordHistoryTracker.sln" ]; then
   echo "Missing DiscordHistoryTracker.sln in working directory!"
   exit 1
 fi
 
 makezip() {
-	pushd "./bin/$1"
+  BIN_PATH="$(pwd)/bin"
-	zip -9 -r "../$1.zip" .
+
+  pushd "$BIN_PATH/$1"
+
+  find . -type d -exec chmod 755 {} \;
+  find . -type f -exec chmod 644 {} \;
+
+  chmod -f 755 DiscordHistoryTracker || true
+  chmod -f 755 DiscordHistoryTracker.exe || true
+
+  find . -type f | sort | zip -9 -X "$BIN_PATH/$1.zip" -@
+
   popd
 }
 
 rm -rf "./bin"
 
-configurations=(win-x64 linux-x64 osx-x64)
+dedicated_runtimes=(win-x64 linux-x64)
+skipped_portable_runtimes=(browser-wasm linux-mips64 linux-s390x linux-ppc64le)
 
-for cfg in ${configurations[@]}; do
+# Dedicated Runtimes
+
+for cfg in "${dedicated_runtimes[@]}"; do
   dotnet publish Desktop -c Release -r "$cfg" -o "./bin/$cfg" --self-contained true
   makezip "$cfg"
 done
 
+# Portable
+
 dotnet publish Desktop -c Release -o "./bin/portable" -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
+
+rm "./bin/portable/DiscordHistoryTracker"
+
+for runtime in "${skipped_portable_runtimes[@]}"; do
+  rm -rf "./bin/portable/runtimes/$runtime"
+done
+
 makezip "portable"

app/build.wsl.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+export TZ=UTC
+
+if [ ! -f "DiscordHistoryTracker.sln" ]; then
+  echo "Missing DiscordHistoryTracker.sln in working directory!"
+  exit 1
+fi
+
+makezip() {
+  TMP_PATH="/tmp/dht-build"
+  BIN_PATH="$(pwd)/bin"
+
+  rm -rf "$TMP_PATH"
+  cp -r "$BIN_PATH/$1/" "$TMP_PATH"
+  pushd "$TMP_PATH"
+
+  find . -type d -exec chmod 755 {} \;
+  find . -type f -exec chmod 644 {} \;
+
+  chmod -f 755 DiscordHistoryTracker || true
+  chmod -f 755 DiscordHistoryTracker.exe || true
+
+  find . -type f | sort | zip -9 -X "$BIN_PATH/$1.zip" -@
+
+  popd
+  rm -rf "$TMP_PATH"
+}
+
+rm -rf "./bin"
+
+dedicated_runtimes=(win-x64 linux-x64)
+skipped_portable_runtimes=(browser-wasm linux-mips64 linux-s390x linux-ppc64le)
+
+# Dedicated Runtimes
+
+for cfg in "${dedicated_runtimes[@]}"; do
+  "/mnt/c/Program Files/dotnet/dotnet.exe" publish Desktop -c Release -r "$cfg" -o "./bin/$cfg" --self-contained true
+  makezip "$cfg"
+done
+
+# Portable
+
+"/mnt/c/Program Files/dotnet/dotnet.exe" publish Desktop -c Release -o "./bin/portable" -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
+
+rm "./bin/portable/DiscordHistoryTracker.exe"
+
+for runtime in "${skipped_portable_runtimes[@]}"; do
+  rm -rf "./bin/portable/runtimes/$runtime"
+done
+
+makezip "portable"

app/global.json

@@ -1,6 +1,6 @@
 {
     "sdk": {
-        "version": "8.0.0",
+        "version": "9.0.0",
         "rollForward": "latestMinor"
     }
 }