Remove orphaned servers, channels, users, and attachments after removing messages

.github/FUNDING.yml

@@ -1 +1,3 @@
+github: chylex
+patreon: chylex
 ko_fi: chylex

README.md

@@ -18,7 +18,7 @@ Folder organization:
 * `app/` contains a Visual Studio solution for the desktop app
 * `web/` contains source code of the [official website](https://dht.chylex.com), which can be used as a template when making your own website
 
-To start editing source code for the desktop app, install the [.NET 9 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/9.0), and then open `app/DiscordHistoryTracker.sln` in [Visual Studio](https://visualstudio.microsoft.com/downloads/) or [Rider](https://www.jetbrains.com/rider/).
+To start editing source code for the desktop app, install the [.NET 8 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/8.0), and then open `app/DiscordHistoryTracker.sln` in [Visual Studio](https://visualstudio.microsoft.com/downloads/) or [Rider](https://www.jetbrains.com/rider/).
 
 ### Building
 
@@ -28,18 +28,18 @@ To build a `Release` version of the desktop app, follow the instructions for you
 
 #### Release – Windows (64-bit)
 
-1. Install Debian in WSL and open a terminal in the project folder.
+1. Install [Powershell 5](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows) or newer (on Windows 10, the included version of Powershell should be enough)
-2. Run the `app/build.wsl.sh` script.
-3. Read the [Distribution](#distribution) section below.
 
-Note: The build script expects `dotnet.exe` to be installed in `C:\Program Files\dotnet`.
+Run the `app/build.bat` script, and read the [Distribution](#distribution) section below.
 
 #### Release – Other Operating Systems
 
-1. Install the `zip` package from your repository.
+1. Install the `zip` package from your repository
-2. Run the `app/build.sh` script.
+
-3. Read the [Distribution](#distribution) section below.
+Run the `app/build.sh` script, and read the [Distribution](#distribution) section below.
 
 #### Distribution
 
-The mentioned build scripts will prepare `Release` builds ready for distribution. Once the script finishes, the `app/bin` folder will contain self-contained executables for each major operating system, and a portable version that works on all other systems but requires the ASP.NET Core Runtime to be installed.
+The mentioned build scripts will prepare `Release` builds ready for distribution. Once the script finishes, the `app/bin` folder will contain self-contained executables for each major operating system, and a portable version that works on all other systems but requires .NET 8 to be installed.
+
+Note that when building on Windows, the generated `.zip` files for Linux and Mac will not have correct file permissions, so it will not be possible to run them by double-clicking the executable. Since .NET 8 fixed several issues with publishing Windows executables on Linux, I recommend using Linux to build the app for all operating systems.

app/Desktop/Dialogs/Progress/ProgressDialog.axaml.cs

@@ -2,15 +2,11 @@ using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using Avalonia.Controls;
-using DHT.Desktop.Dialogs.Message;
-using DHT.Utils.Logging;
 
 namespace DHT.Desktop.Dialogs.Progress;
 
 [SuppressMessage("ReSharper", "MemberCanBeInternal")]
 public sealed partial class ProgressDialog : Window {
-	private static readonly Log Log = Log.ForType<ProgressDialog>();
-	
 	internal static async Task Show(Window owner, string title, Func<ProgressDialog, IProgressCallback, Task> action) {
 		var taskCompletionSource = new TaskCompletionSource();
 		var dialog = new ProgressDialog();
@@ -88,11 +84,6 @@ public sealed partial class ProgressDialog : Window {
 	
 	public async Task ShowProgressDialog(Window owner) {
 		await ShowDialog(owner);
-		try {
+		await progressTask;
-			await progressTask;
-		} catch (Exception e) {
-			Log.Error(e);
-			await Dialog.ShowOk(owner, "Unexpected Error", e.Message);
-		}
 	}
 }

app/Desktop/Main/Pages/DownloadsPage.axaml

@@ -13,8 +13,8 @@
     </Design.DataContext>
 
     <UserControl.Styles>
-        <Style Selector="WrapPanel > Button">
+        <Style Selector="Expander">
-            <Setter Property="Margin" Value="0 0 10 10" />
+            <Setter Property="Margin" Value="0 5 0 0" />
         </Style>
         <Style Selector="DataGridColumnHeader">
             <Setter Property="FontWeight" Value="Medium" />
@@ -30,17 +30,16 @@
         </Style>
     </UserControl.Styles>
 
-    <StackPanel Orientation="Vertical">
+    <StackPanel Orientation="Vertical" Spacing="20">
-        <WrapPanel Orientation="Horizontal">
+        <StackPanel Orientation="Horizontal" Spacing="10">
             <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
             <Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed Downloads</Button>
-            <Button Command="{Binding OnClickDeleteOrphanedDownloads}">Delete Orphaned Downloads</Button>
+        </StackPanel>
-        </WrapPanel>
+        <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
-        <StackPanel Orientation="Vertical" Spacing="20" Margin="0 10 0 0">
+        <TextBlock TextWrapping="Wrap">
-            <controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />
+            Downloading state and filter settings are remembered per-database.
-            <TextBlock TextWrapping="Wrap">
+        </TextBlock>
-                Downloading state and filter settings are remembered per-database.
+        <StackPanel Orientation="Vertical" Spacing="12">
-            </TextBlock>
             <Expander Header="Download Status" IsExpanded="True">
                 <DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
                     <DataGrid.Columns>

app/Desktop/Main/Pages/DownloadsPageModel.cs

@@ -1,17 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
-using Avalonia.Controls;
 using Avalonia.ReactiveUI;
 using CommunityToolkit.Mvvm.ComponentModel;
 using DHT.Desktop.Common;
-using DHT.Desktop.Dialogs.Message;
-using DHT.Desktop.Dialogs.Progress;
 using DHT.Desktop.Main.Controls;
 using DHT.Server;
-using DHT.Server.Data;
 using DHT.Server.Data.Aggregations;
 using DHT.Server.Data.Filters;
 using DHT.Server.Data.Settings;
@@ -53,7 +48,6 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	
 	public bool IsDownloading => state.Downloader.IsDownloading;
 	
-	private readonly Window window;
 	private readonly State state;
 	private readonly ThrottledTask<DownloadStatusStatistics> downloadStatisticsTask;
 	private readonly IDisposable downloadItemCountSubscription;
@@ -61,10 +55,9 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 	private IDisposable? finishedItemsSubscription;
 	private DownloadItemFilter? currentDownloadFilter;
 	
-	public DownloadsPageModel() : this(null!, State.Dummy) {}
+	public DownloadsPageModel() : this(State.Dummy) {}
 	
-	public DownloadsPageModel(Window window, State state) {
+	public DownloadsPageModel(State state) {
-		this.window = window;
 		this.state = state;
 		
 		FilterModel = new DownloadItemFilterPanelModel(state);
@@ -165,50 +158,6 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
 		downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
 	}
 	
-	private const string DeleteOrphanedDownloadsTitle = "Delete Orphaned Downloads";
-	
-	public async Task OnClickDeleteOrphanedDownloads() {
-		await ProgressDialog.Show(window, DeleteOrphanedDownloadsTitle, DeleteOrphanedDownloads);
-	}
-	
-	private async Task DeleteOrphanedDownloads(ProgressDialog dialog, IProgressCallback callback) {
-		await callback.UpdateIndeterminate("Searching for orphaned downloads...");
-		
-		HashSet<string> reachableNormalizedUrls = [];
-		HashSet<string> orphanedNormalizedUrls = [];
-		
-		await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
-			reachableNormalizedUrls.Add(download.NormalizedUrl);
-		}
-		
-		await foreach (Download download in state.Db.Downloads.Get()) {
-			string normalizedUrl = download.NormalizedUrl;
-			if (!reachableNormalizedUrls.Contains(normalizedUrl)) {
-				orphanedNormalizedUrls.Add(normalizedUrl);
-			}
-		}
-		
-		if (orphanedNormalizedUrls.Count == 0) {
-			await Dialog.ShowOk(window, DeleteOrphanedDownloadsTitle, "No orphaned downloads found.");
-			return;
-		}
-		
-		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
-			return;
-		}
-		
-		await callback.UpdateIndeterminate("Deleting orphaned downloads...");
-		await state.Db.Downloads.Remove(orphanedNormalizedUrls);
-		RecomputeDownloadStatistics();
-		
-		if (await Dialog.ShowYesNo(window, DeleteOrphanedDownloadsTitle, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
-			return;
-		}
-		
-		await callback.UpdateIndeterminate("Vacuuming database...");
-		await state.Db.Vacuum();
-	}
-	
 	private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
 		statisticsPending.Items = statusStatistics.PendingCount;
 		statisticsPending.Size = statusStatistics.PendingTotalSize;

app/Desktop/Main/Screens/MainContentScreenModel.cs

@@ -49,7 +49,7 @@ sealed class MainContentScreenModel : IAsyncDisposable {
 		TrackingPageModel = new TrackingPageModel(window);
 		TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
 		
-		DownloadsPageModel = new DownloadsPageModel(window, state);
+		DownloadsPageModel = new DownloadsPageModel(state);
 		DownloadsPage = new DownloadsPage { DataContext = DownloadsPageModel };
 		
 		ViewerPageModel = new ViewerPageModel(window, state);

app/Directory.Build.props

@@ -1,8 +1,8 @@
 <Project>
   
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
-    <LangVersion>13</LangVersion>
+    <LangVersion>12</LangVersion>
     <Nullable>enable</Nullable>
   </PropertyGroup>
   
@@ -39,7 +39,6 @@
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
     <DebugSymbols>false</DebugSymbols>
     <DebugType>none</DebugType>
-    <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
   </PropertyGroup>
   
   <PropertyGroup>

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -32,10 +32,6 @@ public interface IDownloadRepository {
 	
 	Task<int> RetryFailed(CancellationToken cancellationToken = default);
 	
-	Task Remove(ICollection<string> normalizedUrls);
-	
-	IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken = default);
-	
 	internal sealed class Dummy : IDownloadRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
 		
@@ -74,13 +70,5 @@ public interface IDownloadRepository {
 		public Task<int> RetryFailed(CancellationToken cancellationToken) {
 			return Task.FromResult(0);
 		}
-		
-		public Task Remove(ICollection<string> normalizedUrls) {
-			return Task.CompletedTask;
-		}
-		
-		public IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken) {
-			return AsyncEnumerable.Empty<Data.Download>();
-		}
 	}
 }

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -321,62 +321,4 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
 		return await cmd.ExecuteNonQueryAsync(cancellationToken);
 	}
-	
-	public async Task Remove(ICollection<string> normalizedUrls) {
-		await using (var conn = await pool.Take()) {
-			await conn.BeginTransactionAsync();
-			
-			await using (var cmd = conn.Command("DELETE FROM download_metadata WHERE normalized_url = :normalized_url")) {
-				cmd.Add(":normalized_url", SqliteType.Text);
-				
-				foreach (string normalizedUrl in normalizedUrls) {
-					cmd.Set(":normalized_url", normalizedUrl);
-					await cmd.ExecuteNonQueryAsync();
-				}
-			}
-			
-			await conn.CommitTransactionAsync();
-		}
-		
-		UpdateTotalCount();
-	}
-	
-	public async IAsyncEnumerable<Data.Download> FindAllDownloadableUrls([EnumeratorCancellation] CancellationToken cancellationToken = default) {
-		await using var conn = await pool.Take();
-		
-		await using (var cmd = conn.Command("SELECT normalized_url, download_url, type, size FROM attachments")) {
-			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
-			
-			while (await reader.ReadAsync(cancellationToken)) {
-				yield return DownloadLinkExtractor.FromAttachment(reader.GetString(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetUint64(3));
-			}
-		}
-		
-		await using (var cmd = conn.Command("SELECT json FROM message_embeds")) {
-			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
-			
-			while (await reader.ReadAsync(cancellationToken)) {
-				var result = await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0));
-				if (result is not null) {
-					yield return result;
-				}
-			}
-		}
-		
-		await using (var cmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) {
-			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
-			
-			while (await reader.ReadAsync(cancellationToken)) {
-				yield return DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1));
-			}
-		}
-		
-		await using (var cmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM message_reactions WHERE emoji_id IS NOT NULL")) {
-			await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
-			
-			while (await reader.ReadAsync(cancellationToken)) {
-				yield return DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1));
-			}
-		}
-	}
 }

app/Server/Download/DownloadLinkExtractor.cs

@@ -28,11 +28,7 @@ static class DownloadLinkExtractor {
 	}
 	
 	public static Data.Download FromAttachment(Attachment attachment) {
-		return FromAttachment(attachment.NormalizedUrl, attachment.DownloadUrl, attachment.Type, attachment.Size);
+		return new Data.Download(attachment.NormalizedUrl, attachment.DownloadUrl, DownloadStatus.Pending, attachment.Type, attachment.Size);
-	}
-	
-	public static Data.Download FromAttachment(string normalizedUrl, string downloadUrl, string? type, ulong size) {
-		return new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Pending, type, size);
 	}
 	
 	public static async Task<Data.Download?> TryFromEmbedJson(Stream jsonStream) {

app/Version.cs

@@ -8,5 +8,5 @@ using DHT.Utils;
 namespace DHT.Utils;
 
 static class Version {
-	public const string Tag = "45.0.0.0";
+	public const string Tag = "44.0.0.0";
 }

app/build.bat

@@ -0,0 +1,15 @@
+@echo off
+set list=win-x64 linux-x64 osx-x64
+
+rmdir /S /Q bin
+
+(for %%a in (%list%) do (
+  dotnet publish Desktop -c Release -r %%a -o ./bin/%%a --self-contained true
+  powershell "Compress-Archive -Path ./bin/%%a/* -DestinationPath ./bin/%%a.zip -CompressionLevel Optimal"
+))
+
+dotnet publish Desktop -c Release -o ./bin/portable -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
+powershell "Compress-Archive -Path ./bin/portable/* -DestinationPath ./bin/portable.zip -CompressionLevel Optimal"
+
+echo Done
+pause

app/build.sh

@@ -1,49 +1,25 @@
 #!/bin/bash
 set -e
 
-export TZ=UTC
-
 if [ ! -f "DiscordHistoryTracker.sln" ]; then
-  echo "Missing DiscordHistoryTracker.sln in working directory!"
+	echo "Missing DiscordHistoryTracker.sln in working directory!"
-  exit 1
+	exit 1
 fi
 
 makezip() {
-  BIN_PATH="$(pwd)/bin"
+	pushd "./bin/$1"
-
+	zip -9 -r "../$1.zip" .
-  pushd "$BIN_PATH/$1"
+	popd
-
-  find . -type d -exec chmod 755 {} \;
-  find . -type f -exec chmod 644 {} \;
-
-  chmod -f 755 DiscordHistoryTracker || true
-  chmod -f 755 DiscordHistoryTracker.exe || true
-
-  find . -type f | sort | zip -9 -X "$BIN_PATH/$1.zip" -@
-
-  popd
 }
 
 rm -rf "./bin"
 
-dedicated_runtimes=(win-x64 linux-x64)
+configurations=(win-x64 linux-x64 osx-x64)
-skipped_portable_runtimes=(browser-wasm linux-mips64 linux-s390x linux-ppc64le)
 
-# Dedicated Runtimes
+for cfg in ${configurations[@]}; do
-
+	dotnet publish Desktop -c Release -r "$cfg" -o "./bin/$cfg" --self-contained true
-for cfg in "${dedicated_runtimes[@]}"; do
+	makezip "$cfg"
-  dotnet publish Desktop -c Release -r "$cfg" -o "./bin/$cfg" --self-contained true
-  makezip "$cfg"
 done
 
-# Portable
-
 dotnet publish Desktop -c Release -o "./bin/portable" -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
-
-rm "./bin/portable/DiscordHistoryTracker"
-
-for runtime in "${skipped_portable_runtimes[@]}"; do
-  rm -rf "./bin/portable/runtimes/$runtime"
-done
-
 makezip "portable"

app/build.wsl.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-set -e
-
-export TZ=UTC
-
-if [ ! -f "DiscordHistoryTracker.sln" ]; then
-  echo "Missing DiscordHistoryTracker.sln in working directory!"
-  exit 1
-fi
-
-makezip() {
-  TMP_PATH="/tmp/dht-build"
-  BIN_PATH="$(pwd)/bin"
-
-  rm -rf "$TMP_PATH"
-  cp -r "$BIN_PATH/$1/" "$TMP_PATH"
-  pushd "$TMP_PATH"
-
-  find . -type d -exec chmod 755 {} \;
-  find . -type f -exec chmod 644 {} \;
-
-  chmod -f 755 DiscordHistoryTracker || true
-  chmod -f 755 DiscordHistoryTracker.exe || true
-
-  find . -type f | sort | zip -9 -X "$BIN_PATH/$1.zip" -@
-
-  popd
-  rm -rf "$TMP_PATH"
-}
-
-rm -rf "./bin"
-
-dedicated_runtimes=(win-x64 linux-x64)
-skipped_portable_runtimes=(browser-wasm linux-mips64 linux-s390x linux-ppc64le)
-
-# Dedicated Runtimes
-
-for cfg in "${dedicated_runtimes[@]}"; do
-  "/mnt/c/Program Files/dotnet/dotnet.exe" publish Desktop -c Release -r "$cfg" -o "./bin/$cfg" --self-contained true
-  makezip "$cfg"
-done
-
-# Portable
-
-"/mnt/c/Program Files/dotnet/dotnet.exe" publish Desktop -c Release -o "./bin/portable" -p:PublishSingleFile=false -p:PublishTrimmed=false --self-contained false
-
-rm "./bin/portable/DiscordHistoryTracker.exe"
-
-for runtime in "${skipped_portable_runtimes[@]}"; do
-  rm -rf "./bin/portable/runtimes/$runtime"
-done
-
-makezip "portable"

app/global.json

@@ -1,6 +1,6 @@
 {
     "sdk": {
-        "version": "9.0.0",
+        "version": "8.0.0",
         "rollForward": "latestMinor"
     }
 }