1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-04-11 11:15:42 +02:00

Compare commits

...

2 Commits

Author SHA1 Message Date
e30b305eb5
Add option to export downloaded files
Closes #277
2025-03-17 17:27:12 +01:00
fa17d0e224
Fix exception handling in progress dialogs 2025-03-17 17:27:12 +01:00
15 changed files with 408 additions and 154 deletions

View File

@ -8,6 +8,10 @@ using Avalonia.Platform.Storage;
namespace DHT.Desktop.Dialogs.File;
static class FileDialogs {
public static async Task<string[]> OpenFolders(this IStorageProvider storageProvider, FolderPickerOpenOptions options) {
return (await storageProvider.OpenFolderPickerAsync(options)).ToLocalPaths();
}
public static async Task<string[]> OpenFiles(this IStorageProvider storageProvider, FilePickerOpenOptions options) {
return (await storageProvider.OpenFilePickerAsync(options)).ToLocalPaths();
}
@ -26,11 +30,11 @@ static class FileDialogs {
return suggestedDirectory == null ? Task.FromResult<IStorageFolder?>(null) : window.StorageProvider.TryGetFolderFromPathAsync(suggestedDirectory);
}
private static string ToLocalPath(this IStorageFile file) {
return file.TryGetLocalPath() ?? throw new NotSupportedException("Local filesystem is not supported.");
private static string ToLocalPath(this IStorageItem itme) {
return itme.TryGetLocalPath() ?? throw new NotSupportedException("Local filesystem is not supported.");
}
private static string[] ToLocalPaths(this IReadOnlyList<IStorageFile> files) {
return files.Select(ToLocalPath).ToArray();
private static string[] ToLocalPaths(this IReadOnlyList<IStorageItem> items) {
return items.Select(ToLocalPath).ToArray();
}
}

View File

@ -3,7 +3,7 @@ using System.Threading.Tasks;
namespace DHT.Desktop.Dialogs.Progress;
interface IProgressCallback {
Task Update(string message, int finishedItems, int totalItems);
Task Update(string message, long finishedItems, long totalItems);
Task UpdateIndeterminate(string message);
Task Hide();
}

View File

@ -2,7 +2,6 @@ using System;
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Avalonia.Controls;
using DHT.Desktop.Dialogs.Message;
using DHT.Utils.Logging;
namespace DHT.Desktop.Dialogs.Progress;
@ -12,57 +11,37 @@ public sealed partial class ProgressDialog : Window {
private static readonly Log Log = Log.ForType<ProgressDialog>();
internal static async Task Show(Window owner, string title, Func<ProgressDialog, IProgressCallback, Task> action) {
var taskCompletionSource = new TaskCompletionSource();
var dialog = new ProgressDialog();
dialog.DataContext = new ProgressDialogModel(title, async callbacks => {
try {
await action(dialog, callbacks[0]);
taskCompletionSource.SetResult();
} catch (Exception e) {
taskCompletionSource.SetException(e);
}
});
dialog.DataContext = new ProgressDialogModel(title, async callbacks => await action(dialog, callbacks[0]));
await dialog.ShowProgressDialog(owner);
await taskCompletionSource.Task;
}
internal static async Task ShowIndeterminate(Window owner, string title, string message, Func<ProgressDialog, Task> action) {
var taskCompletionSource = new TaskCompletionSource();
var dialog = new ProgressDialog();
dialog.DataContext = new ProgressDialogModel(title, async callbacks => {
await callbacks[0].UpdateIndeterminate(message);
try {
await action(dialog);
taskCompletionSource.SetResult();
} catch (Exception e) {
taskCompletionSource.SetException(e);
}
});
await dialog.ShowProgressDialog(owner);
await taskCompletionSource.Task;
}
internal static async Task<T> ShowIndeterminate<T>(Window owner, string title, string message, Func<ProgressDialog, Task<T>> action) {
internal static async Task<T> Show<T>(Window owner, string title, Func<ProgressDialog, IProgressCallback, Task<T>> action) {
var taskCompletionSource = new TaskCompletionSource<T>();
var dialog = new ProgressDialog();
dialog.DataContext = new ProgressDialogModel(title, async callbacks => {
await callbacks[0].UpdateIndeterminate(message);
try {
taskCompletionSource.SetResult(await action(dialog));
} catch (Exception e) {
taskCompletionSource.SetException(e);
}
taskCompletionSource.SetResult(await action(dialog, callbacks[0]));
});
await dialog.ShowProgressDialog(owner);
return await taskCompletionSource.Task;
}
internal static Task ShowIndeterminate(Window owner, string title, string message, Func<ProgressDialog, Task> action) {
return Show(owner, title, async (dialog, callback) => {
await callback.UpdateIndeterminate(message);
await action(dialog);
});
}
internal static Task<T> ShowIndeterminate<T>(Window owner, string title, string message, Func<ProgressDialog, Task<T>> action) {
return Show<T>(owner, title, async (dialog, callback) => {
await callback.UpdateIndeterminate(message);
return await action(dialog);
});
}
private bool isFinished = false;
private Task progressTask = Task.CompletedTask;
@ -88,11 +67,6 @@ public sealed partial class ProgressDialog : Window {
public async Task ShowProgressDialog(Window owner) {
await ShowDialog(owner);
try {
await progressTask;
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(owner, "Unexpected Error", e.Message);
}
await progressTask;
}
}

View File

@ -38,7 +38,7 @@ sealed class ProgressDialogModel {
this.item = item;
}
public async Task Update(string message, int finishedItems, int totalItems) {
public async Task Update(string message, long finishedItems, long totalItems) {
await Dispatcher.UIThread.InvokeAsync(() => {
item.Message = message;
item.Items = totalItems == 0 ? string.Empty : finishedItems.Format() + " / " + totalItems.Format();

View File

@ -23,7 +23,7 @@ sealed partial class ProgressItem : ObservableObject {
private string items = "";
[ObservableProperty]
private int progress = 0;
private long progress = 0L;
[ObservableProperty]
private bool isIndeterminate;

View File

@ -74,15 +74,28 @@ sealed class DatabasePageModel {
public async Task MergeWithDatabase() {
string[] paths = await DatabaseGui.NewOpenDatabaseFilesDialog(window, Path.GetDirectoryName(Db.Path));
if (paths.Length > 0) {
await ProgressDialog.Show(window, "Database Merge", async (dialog, callback) => await MergeWithDatabaseFromPaths(Db, paths, dialog, callback));
if (paths.Length == 0) {
return;
}
const string Title = "Database Merge";
ImportResult? result;
try {
result = await ProgressDialog.Show(window, Title, async (dialog, callback) => await MergeWithDatabaseFromPaths(Db, paths, dialog, callback));
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, Title, "Could not merge databases: " + e.Message);
return;
}
await Dialog.ShowOk(window, Title, GetImportDialogMessage(result, "database file"));
}
private static async Task MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
private static async Task<ImportResult?> MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
var schemaUpgradeCallbacks = new SchemaUpgradeCallbacks(dialog, paths.Length);
await PerformImport(target, paths, dialog, callback, "Database Merge", "Database Error", "database file", async path => {
return await PerformImport(target, paths, dialog, callback, "Database Merge", async path => {
IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, schemaUpgradeCallbacks);
if (db == null) {
@ -137,15 +150,28 @@ sealed class DatabasePageModel {
AllowMultiple = true,
});
if (paths.Length > 0) {
await ProgressDialog.Show(window, "Legacy Archive Import", async (dialog, callback) => await ImportLegacyArchiveFromPaths(Db, paths, dialog, callback));
if (paths.Length == 0) {
return;
}
const string Title = "Legacy Archive Import";
ImportResult? result;
try {
result = await ProgressDialog.Show(window, Title, async (dialog, callback) => await ImportLegacyArchiveFromPaths(Db, paths, dialog, callback));
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, Title, "Could not import legacy archives: " + e.Message);
return;
}
await Dialog.ShowOk(window, Title, GetImportDialogMessage(result, "archive file"));
}
private static async Task ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
private static async Task<ImportResult?> ImportLegacyArchiveFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
var fakeSnowflake = new FakeSnowflake();
await PerformImport(target, paths, dialog, callback, "Legacy Archive Import", "Legacy Archive Error", "archive file", async path => {
return await PerformImport(target, paths, dialog, callback, "Legacy Archive Import", async path => {
await using var jsonStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
return await LegacyArchiveImport.Read(jsonStream, target, fakeSnowflake, async servers => {
@ -189,7 +215,7 @@ sealed class DatabasePageModel {
.ToDictionary(static item => item.Item, static item => ulong.Parse(item.Value));
}
private static async Task PerformImport(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback, string neutralDialogTitle, string errorDialogTitle, string itemName, Func<string, Task<bool>> performImport) {
private static async Task<ImportResult?> PerformImport(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback, string dialogTitle, Func<string, Task<bool>> performImport) {
int total = paths.Length;
DatabaseStatistics oldStatistics = await DatabaseStatistics.Take(target);
@ -201,7 +227,7 @@ sealed class DatabasePageModel {
++finished;
if (!File.Exists(path)) {
await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' no longer exists.");
await Dialog.ShowOk(dialog, dialogTitle, "File '" + Path.GetFileName(path) + "' no longer exists.");
continue;
}
@ -211,19 +237,18 @@ sealed class DatabasePageModel {
}
} catch (Exception ex) {
Log.Error(ex);
await Dialog.ShowOk(dialog, errorDialogTitle, "File '" + Path.GetFileName(path) + "' could not be imported: " + ex.Message);
await Dialog.ShowOk(dialog, dialogTitle, "File '" + Path.GetFileName(path) + "' could not be imported: " + ex.Message);
}
}
await callback.Update("Done", finished, total);
if (successful == 0) {
await Dialog.ShowOk(dialog, neutralDialogTitle, "Nothing was imported.");
return;
return null;
}
DatabaseStatistics newStatistics = await DatabaseStatistics.Take(target);
await Dialog.ShowOk(dialog, neutralDialogTitle, GetImportDialogMessage(oldStatistics, newStatistics, successful, total, itemName));
return new ImportResult(oldStatistics, newStatistics, successful, total);
}
private sealed record DatabaseStatistics(long ServerCount, long ChannelCount, long UserCount, long MessageCount) {
@ -237,7 +262,16 @@ sealed class DatabasePageModel {
}
}
private static string GetImportDialogMessage(DatabaseStatistics oldStatistics, DatabaseStatistics newStatistics, int successfulItems, int totalItems, string itemName) {
private sealed record ImportResult(DatabaseStatistics OldStatistics, DatabaseStatistics NewStatistics, int SuccessfulItems, int TotalItems);
private static string GetImportDialogMessage(ImportResult? result, string itemName) {
if (result == null) {
return "Nothing was imported.";
}
var oldStatistics = result.OldStatistics;
var newStatistics = result.NewStatistics;
long newServers = newStatistics.ServerCount - oldStatistics.ServerCount;
long newChannels = newStatistics.ChannelCount - oldStatistics.ChannelCount;
long newUsers = newStatistics.UserCount - oldStatistics.UserCount;
@ -246,11 +280,11 @@ sealed class DatabasePageModel {
var message = new StringBuilder();
message.Append("Processed ");
if (successfulItems == totalItems) {
message.Append(successfulItems.Pluralize(itemName));
if (result.SuccessfulItems == result.TotalItems) {
message.Append(result.SuccessfulItems.Pluralize(itemName));
}
else {
message.Append(successfulItems.Format()).Append(" out of ").Append(totalItems.Pluralize(itemName));
message.Append(result.SuccessfulItems.Format()).Append(" out of ").Append(result.TotalItems.Pluralize(itemName));
}
message.Append(" and added:\n\n \u2022 ");
@ -264,7 +298,15 @@ sealed class DatabasePageModel {
public async Task VacuumDatabase() {
const string Title = "Vacuum Database";
await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => Db.Vacuum());
try {
await ProgressDialog.ShowIndeterminate(window, Title, "Vacuuming database...", _ => Db.Vacuum());
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, Title, "Could not vacuum database: " + e.Message);
return;
}
await Dialog.ShowOk(window, Title, "Done.");
}
}

View File

@ -51,7 +51,7 @@ sealed class DebugPageModel {
private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) {
int batchCount = (messageCount + BatchSize - 1) / BatchSize;
await callback.Update("Adding messages in batches of " + BatchSize, finishedItems: 0, batchCount);
await callback.Update("Adding messages in batches of " + BatchSize, finishedItems: 0, totalItems: batchCount);
var rand = new Random();
var server = new DHT.Server.Data.Server {

View File

@ -33,8 +33,9 @@
<StackPanel Orientation="Vertical">
<WrapPanel Orientation="Horizontal">
<Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" />
<Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed Downloads</Button>
<Button Command="{Binding OnClickDeleteOrphanedDownloads}">Delete Orphaned Downloads</Button>
<Button Command="{Binding OnClickRetryFailed}" IsEnabled="{Binding IsRetryFailedOnDownloadsButtonEnabled}">Retry Failed</Button>
<Button Command="{Binding OnClickDeleteOrphaned}">Delete Orphaned</Button>
<Button Command="{Binding OnClickExportAll}" IsEnabled="{Binding HasSuccessfulDownloads}">Export All</Button>
</WrapPanel>
<StackPanel Orientation="Vertical" Spacing="20" Margin="0 10 0 0">
<controls:DownloadItemFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !$parent[UserControl].((pages:DownloadsPageModel)DataContext).IsDownloading}" />

View File

@ -4,9 +4,11 @@ using System.Collections.ObjectModel;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Avalonia.Controls;
using Avalonia.Platform.Storage;
using Avalonia.ReactiveUI;
using CommunityToolkit.Mvvm.ComponentModel;
using DHT.Desktop.Common;
using DHT.Desktop.Dialogs.File;
using DHT.Desktop.Dialogs.Message;
using DHT.Desktop.Dialogs.Progress;
using DHT.Desktop.Main.Controls;
@ -33,6 +35,9 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
private bool isRetryingFailedDownloads = false;
[ObservableProperty(Setter = Access.Private)]
private bool hasSuccessfulDownloads;
[ObservableProperty(Setter = Access.Private)]
[NotifyPropertyChangedFor(nameof(IsRetryFailedOnDownloadsButtonEnabled))]
private bool hasFailedDownloads;
@ -148,7 +153,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
RecomputeDownloadStatistics();
}
public async Task OnClickRetryFailedDownloads() {
public async Task OnClickRetryFailed() {
IsRetryingFailedDownloads = true;
try {
@ -165,46 +170,94 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
downloadStatisticsTask.Post(cancellationToken => state.Db.Downloads.GetStatistics(currentDownloadFilter ?? new DownloadItemFilter(), cancellationToken));
}
public async Task OnClickDeleteOrphanedDownloads() {
public async Task OnClickDeleteOrphaned() {
const string Title = "Delete Orphaned Downloads";
await ProgressDialog.Show(window, Title, async (_, callback) => {
await callback.UpdateIndeterminate("Searching for orphaned downloads...");
HashSet<string> reachableNormalizedUrls = [];
HashSet<string> orphanedNormalizedUrls = [];
await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
reachableNormalizedUrls.Add(download.NormalizedUrl);
}
await foreach (Download download in state.Db.Downloads.Get()) {
string normalizedUrl = download.NormalizedUrl;
if (!reachableNormalizedUrls.Contains(normalizedUrl)) {
orphanedNormalizedUrls.Add(normalizedUrl);
try {
await ProgressDialog.Show(window, Title, async (_, callback) => {
await callback.UpdateIndeterminate("Searching for orphaned downloads...");
HashSet<string> reachableNormalizedUrls = [];
HashSet<string> orphanedNormalizedUrls = [];
await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
reachableNormalizedUrls.Add(download.NormalizedUrl);
}
}
if (orphanedNormalizedUrls.Count == 0) {
await Dialog.ShowOk(window, Title, "No orphaned downloads found.");
return;
}
if (await Dialog.ShowYesNo(window, Title, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
return;
}
await callback.UpdateIndeterminate("Deleting orphaned downloads...");
await state.Db.Downloads.Remove(orphanedNormalizedUrls);
RecomputeDownloadStatistics();
if (await Dialog.ShowYesNo(window, Title, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
return;
}
await callback.UpdateIndeterminate("Vacuuming database...");
await state.Db.Vacuum();
await foreach (Download download in state.Db.Downloads.Get()) {
string normalizedUrl = download.NormalizedUrl;
if (!reachableNormalizedUrls.Contains(normalizedUrl)) {
orphanedNormalizedUrls.Add(normalizedUrl);
}
}
if (orphanedNormalizedUrls.Count == 0) {
await Dialog.ShowOk(window, Title, "No orphaned downloads found.");
return;
}
if (await Dialog.ShowYesNo(window, Title, orphanedNormalizedUrls.Count + " orphaned download(s) will be removed from this database. This action cannot be undone. Proceed?") != DialogResult.YesNo.Yes) {
return;
}
await callback.UpdateIndeterminate("Deleting orphaned downloads...");
await state.Db.Downloads.Remove(orphanedNormalizedUrls);
RecomputeDownloadStatistics();
if (await Dialog.ShowYesNo(window, Title, "Orphaned downloads deleted. Vacuum database now to reclaim space?") != DialogResult.YesNo.Yes) {
return;
}
await callback.UpdateIndeterminate("Vacuuming database...");
await state.Db.Vacuum();
});
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, Title, "Could not delete orphaned downloads: " + e.Message);
}
}
public async Task OnClickExportAll() {
const string Title = "Export Downloaded Files";
string[] folders = await window.StorageProvider.OpenFolders(new FolderPickerOpenOptions {
Title = Title,
AllowMultiple = false,
});
if (folders.Length != 1) {
return;
}
string folderPath = folders[0];
DownloadExporter exporter = new DownloadExporter(state.Db, folderPath);
DownloadExporter.Result result;
try {
result = await ProgressDialog.Show(window, Title, async (_, callback) => {
await callback.UpdateIndeterminate("Exporting downloaded files...");
return await exporter.Export(new ExportProgressReporter(callback));
});
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, Title, "Could not export downloaded files: " + e.Message);
return;
}
string messageStart = "Exported " + result.SuccessfulCount.Pluralize("file");
if (result.FailedCount > 0L) {
await Dialog.ShowOk(window, Title, messageStart + " (" + result.FailedCount.Format() + " failed).");
}
else {
await Dialog.ShowOk(window, Title, messageStart + ".");
}
}
private sealed class ExportProgressReporter(IProgressCallback callback) : DownloadExporter.IProgressReporter {
public Task ReportProgress(long processedCount, long totalCount) {
return callback.Update("Exporting downloaded files...", processedCount, totalCount);
}
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
@ -224,6 +277,7 @@ sealed partial class DownloadsPageModel : ObservableObject, IAsyncDisposable {
statisticsSkipped.Size = statusStatistics.SkippedTotalSize;
statisticsSkipped.HasFilesWithUnknownSize = statusStatistics.SkippedWithUnknownSizeCount > 0;
HasSuccessfulDownloads = statusStatistics.SuccessfulCount > 0;
HasFailedDownloads = statusStatistics.FailedCount > 0;
}

View File

@ -61,18 +61,23 @@ sealed partial class ViewerPageModel : ObservableObject, IDisposable {
}
public async Task OnClickApplyFiltersToDatabase() {
MessageFilter filter = FilterModel.CreateFilter();
long messageCount = await ProgressDialog.ShowIndeterminate(window, "Apply Filters", "Counting matching messages...", _ => state.Db.Messages.Count(filter));
if (DatabaseToolFilterModeKeep) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", messageCount.Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
await ApplyFilterToDatabase(filter, FilterRemovalMode.KeepMatching);
try {
MessageFilter filter = FilterModel.CreateFilter();
long messageCount = await ProgressDialog.ShowIndeterminate(window, "Apply Filters", "Counting matching messages...", _ => state.Db.Messages.Count(filter));
if (DatabaseToolFilterModeKeep) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", messageCount.Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
await ApplyFilterToDatabase(filter, FilterRemovalMode.KeepMatching);
}
}
}
else if (DatabaseToolFilterModeRemove) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", messageCount.Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
await ApplyFilterToDatabase(filter, FilterRemovalMode.RemoveMatching);
else if (DatabaseToolFilterModeRemove) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", messageCount.Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
await ApplyFilterToDatabase(filter, FilterRemovalMode.RemoveMatching);
}
}
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, "Apply Filters", "Could not apply filters: " + e.Message);
}
}

View File

@ -112,38 +112,32 @@ sealed partial class WelcomeScreenModel : ObservableObject {
}
public async Task CheckUpdates() {
var latestVersion = await ProgressDialog.ShowIndeterminate<Version?>(window, "Check Updates", "Checking for updates...", async _ => {
var client = new HttpClient(new SocketsHttpHandler {
AutomaticDecompression = DecompressionMethods.None,
AllowAutoRedirect = false,
UseCookies = false,
string response;
try {
response = await ProgressDialog.ShowIndeterminate<string>(window, "Check Updates", "Checking for updates...", static async _ => {
var client = new HttpClient(new SocketsHttpHandler {
AutomaticDecompression = DecompressionMethods.None,
AllowAutoRedirect = false,
UseCookies = false,
});
client.Timeout = TimeSpan.FromSeconds(30);
client.MaxResponseContentBufferSize = 1024;
client.DefaultRequestHeaders.UserAgent.ParseAdd("DiscordHistoryTracker/" + Program.Version);
return await client.GetStringAsync(Program.Website + "/version");
});
client.Timeout = TimeSpan.FromSeconds(30);
client.MaxResponseContentBufferSize = 1024;
client.DefaultRequestHeaders.UserAgent.ParseAdd("DiscordHistoryTracker/" + Program.Version);
string response;
try {
response = await client.GetStringAsync(Program.Website + "/version");
} catch (TaskCanceledException e) when (e.InnerException is TimeoutException) {
await Dialog.ShowOk(window, "Check Updates", "Request timed out.");
return null;
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, "Check Updates", "Error checking for updates: " + e.Message);
return null;
}
if (!System.Version.TryParse(response, out Version? latestVersion)) {
await Dialog.ShowOk(window, "Check Updates", "Server returned an invalid response.");
return null;
}
return latestVersion;
});
} catch (TaskCanceledException e) when (e.InnerException is TimeoutException) {
await Dialog.ShowOk(window, "Check Updates", "Request timed out.");
return;
} catch (Exception e) {
Log.Error(e);
await Dialog.ShowOk(window, "Check Updates", "Error checking for updates: " + e.Message);
return;
}
if (latestVersion == null) {
if (!System.Version.TryParse(response, out Version? latestVersion)) {
await Dialog.ShowOk(window, "Check Updates", "Server returned an invalid response.");
return;
}

View File

@ -20,7 +20,7 @@ public interface IDownloadRepository {
Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken = default);
IAsyncEnumerable<Data.Download> Get();
IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter = null);
Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor);
@ -51,7 +51,7 @@ public interface IDownloadRepository {
return Task.FromResult(new DownloadStatusStatistics());
}
public IAsyncEnumerable<Data.Download> Get() {
public IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter) {
return AsyncEnumerable.Empty<Data.Download>();
}

View File

@ -201,10 +201,10 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
};
}
public async IAsyncEnumerable<Data.Download> Get() {
public async IAsyncEnumerable<Data.Download> Get(DownloadItemFilter? filter) {
await using var conn = await pool.Take();
await using var cmd = conn.Command("SELECT normalized_url, download_url, status, type, size FROM download_metadata");
await using var cmd = conn.Command("SELECT normalized_url, download_url, status, type, size FROM download_metadata" + filter.GenerateConditions().BuildWhereClause());
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) {

View File

@ -0,0 +1,152 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Tasks;
using Channel = System.Threading.Channels.Channel;
namespace DHT.Server.Download;
public sealed partial class DownloadExporter(IDatabaseFile db, string folderPath) {
private static readonly Log Log = Log.ForType<DownloadExporter>();
private const int Concurrency = 3;
private static Channel<Data.Download> CreateExportChannel() {
return Channel.CreateBounded<Data.Download>(new BoundedChannelOptions(Concurrency * 4) {
SingleWriter = true,
SingleReader = false,
AllowSynchronousContinuations = true,
FullMode = BoundedChannelFullMode.Wait,
});
}
public interface IProgressReporter {
Task ReportProgress(long processedCount, long totalCount);
}
public readonly record struct Result(long SuccessfulCount, long FailedCount) {
internal static Result Combine(Result left, Result right) {
return new Result(left.SuccessfulCount + right.SuccessfulCount, left.FailedCount + right.FailedCount);
}
}
public async Task<Result> Export(IProgressReporter reporter) {
DownloadItemFilter filter = new DownloadItemFilter {
IncludeStatuses = [DownloadStatus.Success]
};
long totalCount = await db.Downloads.Count(filter);
Channel<Data.Download> channel = CreateExportChannel();
ExportRunner exportRunner = new ExportRunner(db, folderPath, channel.Reader, reporter, totalCount);
using CancellableTask progressTask = CancellableTask.Run(exportRunner.RunReportTask);
List<Task<Result>> readerTasks = [];
for (int reader = 0; reader < Concurrency; reader++) {
readerTasks.Add(Task.Run(exportRunner.RunExportTask, CancellationToken.None));
}
await foreach (Data.Download download in db.Downloads.Get(filter).WithCancellation(CancellationToken.None)) {
await channel.Writer.WriteAsync(download, CancellationToken.None);
}
channel.Writer.Complete();
Result result = (await Task.WhenAll(readerTasks)).Aggregate(Result.Combine);
progressTask.Cancel();
await progressTask.Task;
return result;
}
private sealed partial class ExportRunner(IDatabaseFile db, string folderPath, ChannelReader<Data.Download> reader, IProgressReporter reporter, long totalCount) {
private long processedCount;
public async Task RunReportTask(CancellationToken cancellationToken) {
try {
while (true) {
await reporter.ReportProgress(processedCount, totalCount);
await Task.Delay(TimeSpan.FromMilliseconds(25), cancellationToken);
}
} catch (OperationCanceledException) {
await reporter.ReportProgress(processedCount, totalCount);
}
}
public async Task<Result> RunExportTask() {
long successfulCount = 0L;
long failedCount = 0L;
await foreach (Data.Download download in reader.ReadAllAsync()) {
bool success;
try {
success = await db.Downloads.GetDownloadData(download.NormalizedUrl, stream => CopyToFile(download.NormalizedUrl, stream));
} catch (FileAlreadyExistsException) {
success = false;
} catch (Exception e) {
Log.Error(e);
success = false;
}
if (success) {
++successfulCount;
}
else {
++failedCount;
}
Interlocked.Increment(ref processedCount);
}
return new Result(successfulCount, failedCount);
}
private async Task CopyToFile(string normalizedUrl, Stream blobStream) {
string fileName = UrlToFileName(normalizedUrl);
string filePath = Path.Combine(folderPath, fileName);
if (File.Exists(filePath)) {
Log.Error("Skipping existing file: " + fileName);
throw FileAlreadyExistsException.Instance;
}
await using var fileStream = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
await blobStream.CopyToAsync(fileStream);
}
[GeneratedRegex("[^a-zA-Z0-9_.-]")]
private static partial Regex DisallowedFileNameCharactersRegex();
private static string UrlToFileName(string url) {
static string UriToFileName(Uri uri) {
string fileName = uri.AbsolutePath.TrimStart('/');
if (uri.Query.Length > 0) {
int periodIndex = fileName.LastIndexOf('.');
return fileName.Insert(periodIndex == -1 ? fileName.Length : periodIndex, uri.Query.TrimEnd('&'));
}
else {
return fileName;
}
}
string fileName = Uri.TryCreate(url, UriKind.Absolute, out var uri) ? UriToFileName(uri) : url;
return DisallowedFileNameCharactersRegex().Replace(fileName, "_");
}
}
private sealed class FileAlreadyExistsException : Exception {
public static FileAlreadyExistsException Instance { get; } = new ();
}
}

View File

@ -0,0 +1,28 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace DHT.Utils.Tasks;
public sealed class CancellableTask : IDisposable {
public static CancellableTask Run(Func<CancellationToken, Task> action) {
return new CancellableTask(action);
}
public Task Task { get; }
private readonly CancellationTokenSource cancellationTokenSource = new ();
private CancellableTask(Func<CancellationToken, Task> action) {
CancellationToken cancellationToken = cancellationTokenSource.Token;
Task = Task.Run(() => action(cancellationToken));
}
public void Cancel() {
cancellationTokenSource.Cancel();
}
public void Dispose() {
cancellationTokenSource.Dispose();
}
}