1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-10-29 13:24:23 +01:00

6 Commits

26 changed files with 309 additions and 223 deletions

View File

@@ -0,0 +1,17 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using DHT.Utils.Observables;
namespace DHT.Desktop.Common;
static class AvaloniaObsevableValueExtensions {
public static IDisposable SubscribeLastOnUI<T>(this ObservableValue<T> observable, Action<T> action, TimeSpan delayBetweenRuns) {
Task Action(T value, CancellationToken cancellationToken) {
action(value);
return Task.Delay(delayBetweenRuns, cancellationToken);
}
return observable.SubscribeLast(Action, TaskScheduler.FromCurrentSynchronizationContext());
}
}

View File

@@ -2,9 +2,7 @@ using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Avalonia.ReactiveUI;
using DHT.Desktop.Common;
using DHT.Server;
using DHT.Server.Data.Filters;
@@ -49,7 +47,7 @@ sealed partial class DownloadItemFilterPanelModel : IAsyncDisposable {
private readonly State state;
private readonly string verb;
private readonly DelayedThrottledTask<FilterSettings> saveFilterSettingsTask;
private readonly ThrottledTask<FilterSettings> saveFilterSettingsTask;
private bool isLoadingFilterSettings;
private readonly RestartableTask<long> downloadItemCountTask;
@@ -65,10 +63,10 @@ sealed partial class DownloadItemFilterPanelModel : IAsyncDisposable {
this.state = state;
this.verb = verb;
this.saveFilterSettingsTask = new DelayedThrottledTask<FilterSettings>(Log, TimeSpan.FromSeconds(5), SaveFilterSettings);
this.saveFilterSettingsTask = new ThrottledTask<FilterSettings>(Log, SaveFilterSettings, TimeSpan.FromSeconds(5), TaskScheduler.Default);
this.downloadItemCountTask = new RestartableTask<long>(SetMatchingCount, TaskScheduler.FromCurrentSynchronizationContext());
this.downloadItemCountSubscription = state.Db.Downloads.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnDownloadItemCountChanged);
this.downloadItemCountSubscription = state.Db.Downloads.TotalCount.SubscribeLastOnUI(OnDownloadItemCountChanged, TimeSpan.FromMilliseconds(15));
UpdateFilterStatistics();

View File

@@ -4,10 +4,8 @@ using System.Collections.Immutable;
using System.ComponentModel;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Avalonia.Controls;
using Avalonia.ReactiveUI;
using DHT.Desktop.Common;
using DHT.Desktop.Dialogs.CheckBox;
using DHT.Desktop.Dialogs.Message;
@@ -91,9 +89,9 @@ sealed partial class MessageFilterPanelModel : IDisposable {
this.exportedMessageCountTask = new RestartableTask<long>(SetExportedMessageCount, TaskScheduler.FromCurrentSynchronizationContext());
this.messageCountSubscription = state.Db.Messages.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnMessageCountChanged);
this.channelCountSubscription = state.Db.Channels.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnChannelCountChanged);
this.userCountSubscription = state.Db.Users.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnUserCountChanged);
this.messageCountSubscription = state.Db.Messages.TotalCount.SubscribeLastOnUI(OnMessageCountChanged, TimeSpan.FromMilliseconds(15));
this.channelCountSubscription = state.Db.Channels.TotalCount.SubscribeLastOnUI(OnChannelCountChanged, TimeSpan.FromMilliseconds(15));
this.userCountSubscription = state.Db.Users.TotalCount.SubscribeLastOnUI(OnUserCountChanged, TimeSpan.FromMilliseconds(15));
UpdateFilterStatistics();
UpdateChannelFilterLabel();

View File

@@ -1,7 +1,6 @@
using System;
using System.Reactive.Linq;
using Avalonia.ReactiveUI;
using Avalonia.Threading;
using DHT.Desktop.Common;
using DHT.Server;
using DHT.Server.Service;
using PropertyChanged.SourceGenerator;
@@ -41,9 +40,9 @@ sealed partial class StatusBarModel : IDisposable {
public StatusBarModel(State state) {
this.state = state;
serverCountSubscription = state.Db.Servers.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(newServerCount => ServerCount = newServerCount);
channelCountSubscription = state.Db.Channels.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(newChannelCount => ChannelCount = newChannelCount);
messageCountSubscription = state.Db.Messages.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(newMessageCount => MessageCount = newMessageCount);
serverCountSubscription = state.Db.Servers.TotalCount.SubscribeLastOnUI(newServerCount => ServerCount = newServerCount, TimeSpan.FromMilliseconds(15));
channelCountSubscription = state.Db.Channels.TotalCount.SubscribeLastOnUI(newChannelCount => ChannelCount = newChannelCount, TimeSpan.FromMilliseconds(15));
messageCountSubscription = state.Db.Messages.TotalCount.SubscribeLastOnUI(newMessageCount => MessageCount = newMessageCount, TimeSpan.FromMilliseconds(15));
state.Server.StatusChanged += OnStateServerStatusChanged;
serverStatus = state.Server.IsRunning ? ServerManager.Status.Started : ServerManager.Status.Stopped;

View File

@@ -55,11 +55,11 @@ sealed class DatabasePageModel {
break;
case PlatformID.Unix:
Process.Start("xdg-open", [ folder ]);
Process.Start("xdg-open", [folder]);
break;
case PlatformID.MacOSX:
Process.Start("open", [ folder ]);
Process.Start("open", [folder]);
break;
default:
@@ -80,22 +80,25 @@ sealed class DatabasePageModel {
const string Title = "Database Merge";
ImportResult? result;
var result = new TaskCompletionSource<ImportResult?>();
try {
result = await ProgressDialog.Show(window, Title, async (dialog, callback) => await MergeWithDatabaseFromPaths(Db, paths, dialog, callback));
var dialog = new ProgressDialog();
dialog.DataContext = new ProgressDialogModel(Title, async callbacks => result.SetResult(await MergeWithDatabaseFromPaths(Db, paths, dialog, callbacks)), progressItems: 2);
await dialog.ShowProgressDialog(window);
} catch (Exception e) {
Log.Error("Could not merge databases.", e);
await Dialog.ShowOk(window, Title, "Could not merge databases: " + e.Message);
return;
}
await Dialog.ShowOk(window, Title, GetImportDialogMessage(result, "database file"));
await Dialog.ShowOk(window, Title, GetImportDialogMessage(result.Task.Result, "database file"));
}
private static async Task<ImportResult?> MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IProgressCallback callback) {
var schemaUpgradeCallbacks = new SchemaUpgradeCallbacks(dialog, paths.Length);
private static async Task<ImportResult?> MergeWithDatabaseFromPaths(IDatabaseFile target, string[] paths, ProgressDialog dialog, IReadOnlyList<IProgressCallback> callbacks) {
var schemaUpgradeCallbacks = new SchemaUpgradeCallbacks(dialog, callbacks[1], paths.Length);
var databaseMergeProgressCallback = new DatabaseMergeProgressCallback(callbacks[1]);
return await PerformImport(target, paths, dialog, callback, "Database Merge", async path => {
return await PerformImport(target, paths, dialog, callbacks[0], "Database Merge", async path => {
IDatabaseFile? db = await DatabaseGui.TryOpenOrCreateDatabaseFromPath(path, dialog, schemaUpgradeCallbacks);
if (db == null) {
@@ -103,7 +106,7 @@ sealed class DatabasePageModel {
}
try {
await target.AddFrom(db);
await target.Merge(db, databaseMergeProgressCallback);
return true;
} finally {
await db.DisposeAsync();
@@ -111,7 +114,7 @@ sealed class DatabasePageModel {
});
}
private sealed class SchemaUpgradeCallbacks(ProgressDialog dialog, int total) : ISchemaUpgradeCallbacks {
private sealed class SchemaUpgradeCallbacks(ProgressDialog dialog, IProgressCallback callback, int total) : ISchemaUpgradeCallbacks {
private bool? decision;
public Task<InitialDatabaseSettings?> GetInitialDatabaseSettings() {
@@ -125,6 +128,7 @@ sealed class DatabasePageModel {
}
public Task Start(int versionSteps, Func<ISchemaUpgradeCallbacks.IProgressReporter, Task> doUpgrade) {
callback.UpdateIndeterminate("Upgrading database...");
return doUpgrade(new NullReporter());
}
@@ -143,6 +147,20 @@ sealed class DatabasePageModel {
}
}
private sealed class DatabaseMergeProgressCallback(IProgressCallback callback) : DatabaseMerging.IProgressCallback {
public void OnImportingMetadata() {
callback.UpdateIndeterminate("Importing metadata...");
}
public void OnMessagesImported(long finished, long total) {
callback.Update("Importing messages...", finished, total);
}
public void OnDownloadsImported(long finished, long total) {
callback.Update("Importing downloaded files...", finished, total);
}
}
public async Task ImportLegacyArchive() {
string[] paths = await window.StorageProvider.OpenFiles(new FilePickerOpenOptions {
Title = "Open Legacy DHT Archive",
@@ -223,7 +241,7 @@ sealed class DatabasePageModel {
int finished = 0;
foreach (string path in paths) {
await callback.Update(Path.GetFileName(path), finished, total);
await callback.Update("File: " + Path.GetFileName(path), finished, total);
++finished;
if (!File.Exists(path)) {

View File

@@ -2,11 +2,9 @@ using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Avalonia.Controls;
using Avalonia.Platform.Storage;
using Avalonia.ReactiveUI;
using DHT.Desktop.Common;
using DHT.Desktop.Dialogs.File;
using DHT.Desktop.Dialogs.Message;
@@ -19,6 +17,7 @@ using DHT.Server.Data.Filters;
using DHT.Server.Data.Settings;
using DHT.Server.Download;
using DHT.Utils.Logging;
using DHT.Utils.Observables;
using DHT.Utils.Tasks;
using PropertyChanged.SourceGenerator;
@@ -82,8 +81,8 @@ sealed partial class DownloadsPageModel : IAsyncDisposable {
statisticsSkipped,
];
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(Log, UpdateStatistics, TaskScheduler.FromCurrentSynchronizationContext());
downloadItemCountSubscription = state.Db.Downloads.TotalCount.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnDownloadCountChanged);
downloadStatisticsTask = new ThrottledTask<DownloadStatusStatistics>(Log, UpdateStatistics, TimeSpan.FromMilliseconds(100), TaskScheduler.FromCurrentSynchronizationContext());
downloadItemCountSubscription = state.Db.Downloads.TotalCount.SubscribeLastOnUI(OnDownloadCountChanged, TimeSpan.FromMilliseconds(15));
RecomputeDownloadStatistics();
}
@@ -145,8 +144,8 @@ sealed partial class DownloadsPageModel : IAsyncDisposable {
try {
currentDownloadFilter = FilterModel.CreateFilter();
IObservable<DownloadItem> finishedItems = await state.Downloader.Start(currentDownloadFilter);
finishedItemsSubscription = finishedItems.ObserveOn(AvaloniaScheduler.Instance).Subscribe(OnItemFinished);
ObservableValue<DownloadItem> finishedItems = await state.Downloader.Start(currentDownloadFilter);
finishedItemsSubscription = finishedItems.SubscribeLastOnUI(OnItemFinished, TimeSpan.FromMilliseconds(15));
} catch (Exception) {
finishedItemsSubscription?.Dispose();
finishedItemsSubscription = null;
@@ -285,7 +284,7 @@ sealed partial class DownloadsPageModel : IAsyncDisposable {
}
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
private Task UpdateStatistics(DownloadStatusStatistics statusStatistics) {
statisticsPending.Items = statusStatistics.PendingCount;
statisticsPending.Size = statusStatistics.PendingTotalSize;
statisticsPending.HasFilesWithUnknownSize = statusStatistics.PendingWithUnknownSizeCount > 0;
@@ -304,6 +303,8 @@ sealed partial class DownloadsPageModel : IAsyncDisposable {
HasSuccessfulDownloads = statusStatistics.SuccessfulCount > 0;
HasFailedDownloads = statusStatistics.FailedCount > 0;
return Task.CompletedTask;
}
public sealed partial class StatisticsRow(string state) {

View File

@@ -107,7 +107,7 @@ export default (function() {
const isImageUrl = function(url) {
const dot = url.pathname.lastIndexOf(".");
const ext = dot === -1 ? "" : url.pathname.substring(dot).toLowerCase();
return ext === ".png" || ext === ".gif" || ext === ".jpg" || ext === ".jpeg";
return ext === ".png" || ext === ".gif" || ext === ".jpg" || ext === ".jpeg" || ext === ".webp" || ext === ".avif";
};
return {

View File

@@ -1,34 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DHT.Server.Data;
namespace DHT.Server.Database;
public static class DatabaseExtensions {
public static async Task AddFrom(this IDatabaseFile target, IDatabaseFile source) {
await target.Users.Add(await source.Users.Get().ToListAsync());
await target.Servers.Add(await source.Servers.Get().ToListAsync());
await target.Channels.Add(await source.Channels.Get().ToListAsync());
const int MessageBatchSize = 100;
List<Message> batchedMessages = new (MessageBatchSize);
await foreach (Message message in source.Messages.Get()) {
batchedMessages.Add(message);
if (batchedMessages.Count >= MessageBatchSize) {
await target.Messages.Add(batchedMessages);
batchedMessages.Clear();
}
}
await target.Messages.Add(batchedMessages);
await foreach (Data.Download download in source.Downloads.Get()) {
if (download.Status != DownloadStatus.Success || !await source.Downloads.GetDownloadData(download.NormalizedUrl, stream => target.Downloads.AddDownload(download, stream))) {
await target.Downloads.AddDownload(download, stream: null);
}
}
}
}

View File

@@ -0,0 +1,77 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DHT.Server.Data;
namespace DHT.Server.Database.Import;
public static class DatabaseMerging {
public static async Task Merge(this IDatabaseFile target, IDatabaseFile source, IProgressCallback callback) {
// Import downloads first, otherwise automatic downloads would try to re-download files from other imported data.
await MergeDownloads(target, source, callback);
callback.OnImportingMetadata();
await target.Users.Add(await source.Users.Get().ToListAsync());
await target.Servers.Add(await source.Servers.Get().ToListAsync());
await target.Channels.Add(await source.Channels.Get().ToListAsync());
await MergeMessages(target, source, callback);
}
private static async Task MergeDownloads(IDatabaseFile target, IDatabaseFile source, IProgressCallback callback) {
const int ReportBatchSize = 100;
long totalDownloads = await source.Downloads.Count();
long importedDownloads = 0;
callback.OnDownloadsImported(importedDownloads, totalDownloads);
await foreach (Data.Download download in source.Downloads.Get()) {
if (download.Status != DownloadStatus.Success || !await source.Downloads.GetDownloadData(download.NormalizedUrl, stream => target.Downloads.AddDownload(download, stream))) {
await target.Downloads.AddDownload(download, stream: null);
}
if (++importedDownloads % ReportBatchSize == 0) {
callback.OnDownloadsImported(importedDownloads, totalDownloads);
}
}
callback.OnDownloadsImported(totalDownloads, totalDownloads);
}
private static async Task MergeMessages(IDatabaseFile target, IDatabaseFile source, IProgressCallback callback) {
const int MessageBatchSize = 100;
const int ReportEveryBatches = 10;
List<Message> batchedMessages = new (MessageBatchSize);
long totalMessages = await source.Messages.Count();
long importedMessages = 0;
callback.OnMessagesImported(importedMessages, totalMessages);
await foreach (Message message in source.Messages.Get()) {
batchedMessages.Add(message);
if (batchedMessages.Count >= MessageBatchSize) {
await target.Messages.Add(batchedMessages);
importedMessages += batchedMessages.Count;
if (importedMessages % (MessageBatchSize * ReportEveryBatches) == 0) {
callback.OnMessagesImported(importedMessages, totalMessages);
}
batchedMessages.Clear();
}
}
await target.Messages.Add(batchedMessages);
callback.OnMessagesImported(totalMessages, totalMessages);
}
public interface IProgressCallback {
void OnImportingMetadata();
void OnMessagesImported(long finished, long total);
void OnDownloadsImported(long finished, long total);
}
}

View File

@@ -1,15 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Utils.Observables;
namespace DHT.Server.Database.Repositories;
public interface IChannelRepository {
IObservable<long> TotalCount { get; }
ObservableValue<long> TotalCount { get; }
Task Add(IReadOnlyList<Channel> channels);
@@ -20,7 +19,7 @@ public interface IChannelRepository {
Task<int> RemoveUnreachable();
internal sealed class Dummy : IChannelRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public ObservableValue<long> TotalCount { get; } = new (0L);
public Task Add(IReadOnlyList<Channel> channels) {
return Task.CompletedTask;

View File

@@ -2,22 +2,22 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Download;
using DHT.Utils.Observables;
namespace DHT.Server.Database.Repositories;
public interface IDownloadRepository {
IObservable<long> TotalCount { get; }
ObservableValue<long> TotalCount { get; }
Task AddDownload(Data.Download item, Stream? stream);
Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken = default);
Task<long> Count(DownloadItemFilter? filter = null, CancellationToken cancellationToken = default);
Task<DownloadStatusStatistics> GetStatistics(DownloadItemFilter nonSkippedFilter, CancellationToken cancellationToken = default);
@@ -38,13 +38,13 @@ public interface IDownloadRepository {
IAsyncEnumerable<FileUrl> FindReachableFiles(CancellationToken cancellationToken = default);
internal sealed class Dummy : IDownloadRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public ObservableValue<long> TotalCount { get; } = new (0L);
public Task AddDownload(Data.Download item, Stream? stream) {
return Task.CompletedTask;
}
public Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken) {
public Task<long> Count(DownloadItemFilter? filter, CancellationToken cancellationToken) {
return Task.FromResult(0L);
}

View File

@@ -1,16 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Utils.Observables;
namespace DHT.Server.Database.Repositories;
public interface IMessageRepository {
IObservable<long> TotalCount { get; }
ObservableValue<long> TotalCount { get; }
Task Add(IReadOnlyList<Message> messages);
@@ -25,7 +24,7 @@ public interface IMessageRepository {
Task<int> RemoveUnreachableAttachments();
internal sealed class Dummy : IMessageRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public ObservableValue<long> TotalCount { get; } = new (0L);
public Task Add(IReadOnlyList<Message> messages) {
return Task.CompletedTask;

View File

@@ -1,14 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Utils.Observables;
namespace DHT.Server.Database.Repositories;
public interface IServerRepository {
IObservable<long> TotalCount { get; }
ObservableValue<long> TotalCount { get; }
Task Add(IReadOnlyList<Data.Server> servers);
@@ -19,7 +18,7 @@ public interface IServerRepository {
Task<int> RemoveUnreachable();
internal sealed class Dummy : IServerRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public ObservableValue<long> TotalCount { get; } = new (0L);
public Task Add(IReadOnlyList<Data.Server> servers) {
return Task.CompletedTask;

View File

@@ -1,15 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Utils.Observables;
namespace DHT.Server.Database.Repositories;
public interface IUserRepository {
IObservable<long> TotalCount { get; }
ObservableValue<long> TotalCount { get; }
Task Add(IReadOnlyList<User> users);
@@ -20,7 +19,7 @@ public interface IUserRepository {
Task<int> RemoveUnreachable();
internal sealed class Dummy : IUserRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public ObservableValue<long> TotalCount { get; } = new (0L);
public Task Add(IReadOnlyList<User> users) {
return Task.CompletedTask;

View File

@@ -1,23 +1,27 @@
using System;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using DHT.Utils.Logging;
using DHT.Utils.Observables;
using DHT.Utils.Tasks;
namespace DHT.Server.Database.Sqlite.Repositories;
abstract class BaseSqliteRepository : IDisposable {
private readonly ObservableThrottledTask<long> totalCountTask;
private readonly ThrottledTask<long> totalCountTask;
public IObservable<long> TotalCount { get; }
public ObservableValue<long> TotalCount { get; } = new (0L);
protected BaseSqliteRepository(Log log) {
totalCountTask = new ObservableThrottledTask<long>(log, TaskScheduler.Default);
TotalCount = totalCountTask.DistinctUntilChanged();
totalCountTask = new ThrottledTask<long>(log, SetTotalCount, TimeSpan.Zero, TaskScheduler.Default);
UpdateTotalCount();
}
private Task SetTotalCount(long newCount) {
TotalCount.Set(newCount);
return Task.CompletedTask;
}
public void Dispose() {
totalCountTask.Dispose();
}

View File

@@ -306,12 +306,14 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) {
found.Add(new DownloadItem {
NormalizedUrl = reader.GetString(0),
DownloadUrl = reader.GetString(1),
Type = reader.IsDBNull(2) ? null : reader.GetString(2),
Size = reader.IsDBNull(3) ? null : reader.GetUint64(3),
});
var item = new DownloadItem(
NormalizedUrl: reader.GetString(0),
DownloadUrl: reader.GetString(1),
Type: reader.IsDBNull(2) ? null : reader.GetString(2),
Size: reader.IsDBNull(3) ? null : reader.GetUint64(3)
);
found.Add(item);
}
}

View File

@@ -4,12 +4,7 @@ using DHT.Server.Data;
namespace DHT.Server.Download;
public readonly struct DownloadItem {
public string NormalizedUrl { get; init; }
public string DownloadUrl { get; init; }
public string? Type { get; init; }
public ulong? Size { get; init; }
public sealed record DownloadItem(string NormalizedUrl, string DownloadUrl, string? Type, ulong? Size) {
internal Data.Download ToSuccess(long size) {
return new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, (ulong) Math.Max(size, val2: 0));
}

View File

@@ -1,8 +1,8 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Observables;
namespace DHT.Server.Download;
@@ -19,11 +19,11 @@ public sealed class Downloader {
this.concurrentDownloads = concurrentDownloads;
}
public async Task<IObservable<DownloadItem>> Start(DownloadItemFilter filter) {
public async Task<ObservableValue<DownloadItem>> Start(DownloadItemFilter filter) {
await semaphore.WaitAsync();
try {
current ??= new DownloaderTask(db, filter, concurrentDownloads);
return current.FinishedItems;
return current.LastFinishedItem;
} finally {
semaphore.Release();
}

View File

@@ -4,13 +4,13 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Observables;
using DHT.Utils.Tasks;
namespace DHT.Server.Download;
@@ -38,12 +38,11 @@ sealed class DownloaderTask : IAsyncDisposable {
private readonly IDatabaseFile db;
private readonly DownloadItemFilter filter;
private readonly ISubject<DownloadItem> finishedItemPublisher = Subject.Synchronize(new Subject<DownloadItem>());
private readonly Task queueWriterTask;
private readonly Task[] downloadTasks;
public IObservable<DownloadItem> FinishedItems => finishedItemPublisher;
public ObservableValue<DownloadItem> LastFinishedItem { get; } = new (null);
internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter, int? concurrentDownloads) {
this.db = db;
@@ -101,7 +100,7 @@ sealed class DownloaderTask : IAsyncDisposable {
log.Error("Could not download file: " + item.DownloadUrl, e);
} finally {
try {
finishedItemPublisher.OnNext(item);
LastFinishedItem.Set(item);
} catch (Exception e) {
log.Error("Caught exception in event handler: " + e);
}
@@ -145,7 +144,6 @@ sealed class DownloaderTask : IAsyncDisposable {
await Task.WhenAll(downloadTasks).WaitIgnoringCancellation();
} finally {
cancellationTokenSource.Dispose();
finishedItemPublisher.OnCompleted();
}
}
}

View File

@@ -0,0 +1,63 @@
using System;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
namespace DHT.Utils.Observables;
sealed class LastValueObserver<T> : IDisposable {
private readonly ObservableValue<T> observable;
private readonly Func<T, CancellationToken, Task> action;
private readonly TaskScheduler scheduler;
private readonly Channel<T> channel = Channel.CreateBounded<T>(new BoundedChannelOptions(capacity: 1) {
AllowSynchronousContinuations = false,
FullMode = BoundedChannelFullMode.DropOldest,
SingleReader = true,
SingleWriter = false,
});
private readonly CancellationTokenSource cancellationTokenSource = new ();
public LastValueObserver(ObservableValue<T> observable, Func<T, CancellationToken, Task> action, TaskScheduler scheduler) {
this.observable = observable;
this.action = action;
this.scheduler = scheduler;
_ = ReadNextValue();
}
private async Task ReadNextValue() {
CancellationToken cancellationToken = cancellationTokenSource.Token;
try {
await foreach (T value in channel.Reader.ReadAllAsync(cancellationToken)) {
try {
await Task.Factory.StartNew(UseValue, value, CancellationToken.None, TaskCreationOptions.None, scheduler).WaitAsync(cancellationToken);
} catch (Exception) {
// Ignore.
}
}
} finally {
cancellationTokenSource.Dispose();
}
}
private Task UseValue(object? value) {
return action((T) value!, cancellationTokenSource.Token);
}
public void Notify(T value) {
channel.Writer.TryWrite(value);
}
public void Dispose() {
observable.Unsubscribe(this);
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
channel.Writer.TryComplete();
}
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace DHT.Utils.Observables;
public sealed class ObservableValue<T>(T? value) {
private readonly List<LastValueObserver<T>> observers = [];
private T? value = value;
public void Set(T value) {
lock (this) {
if (EqualityComparer<T>.Default.Equals(value, this.value)) {
return;
}
this.value = value;
foreach (var observer in observers) {
observer.Notify(value);
}
}
}
public IDisposable SubscribeLast(Func<T, CancellationToken, Task> action, TaskScheduler scheduler) {
var observer = new LastValueObserver<T>(this, action, scheduler);
lock (this) {
observers.Add(observer);
if (value is not null) {
observer.Notify(value);
}
}
return observer;
}
internal void Unsubscribe(LastValueObserver<T> observer) {
lock (this) {
observers.Remove(observer);
}
}
}

View File

@@ -1,64 +0,0 @@
using System;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using DHT.Utils.Logging;
namespace DHT.Utils.Tasks;
public sealed class DelayedThrottledTask<T> : IDisposable {
private readonly Channel<T> taskChannel = Channel.CreateBounded<T>(new BoundedChannelOptions(capacity: 1) {
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false,
FullMode = BoundedChannelFullMode.DropOldest,
});
private readonly CancellationTokenSource cancellationTokenSource = new ();
private readonly Log log;
private readonly TimeSpan delay;
private readonly Func<T, Task> inputProcessor;
public DelayedThrottledTask(Log log, TimeSpan delay, Func<T, Task> inputProcessor) {
this.log = log;
this.delay = delay;
this.inputProcessor = inputProcessor;
Task.Run(ReaderTask);
}
private async Task ReaderTask() {
CancellationToken cancellationToken = cancellationTokenSource.Token;
try {
while (await taskChannel.Reader.WaitToReadAsync(cancellationToken)) {
await Task.Delay(delay, cancellationToken);
T input = await taskChannel.Reader.ReadAsync(cancellationToken);
try {
await inputProcessor(input);
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
log.Error("Caught exception in task: " + e);
}
}
} catch (OperationCanceledException) {
// Ignore.
} finally {
cancellationTokenSource.Dispose();
}
}
public void Post(T input) {
taskChannel.Writer.TryWrite(input);
}
public void Dispose() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
taskChannel.Writer.Complete();
}
}

View File

@@ -1,32 +0,0 @@
using System;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
using DHT.Utils.Logging;
namespace DHT.Utils.Tasks;
public sealed class ObservableThrottledTask<T> : IObservable<T>, IDisposable {
private readonly ReplaySubject<T> subject;
private readonly ThrottledTask<T> task;
public ObservableThrottledTask(Log log, TaskScheduler resultScheduler) {
this.subject = new ReplaySubject<T>(bufferSize: 1);
this.task = new ThrottledTask<T>(log, subject.OnNext, resultScheduler);
}
public void Post(Func<CancellationToken, Task<T>> resultComputer) {
task.Post(resultComputer);
}
public IDisposable Subscribe(IObserver<T> observer) {
return subject.Subscribe(observer);
}
public void Dispose() {
task.Dispose();
subject.OnCompleted();
subject.Dispose();
}
}

View File

@@ -14,11 +14,13 @@ public abstract class ThrottledTaskBase<T> : IDisposable {
FullMode = BoundedChannelFullMode.DropOldest,
});
private readonly CancellationTokenSource cancellationTokenSource = new ();
private readonly Log log;
private readonly TimeSpan delayBetweenRuns;
private readonly CancellationTokenSource cancellationTokenSource = new ();
internal ThrottledTaskBase(Log log) {
internal ThrottledTaskBase(Log log, TimeSpan delayBetweenRuns) {
this.log = log;
this.delayBetweenRuns = delayBetweenRuns;
}
protected async Task ReaderTask() {
@@ -33,9 +35,9 @@ public abstract class ThrottledTaskBase<T> : IDisposable {
} catch (Exception e) {
log.Error("Caught exception in task: " + e);
}
await Task.Delay(delayBetweenRuns, cancellationToken);
}
} catch (OperationCanceledException) {
// Ignore.
} finally {
cancellationTokenSource.Dispose();
}
@@ -48,20 +50,20 @@ public abstract class ThrottledTaskBase<T> : IDisposable {
}
public void Dispose() {
taskChannel.Writer.Complete();
cancellationTokenSource.Cancel();
taskChannel.Writer.Complete();
}
}
public sealed class ThrottledTask : ThrottledTaskBase<Task> {
private readonly Action resultProcessor;
private readonly Func<Task> resultProcessor;
private readonly TaskScheduler resultScheduler;
public ThrottledTask(Log log, Action resultProcessor, TaskScheduler resultScheduler) : base(log) {
public ThrottledTask(Log log, Func<Task> resultProcessor, TimeSpan delayBetweenRuns, TaskScheduler resultScheduler) : base(log, delayBetweenRuns) {
this.resultProcessor = resultProcessor;
this.resultScheduler = resultScheduler;
Task.Run(ReaderTask);
_ = ReaderTask();
}
protected override async Task Run(Func<CancellationToken, Task> func, CancellationToken cancellationToken) {
@@ -71,18 +73,22 @@ public sealed class ThrottledTask : ThrottledTaskBase<Task> {
}
public sealed class ThrottledTask<T> : ThrottledTaskBase<Task<T>> {
private readonly Action<T> resultProcessor;
private readonly Func<T, Task> resultProcessor;
private readonly TaskScheduler resultScheduler;
public ThrottledTask(Log log, Action<T> resultProcessor, TaskScheduler resultScheduler) : base(log) {
public ThrottledTask(Log log, Func<T, Task> resultProcessor, TimeSpan delayBetweenRuns, TaskScheduler resultScheduler) : base(log, delayBetweenRuns) {
this.resultProcessor = resultProcessor;
this.resultScheduler = resultScheduler;
Task.Run(ReaderTask);
_ = ReaderTask();
}
protected override async Task Run(Func<CancellationToken, Task<T>> func, CancellationToken cancellationToken) {
T result = await func(cancellationToken);
await Task.Factory.StartNew(() => resultProcessor(result), cancellationToken, TaskCreationOptions.None, resultScheduler);
}
public void Post(T result) {
base.Post(_ => Task.FromResult(result));
}
}

View File

@@ -16,7 +16,6 @@
<ItemGroup>
<PackageReference Include="JetBrains.Annotations" Version="2023.2.0" />
<PackageReference Include="System.Reactive" Version="6.0.0" />
</ItemGroup>
<ItemGroup>

View File

@@ -8,5 +8,5 @@ using DHT.Utils;
namespace DHT.Utils;
static class Version {
public const string Tag = "47.1.0.0";
public const string Tag = "47.2.0.0";
}