1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-11-25 14:42:44 +01:00

Compare commits

..

No commits in common. "d2934f4d6ae4a2d242e90a028219c5fb67fc97bd" and "3d9d6a454a0ad48e27cc588028f594c18b28f9bb" have entirely different histories.

15 changed files with 200 additions and 218 deletions

View File

@ -45,17 +45,17 @@
<Rectangle /> <Rectangle />
<StackPanel Orientation="Vertical"> <StackPanel Orientation="Vertical">
<TextBlock Classes="label">Servers</TextBlock> <TextBlock Classes="label">Servers</TextBlock>
<TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalServers, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" /> <TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalServers, Converter={StaticResource NumberValueConverter}}" />
</StackPanel> </StackPanel>
<Rectangle /> <Rectangle />
<StackPanel Orientation="Vertical"> <StackPanel Orientation="Vertical">
<TextBlock Classes="label">Channels</TextBlock> <TextBlock Classes="label">Channels</TextBlock>
<TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalChannels, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" /> <TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalChannels, Converter={StaticResource NumberValueConverter}}" />
</StackPanel> </StackPanel>
<Rectangle /> <Rectangle />
<StackPanel Orientation="Vertical"> <StackPanel Orientation="Vertical">
<TextBlock Classes="label">Messages</TextBlock> <TextBlock Classes="label">Messages</TextBlock>
<TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalMessages, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" /> <TextBlock Classes="value" Text="{Binding DatabaseStatistics.TotalMessages, Converter={StaticResource NumberValueConverter}}" />
</StackPanel> </StackPanel>
</StackPanel> </StackPanel>

View File

@ -20,9 +20,9 @@ public sealed partial class MainWindow : Window {
DataContext = new MainWindowModel(this, args); DataContext = new MainWindowModel(this, args);
} }
public async void OnClosed(object? sender, EventArgs e) { public void OnClosed(object? sender, EventArgs e) {
if (DataContext is MainWindowModel model) { if (DataContext is IDisposable disposable) {
await model.DisposeAsync(); disposable.Dispose();
} }
foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) { foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {

View File

@ -12,7 +12,7 @@ using DHT.Utils.Models;
namespace DHT.Desktop.Main; namespace DHT.Desktop.Main;
sealed class MainWindowModel : BaseModel, IAsyncDisposable { sealed class MainWindowModel : BaseModel, IDisposable {
private const string DefaultTitle = "Discord History Tracker"; private const string DefaultTitle = "Discord History Tracker";
public string Title { get; private set; } = DefaultTitle; public string Title { get; private set; } = DefaultTitle;
@ -75,7 +75,7 @@ sealed class MainWindowModel : BaseModel, IAsyncDisposable {
if (e.PropertyName == nameof(welcomeScreenModel.Db)) { if (e.PropertyName == nameof(welcomeScreenModel.Db)) {
if (mainContentScreenModel != null) { if (mainContentScreenModel != null) {
mainContentScreenModel.DatabaseClosed -= MainContentScreenModelOnDatabaseClosed; mainContentScreenModel.DatabaseClosed -= MainContentScreenModelOnDatabaseClosed;
await mainContentScreenModel.DisposeAsync(); mainContentScreenModel.Dispose();
} }
db?.Dispose(); db?.Dispose();
@ -107,13 +107,9 @@ sealed class MainWindowModel : BaseModel, IAsyncDisposable {
welcomeScreenModel.CloseDatabase(); welcomeScreenModel.CloseDatabase();
} }
public async ValueTask DisposeAsync() { public void Dispose() {
welcomeScreenModel.Dispose(); welcomeScreenModel.Dispose();
mainContentScreenModel?.Dispose();
if (mainContentScreenModel != null) {
await mainContentScreenModel.DisposeAsync();
}
db?.Dispose(); db?.Dispose();
db = null; db = null;
} }

View File

@ -33,7 +33,7 @@
<StackPanel Orientation="Vertical" Spacing="20"> <StackPanel Orientation="Vertical" Spacing="20">
<DockPanel> <DockPanel>
<Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" /> <Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" />
<TextBlock Text="{Binding DownloadMessage}" MinWidth="100" Margin="10 0 0 0" VerticalAlignment="Center" TextAlignment="Right" DockPanel.Dock="Left" /> <TextBlock Text="{Binding DownloadMessage}" Margin="10 0 0 0" VerticalAlignment="Center" DockPanel.Dock="Left" />
<ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" /> <ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" />
</DockPanel> </DockPanel>
<controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !IsDownloading, RelativeSource={RelativeSource AncestorType=pages:AttachmentsPageModel}}" /> <controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !IsDownloading, RelativeSource={RelativeSource AncestorType=pages:AttachmentsPageModel}}" />
@ -42,8 +42,8 @@
<DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True"> <DataGrid ItemsSource="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
<DataGrid.Columns> <DataGrid.Columns>
<DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" /> <DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" />
<DataGridTextColumn Header="Attachments" Binding="{Binding Items, Mode=OneWay, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" /> <DataGridTextColumn Header="Attachments" Binding="{Binding Items, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
<DataGridTextColumn Header="Size" Binding="{Binding Size, Mode=OneWay, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" /> <DataGridTextColumn Header="Size" Binding="{Binding Size, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" />
</DataGrid.Columns> </DataGrid.Columns>
</DataGrid> </DataGrid>
</Expander> </Expander>

View File

@ -2,7 +2,6 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Threading; using System.Threading;
using System.Threading.Tasks;
using Avalonia.Threading; using Avalonia.Threading;
using DHT.Desktop.Common; using DHT.Desktop.Common;
using DHT.Desktop.Main.Controls; using DHT.Desktop.Main.Controls;
@ -16,17 +15,16 @@ using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Pages; namespace DHT.Desktop.Main.Pages;
sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable { sealed class AttachmentsPageModel : BaseModel, IDisposable {
private static readonly DownloadItemFilter EnqueuedItemFilter = new() { private static readonly DownloadItemFilter EnqueuedItemFilter = new() {
IncludeStatuses = new HashSet<DownloadStatus> { IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued, DownloadStatus.Enqueued
DownloadStatus.Downloading
} }
}; };
private bool isThreadDownloadButtonEnabled = true; private bool isThreadDownloadButtonEnabled = true;
public string ToggleDownloadButtonText => downloader == null ? "Start Downloading" : "Stop Downloading"; public string ToggleDownloadButtonText => downloadThread == null ? "Start Downloading" : "Stop Downloading";
public bool IsToggleDownloadButtonEnabled { public bool IsToggleDownloadButtonEnabled {
get => isThreadDownloadButtonEnabled; get => isThreadDownloadButtonEnabled;
@ -34,7 +32,7 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
} }
public string DownloadMessage { get; set; } = ""; public string DownloadMessage { get; set; } = "";
public double DownloadProgress => totalItemsToDownloadCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / totalItemsToDownloadCount.Value; public double DownloadProgress => allItemsCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / allItemsCount.Value;
public AttachmentFilterPanelModel FilterModel { get; } public AttachmentFilterPanelModel FilterModel { get; }
@ -54,16 +52,15 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
} }
} }
public bool IsDownloading => downloader != null; public bool IsDownloading => downloadThread != null;
public bool HasFailedDownloads => statisticsFailed.Items > 0; public bool HasFailedDownloads => statisticsFailed.Items > 0;
private readonly IDatabaseFile db; private readonly IDatabaseFile db;
private readonly AsyncValueComputer<DownloadStatusStatistics>.Single downloadStatisticsComputer; private readonly AsyncValueComputer<DownloadStatusStatistics>.Single downloadStatisticsComputer;
private BackgroundDownloader? downloader; private BackgroundDownloadThread? downloadThread;
private int doneItemsCount; private int doneItemsCount;
private int initialFinishedCount; private int? allItemsCount;
private int? totalItemsToDownloadCount;
public AttachmentsPageModel() : this(DummyDatabaseFile.Instance) {} public AttachmentsPageModel() : this(DummyDatabaseFile.Instance) {}
@ -77,11 +74,11 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
db.Statistics.PropertyChanged += OnDbStatisticsChanged; db.Statistics.PropertyChanged += OnDbStatisticsChanged;
} }
public async ValueTask DisposeAsync() { public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged; db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose(); FilterModel.Dispose();
await DisposeDownloader(); DisposeDownloadThread();
} }
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) { private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
@ -127,42 +124,44 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
OnPropertyChanged(nameof(HasFailedDownloads)); OnPropertyChanged(nameof(HasFailedDownloads));
} }
totalItemsToDownloadCount = statisticsEnqueued.Items + statisticsDownloaded.Items + statisticsFailed.Items - initialFinishedCount; allItemsCount = doneItemsCount + statisticsEnqueued.Items;
UpdateDownloadMessage(); UpdateDownloadMessage();
} }
private void UpdateDownloadMessage() { private void UpdateDownloadMessage() {
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (totalItemsToDownloadCount?.Format() ?? "?") : ""; DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (allItemsCount?.Format() ?? "?") : "";
OnPropertyChanged(nameof(DownloadMessage)); OnPropertyChanged(nameof(DownloadMessage));
OnPropertyChanged(nameof(DownloadProgress)); OnPropertyChanged(nameof(DownloadProgress));
} }
private void DownloaderOnOnItemFinished(object? sender, DownloadItem e) { private void DownloadThreadOnOnItemFinished(object? sender, DownloadItem e) {
Interlocked.Increment(ref doneItemsCount); Interlocked.Increment(ref doneItemsCount);
Dispatcher.UIThread.Invoke(UpdateDownloadMessage); Dispatcher.UIThread.Invoke(UpdateDownloadMessage);
downloadStatisticsComputer.Recompute(); downloadStatisticsComputer.Recompute();
} }
public async Task OnClickToggleDownload() { private void DownloadThreadOnOnServerStopped(object? sender, EventArgs e) {
if (downloader == null) { downloadStatisticsComputer.Recompute();
initialFinishedCount = statisticsDownloaded.Items + statisticsFailed.Items; IsToggleDownloadButtonEnabled = true;
}
public void OnClickToggleDownload() {
if (downloadThread == null) {
EnqueueDownloadItems(); EnqueueDownloadItems();
downloader = new BackgroundDownloader(db); downloadThread = new BackgroundDownloadThread(db);
downloader.OnItemFinished += DownloaderOnOnItemFinished; downloadThread.OnItemFinished += DownloadThreadOnOnItemFinished;
downloadThread.OnServerStopped += DownloadThreadOnOnServerStopped;
} }
else { else {
IsToggleDownloadButtonEnabled = false; IsToggleDownloadButtonEnabled = false;
await DisposeDownloader(); DisposeDownloadThread();
downloadStatisticsComputer.Recompute();
IsToggleDownloadButtonEnabled = true;
db.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching); db.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
doneItemsCount = 0; doneItemsCount = 0;
initialFinishedCount = 0; allItemsCount = null;
totalItemsToDownloadCount = null;
UpdateDownloadMessage(); UpdateDownloadMessage();
} }
@ -174,7 +173,6 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
var allExceptFailedFilter = new DownloadItemFilter { var allExceptFailedFilter = new DownloadItemFilter {
IncludeStatuses = new HashSet<DownloadStatus> { IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued, DownloadStatus.Enqueued,
DownloadStatus.Downloading,
DownloadStatus.Success DownloadStatus.Success
} }
}; };
@ -186,13 +184,13 @@ sealed class AttachmentsPageModel : BaseModel, IAsyncDisposable {
} }
} }
private async Task DisposeDownloader() { private void DisposeDownloadThread() {
if (downloader != null) { if (downloadThread != null) {
downloader.OnItemFinished -= DownloaderOnOnItemFinished; downloadThread.OnItemFinished -= DownloadThreadOnOnItemFinished;
await downloader.Stop(); downloadThread.StopThread();
} }
downloader = null; downloadThread = null;
} }
public sealed class StatisticsRow { public sealed class StatisticsRow {

View File

@ -11,7 +11,7 @@ using DHT.Utils.Logging;
namespace DHT.Desktop.Main.Screens; namespace DHT.Desktop.Main.Screens;
sealed class MainContentScreenModel : IAsyncDisposable { sealed class MainContentScreenModel : IDisposable {
private static readonly Log Log = Log.ForType<MainContentScreenModel>(); private static readonly Log Log = Log.ForType<MainContentScreenModel>();
public DatabasePage DatabasePage { get; } public DatabasePage DatabasePage { get; }
@ -97,9 +97,9 @@ sealed class MainContentScreenModel : IAsyncDisposable {
serverManager.Launch(); serverManager.Launch();
} }
public async ValueTask DisposeAsync() { public void Dispose() {
ServerLauncher.ServerManagementExceptionCaught -= ServerLauncherOnServerManagementExceptionCaught; ServerLauncher.ServerManagementExceptionCaught -= ServerLauncherOnServerManagementExceptionCaught;
await AttachmentsPageModel.DisposeAsync(); AttachmentsPageModel.Dispose();
ViewerPageModel.Dispose(); ViewerPageModel.Dispose();
serverManager.Dispose(); serverManager.Dispose();
} }

View File

@ -8,6 +8,5 @@ namespace DHT.Server.Data;
public enum DownloadStatus { public enum DownloadStatus {
Enqueued = 0, Enqueued = 0,
GenericError = 1, GenericError = 1,
Downloading = 2,
Success = HttpStatusCode.OK Success = HttpStatusCode.OK
} }

View File

@ -74,7 +74,7 @@ public sealed class DummyDatabaseFile : IDatabaseFile {
public void EnqueueDownloadItems(AttachmentFilter? filter = null) {} public void EnqueueDownloadItems(AttachmentFilter? filter = null) {}
public List<DownloadItem> PullEnqueuedDownloadItems(int count) { public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
return new(); return new();
} }

View File

@ -35,7 +35,7 @@ public interface IDatabaseFile : IDisposable {
DownloadedAttachment? GetDownloadedAttachment(string url); DownloadedAttachment? GetDownloadedAttachment(string url);
void EnqueueDownloadItems(AttachmentFilter? filter = null); void EnqueueDownloadItems(AttachmentFilter? filter = null);
List<DownloadItem> PullEnqueuedDownloadItems(int count); List<DownloadItem> GetEnqueuedDownloadItems(int count);
void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode); void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);
DownloadStatusStatistics GetDownloadStatusStatistics(); DownloadStatusStatistics GetDownloadStatusStatistics();

View File

@ -174,8 +174,8 @@ sealed class Schema {
int processedUrls = -1; int processedUrls = -1;
await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) { await using (var updateCmd = conn.Command("UPDATE attachments SET download_url = url, url = :normalized_url WHERE attachment_id = :attachment_id")) {
updateCmd.Add(":attachment_id", SqliteType.Integer); updateCmd.Parameters.Add(":attachment_id", SqliteType.Integer);
updateCmd.Add(":normalized_url", SqliteType.Text); updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
foreach (var (attachmentId, normalizedUrl) in normalizedUrls) { foreach (var (attachmentId, normalizedUrl) in normalizedUrls) {
if (++processedUrls % 1000 == 0) { if (++processedUrls % 1000 == 0) {
@ -235,8 +235,8 @@ sealed class Schema {
tx = conn.BeginTransaction(); tx = conn.BeginTransaction();
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) { await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
updateCmd.Add(":normalized_url", SqliteType.Text); updateCmd.Parameters.Add(":normalized_url", SqliteType.Text);
updateCmd.Add(":download_url", SqliteType.Text); updateCmd.Parameters.Add(":download_url", SqliteType.Text);
foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) { foreach (var (normalizedUrl, downloadUrl) in normalizedUrlsToOriginalUrls) {
if (++processedUrls % 100 == 0) { if (++processedUrls % 100 == 0) {

View File

@ -522,42 +522,25 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
} }
public List<DownloadItem> PullEnqueuedDownloadItems(int count) { public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
var found = new List<DownloadItem>(); var list = new List<DownloadItem>();
var pulled = new List<DownloadItem>();
using var conn = pool.Take(); using var conn = pool.Take();
using (var cmd = conn.Command("SELECT normalized_url, download_url, size FROM downloads WHERE status = :enqueued LIMIT :limit")) { using var cmd = conn.Command("SELECT normalized_url, download_url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued); cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count)); cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
using var reader = cmd.ExecuteReader(); using var reader = cmd.ExecuteReader();
while (reader.Read()) { while (reader.Read()) {
found.Add(new DownloadItem { list.Add(new DownloadItem {
NormalizedUrl = reader.GetString(0), NormalizedUrl = reader.GetString(0),
DownloadUrl = reader.GetString(1), DownloadUrl = reader.GetString(1),
Size = reader.GetUint64(2), Size = reader.GetUint64(2),
}); });
} }
}
if (found.Count != 0) { return list;
using var cmd = conn.Command("UPDATE downloads SET status = :downloading WHERE normalized_url = :normalized_url AND status = :enqueued");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":downloading", SqliteType.Integer, (int) DownloadStatus.Downloading);
cmd.Add(":normalized_url", SqliteType.Text);
foreach (var item in found) {
cmd.Set(":normalized_url", item.NormalizedUrl);
if (cmd.ExecuteNonQuery() == 1) {
pulled.Add(item);
}
}
}
return pulled;
} }
public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) { public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
@ -579,16 +562,15 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) { static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
using var cmd = conn.Command(""" using var cmd = conn.Command("""
SELECT SELECT
IFNULL(SUM(CASE WHEN status IN (:enqueued, :downloading) THEN 1 ELSE 0 END), 0), IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status IN (:enqueued, :downloading) THEN size ELSE 0 END), 0), IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0), IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0), IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status NOT IN (:enqueued, :downloading) AND status != :success THEN 1 ELSE 0 END), 0), IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status NOT IN (:enqueued, :downloading) AND status != :success THEN size ELSE 0 END), 0) IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
FROM downloads FROM downloads
"""); """);
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued); cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":downloading", SqliteType.Integer, (int) DownloadStatus.Downloading);
cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success); cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
using var reader = cmd.ExecuteReader(); using var reader = cmd.ExecuteReader();

View File

@ -62,10 +62,6 @@ static class SqliteExtensions {
} }
} }
public static void Add(this SqliteCommand cmd, string key, SqliteType type) {
cmd.Parameters.Add(key, type);
}
public static void AddAndSet(this SqliteCommand cmd, string key, SqliteType type, object? value) { public static void AddAndSet(this SqliteCommand cmd, string key, SqliteType type, object? value) {
cmd.Parameters.Add(key, type).Value = value ?? DBNull.Value; cmd.Parameters.Add(key, type).Value = value ?? DBNull.Value;
} }

View File

@ -0,0 +1,130 @@
using System;
using System.Collections.Concurrent;
using System.Net.Http;
using System.Threading;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Models;
namespace DHT.Server.Download;
public sealed class BackgroundDownloadThread : BaseModel {
private static readonly Log Log = Log.ForType<BackgroundDownloadThread>();
public event EventHandler<DownloadItem>? OnItemFinished {
add => parameters.OnItemFinished += value;
remove => parameters.OnItemFinished -= value;
}
public event EventHandler? OnServerStopped {
add => parameters.OnServerStopped += value;
remove => parameters.OnServerStopped -= value;
}
private readonly CancellationTokenSource cancellationTokenSource;
private readonly ThreadInstance.Parameters parameters;
public BackgroundDownloadThread(IDatabaseFile db) {
this.cancellationTokenSource = new CancellationTokenSource();
this.parameters = new ThreadInstance.Parameters(db, cancellationTokenSource);
var thread = new Thread(new ThreadInstance().Work) {
Name = "DHT download thread"
};
thread.Start(parameters);
}
public void StopThread() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {
Log.Warn("Attempted to stop background download thread after the cancellation token has been disposed.");
}
}
private sealed class ThreadInstance {
private const int QueueSize = 32;
public sealed class Parameters {
public event EventHandler<DownloadItem>? OnItemFinished;
public event EventHandler? OnServerStopped;
public IDatabaseFile Db { get; }
public CancellationTokenSource CancellationTokenSource { get; }
public Parameters(IDatabaseFile db, CancellationTokenSource cancellationTokenSource) {
Db = db;
CancellationTokenSource = cancellationTokenSource;
}
public void FireOnItemFinished(DownloadItem item) {
OnItemFinished?.Invoke(null, item);
}
public void FireOnServerStopped() {
OnServerStopped?.Invoke(null, EventArgs.Empty);
}
}
private readonly HttpClient client = new ();
public ThreadInstance() {
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36");
}
public async void Work(object? obj) {
var parameters = (Parameters) obj!;
var cancellationTokenSource = parameters.CancellationTokenSource;
var cancellationToken = cancellationTokenSource.Token;
var db = parameters.Db;
var queue = new ConcurrentQueue<DownloadItem>();
try {
while (!cancellationToken.IsCancellationRequested) {
FillQueue(db, queue, cancellationToken);
while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var item)) {
var downloadUrl = item.DownloadUrl;
Log.Debug("Downloading " + downloadUrl + "...");
try {
db.AddDownload(Data.Download.NewSuccess(item, await client.GetByteArrayAsync(downloadUrl, cancellationToken)));
} catch (HttpRequestException e) {
db.AddDownload(Data.Download.NewFailure(item, e.StatusCode, item.Size));
Log.Error(e);
} catch (Exception e) {
db.AddDownload(Data.Download.NewFailure(item, null, item.Size));
Log.Error(e);
} finally {
parameters.FireOnItemFinished(item);
}
}
}
} catch (OperationCanceledException) {
//
} catch (ObjectDisposedException) {
//
} finally {
cancellationTokenSource.Dispose();
parameters.FireOnServerStopped();
}
}
private static void FillQueue(IDatabaseFile db, ConcurrentQueue<DownloadItem> queue, CancellationToken cancellationToken) {
while (!cancellationToken.IsCancellationRequested && queue.IsEmpty) {
var newItems = db.GetEnqueuedDownloadItems(QueueSize);
if (newItems.Count == 0) {
Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
else {
foreach (var item in newItems) {
queue.Enqueue(item);
}
}
}
}
}
}

View File

@ -1,107 +0,0 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Server.Download;
public sealed class BackgroundDownloader : BaseModel {
private static readonly Log Log = Log.ForType<BackgroundDownloader>();
private const int DownloadTasks = 4;
private const int QueueSize = 25;
private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
public event EventHandler<DownloadItem>? OnItemFinished;
private readonly Channel<DownloadItem> downloadQueue = Channel.CreateBounded<DownloadItem>(new BoundedChannelOptions(QueueSize) {
SingleReader = false,
SingleWriter = true,
AllowSynchronousContinuations = false,
FullMode = BoundedChannelFullMode.Wait
});
private readonly CancellationTokenSource cancellationTokenSource = new ();
private readonly CancellationToken cancellationToken;
private readonly IDatabaseFile db;
private readonly Task queueWriterTask;
private readonly Task[] downloadTasks;
public BackgroundDownloader(IDatabaseFile db) {
this.cancellationToken = cancellationTokenSource.Token;
this.db = db;
this.queueWriterTask = Task.Run(RunQueueWriterTask);
this.downloadTasks = Enumerable.Range(1, DownloadTasks).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray();
}
private async Task RunQueueWriterTask() {
while (await downloadQueue.Writer.WaitToWriteAsync(cancellationToken)) {
var newItems = db.PullEnqueuedDownloadItems(QueueSize);
if (newItems.Count == 0) {
await Task.Delay(TimeSpan.FromMilliseconds(50), cancellationToken);
continue;
}
foreach (var newItem in newItems) {
await downloadQueue.Writer.WriteAsync(newItem, cancellationToken);
}
}
}
private async Task RunDownloadTask(int taskIndex) {
var log = Log.ForType<BackgroundDownloader>("Task " + taskIndex);
var client = new HttpClient();
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
client.Timeout = TimeSpan.FromSeconds(30);
while (!cancellationToken.IsCancellationRequested) {
var item = await downloadQueue.Reader.ReadAsync(cancellationToken);
log.Debug("Downloading " + item.DownloadUrl + "...");
try {
var downloadedBytes = await client.GetByteArrayAsync(item.DownloadUrl, cancellationToken);
db.AddDownload(Data.Download.NewSuccess(item, downloadedBytes));
} catch (OperationCanceledException) {
// Ignore.
} catch (HttpRequestException e) {
db.AddDownload(Data.Download.NewFailure(item, e.StatusCode, item.Size));
log.Error(e);
} catch (Exception e) {
db.AddDownload(Data.Download.NewFailure(item, null, item.Size));
log.Error(e);
} finally {
try {
OnItemFinished?.Invoke(this, item);
} catch (Exception e) {
log.Error("Caught exception in event handler: " + e);
}
}
}
}
public async Task Stop() {
try {
await cancellationTokenSource.CancelAsync();
} catch (Exception) {
Log.Warn("Attempted to stop background download twice.");
return;
}
downloadQueue.Writer.Complete();
try {
await queueWriterTask.WaitIgnoringCancellation();
await Task.WhenAll(downloadTasks).WaitIgnoringCancellation();
} finally {
cancellationTokenSource.Dispose();
}
}
}

View File

@ -1,12 +0,0 @@
using System;
using System.Threading.Tasks;
namespace DHT.Utils.Tasks;
public static class TaskExtensions {
public static async Task WaitIgnoringCancellation(this Task task) {
try {
await task;
} catch (OperationCanceledException) {}
}
}