1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-09-16 04:32:09 +02:00

14 Commits

43 changed files with 1173 additions and 120 deletions

View File

@@ -8,11 +8,13 @@
<entry key="Desktop/Dialogs/Message/MessageDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/FilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/AttachmentFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/AttachmentsPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/DebugPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/TrackingPage.axaml" value="Desktop/Desktop.csproj" />

View File

@@ -7,6 +7,7 @@
<Application.Styles>
<FluentTheme Mode="Light" />
<StyleInclude Source="avares://Avalonia.Controls.DataGrid/Themes/Default.xaml"/>
<Style Selector="Button, CheckBox, RadioButton, Expander /template/ ToggleButton#ExpanderHeader">
<Setter Property="Cursor" Value="Hand" />
@@ -105,6 +106,7 @@
<Application.Resources>
<common:NumberValueConverter x:Key="NumberValueConverter" />
<common:BytesValueConverter x:Key="BytesValueConverter" />
<system:Double x:Key="ControlContentThemeFontSize">14</system:Double>
<CornerRadius x:Key="ControlCornerRadius">0</CornerRadius>
@@ -154,7 +156,7 @@
<SolidColorBrush x:Key="TextControlPlaceholderForegroundDisabled" Color="#AAAAAA" />
<Thickness x:Key="ExpanderHeaderPadding">15,0</Thickness>
<Thickness x:Key="ExpanderContentPadding">15</Thickness>
<Thickness x:Key="ExpanderContentPadding">12</Thickness>
<SolidColorBrush x:Key="ExpanderBorderBrush" Color="#697DAB" />
<SolidColorBrush x:Key="ExpanderBackground" Color="#697DAB" />
<SolidColorBrush x:Key="ExpanderForeground" Color="#FFFFFF" />

View File

@@ -0,0 +1,45 @@
using System;
using System.Globalization;
using Avalonia.Data.Converters;
namespace DHT.Desktop.Common {
sealed class BytesValueConverter : IValueConverter {
private static readonly string[] Units = {
"B",
"kB",
"MB",
"GB",
"TB"
};
private const int Scale = 1000;
private static string Convert(ulong size) {
int power = size == 0L ? 0 : (int) Math.Log(size, Scale);
int unit = power >= Units.Length ? Units.Length - 1 : power;
if (unit == 0) {
return string.Format(Program.Culture, "{0:n0}", size) + " " + Units[unit];
}
else {
double humanReadableSize = size / Math.Pow(Scale, unit);
return string.Format(Program.Culture, "{0:n0}", humanReadableSize) + " " + Units[unit];
}
}
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) {
if (value is long size and >= 0L) {
return Convert((ulong) size);
}
else if (value is ulong usize) {
return Convert(usize);
}
else {
return "-";
}
}
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) {
throw new NotSupportedException();
}
}
}

View File

@@ -21,9 +21,10 @@
<DebugType>none</DebugType>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Avalonia" Version="0.10.13" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.13" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.13" Condition=" '$(Configuration)' == 'Debug' " />
<PackageReference Include="Avalonia" Version="0.10.14" />
<PackageReference Include="Avalonia.Controls.DataGrid" Version="0.10.14" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.14" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.14" Condition=" '$(Configuration)' == 'Debug' " />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Server\Server.csproj" />

View File

@@ -0,0 +1,51 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.AttachmentFilterPanel">
<Design.DataContext>
<controls:AttachmentFilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="ComboBox">
<Setter Property="Margin" Value="8 0 0 0" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<StackPanel>
<TextBlock Text="{Binding FilterStatisticsText}" />
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding LimitSize}">Limit Size</CheckBox>
<StackPanel Orientation="Horizontal">
<TextBox Text="{Binding MaximumSize}" IsEnabled="{Binding LimitSize}" HorizontalContentAlignment="Right" />
<ComboBox IsEnabled="{Binding LimitSize}" Items="{Binding Units}" SelectedItem="{Binding MaximumSizeUnit}">
<ComboBox.ItemTemplate>
<DataTemplate>
<TextBlock Text="{Binding Name}" />
</DataTemplate>
</ComboBox.ItemTemplate>
</ComboBox>
</StackPanel>
</StackPanel>
</WrapPanel>
</StackPanel>
</UserControl>

View File

@@ -0,0 +1,17 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Controls {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class AttachmentFilterPanel : UserControl {
public AttachmentFilterPanel() {
InitializeComponent();
}
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
}
}

View File

@@ -0,0 +1,125 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using DHT.Desktop.Common;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls {
sealed class AttachmentFilterPanelModel : BaseModel, IDisposable {
public sealed record Unit(string Name, int Scale);
private static readonly Unit[] AllUnits = {
new ("B", 1),
new ("kB", 1024),
new ("MB", 1024 * 1024)
};
private static readonly HashSet<string> FilterProperties = new () {
nameof(LimitSize),
nameof(MaximumSize),
nameof(MaximumSizeUnit)
};
public string FilterStatisticsText { get; private set; } = "";
private bool limitSize = false;
private int maximumSize = 0;
private Unit maximumSizeUnit = AllUnits[0];
public bool LimitSize {
get => limitSize;
set => Change(ref limitSize, value);
}
public int MaximumSize {
get => maximumSize;
set => Change(ref maximumSize, value);
}
public Unit MaximumSizeUnit {
get => maximumSizeUnit;
set => Change(ref maximumSizeUnit, value);
}
public IEnumerable<Unit> Units => AllUnits;
private readonly IDatabaseFile db;
private readonly string verb;
private readonly AsyncValueComputer<long> matchingAttachmentCountComputer;
private long? matchingAttachmentCount;
private long? totalAttachmentCount;
[Obsolete("Designer")]
public AttachmentFilterPanelModel() : this(DummyDatabaseFile.Instance) {}
public AttachmentFilterPanelModel(IDatabaseFile db, string verb = "Matches") {
this.db = db;
this.verb = verb;
this.matchingAttachmentCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetAttachmentCounts).Build();
UpdateFilterStatistics();
PropertyChanged += OnPropertyChanged;
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
}
private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
UpdateFilterStatistics();
}
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
totalAttachmentCount = db.Statistics.TotalAttachments;
UpdateFilterStatistics();
}
}
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
matchingAttachmentCountComputer.Cancel();
matchingAttachmentCount = totalAttachmentCount;
UpdateFilterStatisticsText();
}
else {
matchingAttachmentCount = null;
UpdateFilterStatisticsText();
matchingAttachmentCountComputer.Compute(() => db.CountAttachments(filter));
}
}
private void SetAttachmentCounts(long matchingAttachmentCount) {
this.matchingAttachmentCount = matchingAttachmentCount;
UpdateFilterStatisticsText();
}
private void UpdateFilterStatisticsText() {
var matchingAttachmentCountStr = matchingAttachmentCount?.Format() ?? "(...)";
var totalAttachmentCountStr = totalAttachmentCount?.Format() ?? "(...)";
FilterStatisticsText = verb + " " + matchingAttachmentCountStr + " out of " + totalAttachmentCountStr + " attachment" + (totalAttachmentCount is null or 1 ? "." : "s.");
OnPropertyChanged(nameof(FilterStatisticsText));
}
public AttachmentFilter CreateFilter() {
AttachmentFilter filter = new();
if (LimitSize) {
filter.MaxBytes = maximumSize * maximumSizeUnit.Scale;
}
return filter;
}
}
}

View File

@@ -4,10 +4,10 @@
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.FilterPanel">
x:Class="DHT.Desktop.Main.Controls.MessageFilterPanel">
<Design.DataContext>
<controls:FilterPanelModel />
<controls:MessageFilterPanelModel />
</Design.DataContext>
<UserControl.Styles>

View File

@@ -4,11 +4,11 @@ using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Controls {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class FilterPanel : UserControl {
public sealed class MessageFilterPanel : UserControl {
private CalendarDatePicker StartDatePicker => this.FindControl<CalendarDatePicker>("StartDatePicker");
private CalendarDatePicker EndDatePicker => this.FindControl<CalendarDatePicker>("EndDatePicker");
public FilterPanel() {
public MessageFilterPanel() {
InitializeComponent();
}
@@ -25,7 +25,7 @@ namespace DHT.Desktop.Main.Controls {
}
public void CalendarDatePicker_OnSelectedDateChanged(object? sender, SelectionChangedEventArgs e) {
if (DataContext is FilterPanelModel model) {
if (DataContext is MessageFilterPanelModel model) {
model.StartDate = StartDatePicker.SelectedDate;
model.EndDate = EndDatePicker.SelectedDate;
}

View File

@@ -15,7 +15,7 @@ using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls {
sealed class FilterPanelModel : BaseModel, IDisposable {
sealed class MessageFilterPanelModel : BaseModel, IDisposable {
private static readonly HashSet<string> FilterProperties = new () {
nameof(FilterByDate),
nameof(StartDate),
@@ -91,21 +91,23 @@ namespace DHT.Desktop.Main.Controls {
private readonly Window window;
private readonly IDatabaseFile db;
private readonly string verb;
private readonly AsyncValueComputer<long> exportedMessageCountComputer;
private long? exportedMessageCount;
private long? totalMessageCount;
[Obsolete("Designer")]
public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
public MessageFilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
public FilterPanelModel(Window window, IDatabaseFile db) {
public MessageFilterPanelModel(Window window, IDatabaseFile db, string verb = "Matches") {
this.window = window;
this.db = db;
this.verb = verb;
this.exportedMessageCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetExportedMessageCount).Build();
UpdateFilterStatisticsText();
UpdateFilterStatistics();
UpdateChannelFilterLabel();
UpdateUserFilterLabel();
@@ -147,13 +149,14 @@ namespace DHT.Desktop.Main.Controls {
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
exportedMessageCountComputer.Cancel();
exportedMessageCount = totalMessageCount;
UpdateFilterStatisticsText();
}
else {
exportedMessageCount = null;
UpdateFilterStatisticsText();
exportedMessageCountComputer.Compute(_ => db.CountMessages(filter));
exportedMessageCountComputer.Compute(() => db.CountMessages(filter));
}
}
@@ -166,7 +169,7 @@ namespace DHT.Desktop.Main.Controls {
var exportedMessageCountStr = exportedMessageCount?.Format() ?? "(...)";
var totalMessageCountStr = totalMessageCount?.Format() ?? "(...)";
FilterStatisticsText = "Will export " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or > 0 ? "s." : ".");
FilterStatisticsText = verb + " " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or 0 ? "." : "s.");
OnPropertyChanged(nameof(FilterStatisticsText));
}

View File

@@ -24,7 +24,7 @@
<StackPanel>
<Button Command="{Binding OnClickToggleServerButton}" Content="{Binding ToggleServerButtonText}" IsEnabled="{Binding IsToggleServerButtonEnabled}" />
<TextBlock TextWrapping="Wrap" Margin="0 15">
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy and apply the tracking script again.
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy/paste the tracking script again.
</TextBlock>
<WrapPanel>
<StackPanel>

View File

@@ -8,7 +8,7 @@
Title="{Binding Title}"
Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
Width="800" Height="500"
MinWidth="500" MinHeight="275"
MinWidth="520" MinHeight="300"
WindowStartupLocation="CenterScreen"
Closed="OnClosed">

View File

@@ -0,0 +1,54 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="DHT.Desktop.Main.Pages.AttachmentsPage">
<Design.DataContext>
<pages:AttachmentsPageModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="Expander">
<Setter Property="Margin" Value="0 5 0 0" />
</Style>
<Style Selector="DataGridColumnHeader">
<Setter Property="FontWeight" Value="Medium" />
</Style>
<Style Selector="DataGridColumnHeader:nth-child(2)">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
<Style Selector="DataGridColumnHeader:nth-child(3)">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
<Style Selector="DataGridCell.right">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
</UserControl.Styles>
<StackPanel Orientation="Vertical" Spacing="20">
<DockPanel>
<Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" />
<TextBlock Text="{Binding DownloadMessage}" Margin="10 0 0 0" VerticalAlignment="Center" DockPanel.Dock="Left" />
<ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" />
</DockPanel>
<controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !DataContext.IsDownloading, RelativeSource={RelativeSource AncestorType=UserControl}}" />
<StackPanel Orientation="Vertical" Spacing="12">
<Expander Header="Download Status" IsExpanded="True">
<DataGrid Items="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
<DataGrid.Columns>
<DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" />
<DataGridTextColumn Header="Attachments" Binding="{Binding Items, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
<DataGridTextColumn Header="Size" Binding="{Binding Size, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" />
</DataGrid.Columns>
</DataGrid>
</Expander>
<StackPanel Orientation="Horizontal" Spacing="10">
<Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding HasFailedDownloads}">Retry Failed Downloads</Button>
</StackPanel>
</StackPanel>
</StackPanel>
</UserControl>

View File

@@ -0,0 +1,16 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Pages {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class AttachmentsPage : UserControl {
public AttachmentsPage() {
InitializeComponent();
}
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
}
}

View File

@@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using DHT.Desktop.Common;
using DHT.Desktop.Main.Controls;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Server.Download;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Pages {
sealed class AttachmentsPageModel : BaseModel, IDisposable {
private static readonly DownloadItemFilter EnqueuedItemFilter = new() {
IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued
}
};
private bool isThreadDownloadButtonEnabled = true;
public string ToggleDownloadButtonText => downloadThread == null ? "Start Downloading" : "Stop Downloading";
public bool IsToggleDownloadButtonEnabled {
get => isThreadDownloadButtonEnabled;
set => Change(ref isThreadDownloadButtonEnabled, value);
}
public string DownloadMessage { get; set; } = "";
public double DownloadProgress => allItemsCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / allItemsCount.Value;
public AttachmentFilterPanelModel FilterModel { get; }
private readonly StatisticsRow statisticsEnqueued = new ("Enqueued");
private readonly StatisticsRow statisticsDownloaded = new ("Downloaded");
private readonly StatisticsRow statisticsFailed = new ("Failed");
private readonly StatisticsRow statisticsSkipped = new ("Skipped");
public List<StatisticsRow> StatisticsRows {
get {
return new List<StatisticsRow> {
statisticsEnqueued,
statisticsDownloaded,
statisticsFailed,
statisticsSkipped
};
}
}
public bool IsDownloading => downloadThread != null;
public bool HasFailedDownloads => statisticsFailed.Items > 0;
private readonly IDatabaseFile db;
private readonly AsyncValueComputer<DownloadStatusStatistics>.Single downloadStatisticsComputer;
private BackgroundDownloadThread? downloadThread;
private int doneItemsCount;
private int? allItemsCount;
public AttachmentsPageModel() : this(DummyDatabaseFile.Instance) {}
public AttachmentsPageModel(IDatabaseFile db) {
this.db = db;
this.FilterModel = new AttachmentFilterPanelModel(db);
this.downloadStatisticsComputer = AsyncValueComputer<DownloadStatusStatistics>.WithResultProcessor(UpdateStatistics).WithOutdatedResults().BuildWithComputer(db.GetDownloadStatusStatistics);
this.downloadStatisticsComputer.Recompute();
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose();
DisposeDownloadThread();
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
if (IsDownloading) {
EnqueueDownloadItems();
}
else {
downloadStatisticsComputer.Recompute();
}
}
}
private void EnqueueDownloadItems() {
var filter = FilterModel.CreateFilter();
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
db.EnqueueDownloadItems(filter);
downloadStatisticsComputer.Recompute();
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
var hadFailedDownloads = HasFailedDownloads;
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
statisticsFailed.Items = statusStatistics.FailedCount;
statisticsFailed.Size = statusStatistics.FailedSize;
statisticsSkipped.Items = statusStatistics.SkippedCount;
statisticsSkipped.Size = statusStatistics.SkippedSize;
OnPropertyChanged(nameof(StatisticsRows));
if (hadFailedDownloads != HasFailedDownloads) {
OnPropertyChanged(nameof(HasFailedDownloads));
}
allItemsCount = doneItemsCount + statisticsEnqueued.Items;
UpdateDownloadMessage();
}
private void UpdateDownloadMessage() {
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (allItemsCount?.Format() ?? "?") : "";
OnPropertyChanged(nameof(DownloadMessage));
OnPropertyChanged(nameof(DownloadProgress));
}
private void DownloadThreadOnOnItemFinished(object? sender, DownloadItem e) {
++doneItemsCount;
UpdateDownloadMessage();
downloadStatisticsComputer.Recompute();
}
private void DownloadThreadOnOnServerStopped(object? sender, EventArgs e) {
downloadStatisticsComputer.Recompute();
IsToggleDownloadButtonEnabled = true;
}
public void OnClickToggleDownload() {
if (downloadThread == null) {
EnqueueDownloadItems();
downloadThread = new BackgroundDownloadThread(db);
downloadThread.OnItemFinished += DownloadThreadOnOnItemFinished;
downloadThread.OnServerStopped += DownloadThreadOnOnServerStopped;
}
else {
IsToggleDownloadButtonEnabled = false;
DisposeDownloadThread();
db.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
doneItemsCount = 0;
allItemsCount = null;
UpdateDownloadMessage();
}
OnPropertyChanged(nameof(ToggleDownloadButtonText));
OnPropertyChanged(nameof(IsDownloading));
}
public void OnClickRetryFailedDownloads() {
var allExceptFailedFilter = new DownloadItemFilter {
IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued,
DownloadStatus.Success
}
};
db.RemoveDownloadItems(allExceptFailedFilter, FilterRemovalMode.KeepMatching);
downloadStatisticsComputer.Recompute();
if (IsDownloading) {
EnqueueDownloadItems();
}
}
private void DisposeDownloadThread() {
if (downloadThread != null) {
downloadThread.OnItemFinished -= DownloadThreadOnOnItemFinished;
downloadThread.StopThread();
}
downloadThread = null;
}
public sealed class StatisticsRow {
public string State { get; }
public int Items { get; set; }
public ulong? Size { get; set; }
public StatisticsRow(string state) {
State = state;
}
}
}
}

View File

@@ -13,16 +13,16 @@
<UserControl.Styles>
<Style Selector="Expander">
<Setter Property="Margin" Value="0 25 0 0" />
<Setter Property="Margin" Value="0 5 0 0" />
</Style>
</UserControl.Styles>
<StackPanel Orientation="Vertical">
<StackPanel Orientation="Vertical" Spacing="20">
<StackPanel Orientation="Horizontal" VerticalAlignment="Top">
<Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
<Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
</StackPanel>
<controls:FilterPanel DataContext="{Binding FilterModel}" Margin="0 20 0 0" />
<controls:MessageFilterPanel DataContext="{Binding FilterModel}" />
<Expander Header="Database Tools">
<StackPanel Orientation="Vertical" Spacing="10">
<StackPanel Orientation="Vertical" Spacing="4">

View File

@@ -31,7 +31,7 @@ namespace DHT.Desktop.Main.Pages {
set => Change(ref hasFilters, value);
}
private FilterPanelModel FilterModel { get; }
private MessageFilterPanelModel FilterModel { get; }
private readonly Window window;
private readonly IDatabaseFile db;
@@ -43,7 +43,7 @@ namespace DHT.Desktop.Main.Pages {
this.window = window;
this.db = db;
FilterModel = new FilterPanelModel(window, db);
FilterModel = new MessageFilterPanelModel(window, db, "Will export");
FilterModel.FilterPropertyChanged += OnFilterPropertyChanged;
}
@@ -135,12 +135,12 @@ namespace DHT.Desktop.Main.Pages {
if (DatabaseToolFilterModeKeep) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.KeepMatching);
db.RemoveMessages(filter, FilterRemovalMode.KeepMatching);
}
}
else if (DatabaseToolFilterModeRemove) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.RemoveMatching);
db.RemoveMessages(filter, FilterRemovalMode.RemoveMatching);
}
}
}

View File

@@ -73,14 +73,14 @@
<DockPanel>
<Border Classes="statusBar" DockPanel.Dock="Bottom">
<DockPanel>
<TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Advanced</TextBlock>
<TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Attachments</TextBlock>
<controls:StatusBar DataContext="{Binding StatusBarModel}" DockPanel.Dock="Right" />
</DockPanel>
</Border>
<TabControl x:Name="TabControl" TabStripPlacement="Left" DockPanel.Dock="Top">
<TabControl.ItemsPanel>
<ItemsPanelTemplate>
<Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,*,Auto,Auto" />
<Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,Auto,*,Auto,Auto" />
</ItemsPanelTemplate>
</TabControl.ItemsPanel>
<TabItem x:Name="TabDatabase" Header="Database" Classes="first" Grid.Row="0">
@@ -93,17 +93,22 @@
<ContentPresenter Content="{Binding TrackingPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="2">
<TabItem x:Name="TabAttachments" Header="Attachments" Grid.Row="2">
<ScrollViewer>
<ContentPresenter Content="{Binding AttachmentsPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="3">
<ScrollViewer>
<ContentPresenter Content="{Binding ViewerPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabAdvanced" Header="Advanced" Grid.Row="4">
<TabItem x:Name="TabAdvanced" Header="Advanced" Grid.Row="5">
<ScrollViewer>
<ContentPresenter Content="{Binding AdvancedPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabDebug" Header="Debug" Grid.Row="5" IsVisible="{Binding HasDebugPage}">
<TabItem x:Name="TabDebug" Header="Debug" Grid.Row="6" IsVisible="{Binding HasDebugPage}">
<ScrollViewer>
<ContentPresenter Content="{Binding DebugPage}" Classes="page" />
</ScrollViewer>

View File

@@ -19,6 +19,9 @@ namespace DHT.Desktop.Main.Screens {
public TrackingPage TrackingPage { get; }
private TrackingPageModel TrackingPageModel { get; }
public AttachmentsPage AttachmentsPage { get; }
private AttachmentsPageModel AttachmentsPageModel { get; }
public ViewerPage ViewerPage { get; }
private ViewerPageModel ViewerPageModel { get; }
@@ -63,6 +66,9 @@ namespace DHT.Desktop.Main.Screens {
TrackingPageModel = new TrackingPageModel(window);
TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
AttachmentsPageModel = new AttachmentsPageModel(db);
AttachmentsPage = new AttachmentsPage { DataContext = AttachmentsPageModel };
ViewerPageModel = new ViewerPageModel(window, db);
ViewerPage = new ViewerPage { DataContext = ViewerPageModel };
@@ -92,6 +98,7 @@ namespace DHT.Desktop.Main.Screens {
public void Dispose() {
ServerLauncher.ServerManagementExceptionCaught -= ServerLauncherOnServerManagementExceptionCaught;
AttachmentsPageModel.Dispose();
ViewerPageModel.Dispose();
serverManager.Dispose();
}

View File

@@ -2,6 +2,10 @@
margin-bottom: 48px !important;
}
#app-mount div[class*="app-"] > div[class*="app-"] {
margin-bottom: 0 !important;
}
#dht-ctrl {
position: absolute;
bottom: 0;

View File

@@ -0,0 +1,15 @@
namespace DHT.Server.Data.Aggregations {
public sealed class DownloadStatusStatistics {
public int EnqueuedCount { get; internal set; }
public ulong EnqueuedSize { get; internal set; }
public int SuccessfulCount { get; internal set; }
public ulong SuccessfulSize { get; internal set; }
public int FailedCount { get; internal set; }
public ulong FailedSize { get; internal set; }
public int SkippedCount { get; internal set; }
public ulong SkippedSize { get; internal set; }
}
}

View File

@@ -0,0 +1,30 @@
using System;
using System.Net;
namespace DHT.Server.Data {
public readonly struct Download {
internal static Download NewEnqueued(string url, ulong size) {
return new Download(url, DownloadStatus.Enqueued, size);
}
internal static Download NewSuccess(string url, byte[] data) {
return new Download(url, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
}
internal static Download NewFailure(string url, HttpStatusCode? statusCode, ulong size) {
return new Download(url, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
}
public string Url { get; }
public DownloadStatus Status { get; }
public ulong Size { get; }
public byte[]? Data { get; }
private Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
Url = url;
Status = status;
Size = size;
Data = data;
}
}
}

View File

@@ -0,0 +1,12 @@
using System.Net;
namespace DHT.Server.Data {
/// <summary>
/// Extends <see cref="HttpStatusCode"/> with custom status codes in the range 0-99.
/// </summary>
public enum DownloadStatus {
Enqueued = 0,
GenericError = 1,
Success = HttpStatusCode.OK
}
}

View File

@@ -0,0 +1,15 @@
namespace DHT.Server.Data.Filters {
public sealed class AttachmentFilter {
public long? MaxBytes { get; set; } = null;
public DownloadItemRules? DownloadItemRule { get; set; } = null;
public bool IsEmpty => MaxBytes == null &&
DownloadItemRule == null;
public enum DownloadItemRules {
OnlyNotPresent,
OnlyPresent
}
}
}

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
namespace DHT.Server.Data.Filters {
public sealed class DownloadItemFilter {
public HashSet<DownloadStatus>? IncludeStatuses { get; set; } = null;
public HashSet<DownloadStatus>? ExcludeStatuses { get; set; } = null;
public bool IsEmpty => IncludeStatuses == null && ExcludeStatuses == null;
}
}

View File

@@ -1,5 +1,5 @@
namespace DHT.Server.Data.Filters {
public enum MessageFilterRemovalMode {
public enum FilterRemovalMode {
KeepMatching,
RemoveMatching
}

View File

@@ -6,6 +6,7 @@ namespace DHT.Server.Database {
private long totalChannels;
private long totalUsers;
private long? totalMessages;
private long? totalAttachments;
public long TotalServers {
get => totalServers;
@@ -27,12 +28,18 @@ namespace DHT.Server.Database {
internal set => Change(ref totalMessages, value);
}
public long? TotalAttachments {
get => totalAttachments;
internal set => Change(ref totalAttachments, value);
}
public DatabaseStatistics Clone() {
return new DatabaseStatistics {
totalServers = totalServers,
totalChannels = totalChannels,
totalUsers = TotalUsers,
totalMessages = totalMessages
totalMessages = totalMessages,
totalAttachments = totalAttachments
};
}
}

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Download;
namespace DHT.Server.Database {
public sealed class DummyDatabaseFile : IDatabaseFile {
@@ -39,7 +41,25 @@ namespace DHT.Server.Database {
return new();
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {}
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {}
public int CountAttachments(AttachmentFilter? filter = null) {
return new();
}
public void AddDownloads(IEnumerable<Data.Download> downloads) {}
public void EnqueueDownloadItems(AttachmentFilter? filter = null) {}
public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
return new();
}
public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {}
public DownloadStatusStatistics GetDownloadStatusStatistics() {
return new();
}
public void Vacuum() {}

View File

@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Download;
namespace DHT.Server.Database {
public interface IDatabaseFile : IDisposable {
@@ -20,7 +22,15 @@ namespace DHT.Server.Database {
void AddMessages(Message[] messages);
int CountMessages(MessageFilter? filter = null);
List<Message> GetMessages(MessageFilter? filter = null);
void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode);
void RemoveMessages(MessageFilter filter, FilterRemovalMode mode);
int CountAttachments(AttachmentFilter? filter = null);
void AddDownloads(IEnumerable<Data.Download> downloads);
void EnqueueDownloadItems(AttachmentFilter? filter = null);
List<DownloadItem> GetEnqueuedDownloadItems(int count);
void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);
DownloadStatusStatistics GetDownloadStatusStatistics();
void Vacuum();
}

View File

@@ -6,7 +6,7 @@ using DHT.Utils.Logging;
namespace DHT.Server.Database.Sqlite {
sealed class Schema {
internal const int Version = 3;
internal const int Version = 4;
private static readonly Log Log = Log.ForType<Schema>();
@@ -94,6 +94,7 @@ namespace DHT.Server.Database.Sqlite {
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
CreateDownloadsTable();
Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
@@ -114,6 +115,14 @@ namespace DHT.Server.Database.Sqlite {
replied_to_id INTEGER NOT NULL)");
}
private void CreateDownloadsTable() {
Execute(@"CREATE TABLE downloads (
url TEXT NOT NULL PRIMARY KEY,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB)");
}
private void UpgradeSchemas(int dbVersion) {
var perf = Log.Start("from version " + dbVersion);
@@ -145,6 +154,11 @@ namespace DHT.Server.Database.Sqlite {
perf.Step("Vacuum");
}
if (dbVersion <= 3) {
CreateDownloadsTable();
perf.Step("Upgrade to version 4");
}
perf.End();
}
}

View File

@@ -2,11 +2,12 @@ using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Server.Download;
using DHT.Utils.Collections;
using DHT.Utils.Logging;
using DHT.Utils.Tasks;
@@ -39,11 +40,14 @@ namespace DHT.Server.Database.Sqlite {
private readonly Log log;
private readonly SqliteConnectionPool pool;
private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
private readonly AsyncValueComputer<long>.Single totalAttachmentsComputer;
private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
this.pool = pool;
this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
this.totalAttachmentsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateAttachmentStatistics).WithOutdatedResults().BuildWithComputer(ComputeAttachmentStatistics);
this.Path = path;
this.Statistics = new DatabaseStatistics();
@@ -55,6 +59,7 @@ namespace DHT.Server.Database.Sqlite {
}
totalMessagesComputer.Recompute();
totalAttachmentsComputer.Recompute();
}
public void Dispose() {
@@ -194,6 +199,8 @@ namespace DHT.Server.Database.Sqlite {
cmd.ExecuteNonQuery();
}
bool addedAttachments = false;
using (var conn = pool.Take()) {
using var tx = conn.BeginTransaction();
@@ -274,6 +281,8 @@ namespace DHT.Server.Database.Sqlite {
}
if (!message.Attachments.IsEmpty) {
addedAttachments = true;
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
@@ -309,6 +318,10 @@ namespace DHT.Server.Database.Sqlite {
}
totalMessagesComputer.Recompute();
if (addedAttachments) {
totalAttachmentsComputer.Recompute();
}
}
public int CountMessages(MessageFilter? filter = null) {
@@ -356,27 +369,125 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
return list;
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == MessageFilterRemovalMode.KeepMatching);
if (string.IsNullOrEmpty(whereClause)) {
return;
}
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
if (!string.IsNullOrEmpty(whereClause)) {
var perf = log.Start();
// Rider is being stupid...
StringBuilder build = new StringBuilder()
.Append("DELETE ")
.Append("FROM messages")
.Append(whereClause);
DeleteFromTable("messages", whereClause);
totalMessagesComputer.Recompute();
using (var conn = pool.Take()) {
using var cmd = conn.Command(build.ToString());
perf.End();
}
}
public int CountAttachments(AttachmentFilter? filter = null) {
using var conn = pool.Take();
using var cmd = conn.Command("SELECT COUNT(*) FROM attachments a" + filter.GenerateWhereClause("a"));
using var reader = cmd.ExecuteReader();
return reader.Read() ? reader.GetInt32(0) : 0;
}
public void AddDownloads(IEnumerable<Data.Download> downloads) {
using var conn = pool.Take();
using var tx = conn.BeginTransaction();
using var cmd = conn.Upsert("downloads", new[] {
("url", SqliteType.Text),
("status", SqliteType.Integer),
("size", SqliteType.Integer),
("blob", SqliteType.Blob)
});
foreach (var download in downloads) {
cmd.Set(":url", download.Url);
cmd.Set(":status", (int) download.Status);
cmd.Set(":size", download.Size);
cmd.Set(":blob", download.Data);
cmd.ExecuteNonQuery();
}
totalMessagesComputer.Recompute();
perf.End();
tx.Commit();
}
public void EnqueueDownloadItems(AttachmentFilter? filter = null) {
using var conn = pool.Take();
using var cmd = conn.Command("INSERT INTO downloads (url, status, size) SELECT a.url, :enqueued, a.size FROM attachments a" + filter.GenerateWhereClause("a"));
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.ExecuteNonQuery();
}
public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
var list = new List<DownloadItem>();
using var conn = pool.Take();
using var cmd = conn.Command("SELECT url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
using var reader = cmd.ExecuteReader();
while (reader.Read()) {
list.Add(new DownloadItem {
Url = reader.GetString(0),
Size = reader.GetUint64(1)
});
}
return list;
}
public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
if (!string.IsNullOrEmpty(whereClause)) {
DeleteFromTable("downloads", whereClause);
}
}
public DownloadStatusStatistics GetDownloadStatusStatistics() {
static void LoadUndownloadedStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
using var cmd = conn.Command("SELECT IFNULL(COUNT(filtered.size), 0), IFNULL(SUM(filtered.size), 0) FROM (SELECT DISTINCT a.url, a.size FROM attachments a WHERE a.url NOT IN (SELECT d.url FROM downloads d)) filtered");
using var reader = cmd.ExecuteReader();
if (reader.Read()) {
result.SkippedCount = reader.GetInt32(0);
result.SkippedSize = reader.GetUint64(1);
}
}
static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
using var cmd = conn.Command(@"SELECT
IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
FROM downloads");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
using var reader = cmd.ExecuteReader();
if (reader.Read()) {
result.EnqueuedCount = reader.GetInt32(0);
result.EnqueuedSize = reader.GetUint64(1);
result.SuccessfulCount = reader.GetInt32(2);
result.SuccessfulSize = reader.GetUint64(3);
result.FailedCount = reader.GetInt32(4);
result.FailedSize = reader.GetUint64(5);
}
}
var result = new DownloadStatusStatistics();
using var conn = pool.Take();
LoadUndownloadedStatistics(conn, result);
LoadSuccessStatistics(conn, result);
return result;
}
private MultiDictionary<ulong, Attachment> GetAllAttachments() {
@@ -440,6 +551,19 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
return dict;
}
private void DeleteFromTable(string table, string whereClause) {
// Rider is being stupid...
StringBuilder build = new StringBuilder()
.Append("DELETE ")
.Append("FROM ")
.Append(table)
.Append(whereClause);
using var conn = pool.Take();
using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery();
}
public void Vacuum() {
using var conn = pool.Take();
using var cmd = conn.Command("VACUUM");
@@ -458,7 +582,7 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0;
}
private long ComputeMessageStatistics(CancellationToken token) {
private long ComputeMessageStatistics() {
using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
}
@@ -466,5 +590,14 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
private void UpdateMessageStatistics(long totalMessages) {
Statistics.TotalMessages = totalMessages;
}
private long ComputeAttachmentStatistics() {
using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM attachments") as long? ?? 0L;
}
private void UpdateAttachmentStatistics(long totalAttachments) {
Statistics.TotalAttachments = totalAttachments;
}
}
}

View File

@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite {
static class SqliteFilters {
public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.StartDate != null) {
where.AddCondition("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
}
if (filter.EndDate != null) {
where.AddCondition("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
}
if (filter.ChannelIds != null) {
where.AddCondition("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
}
if (filter.UserIds != null) {
where.AddCondition("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
}
if (filter.MessageIds != null) {
where.AddCondition("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
}
return where.Generate();
}
public static string GenerateWhereClause(this AttachmentFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.MaxBytes != null) {
where.AddCondition("size <= " + filter.MaxBytes);
}
if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyNotPresent) {
where.AddCondition("url NOT IN (SELECT url FROM downloads)");
}
else if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyPresent) {
where.AddCondition("url IN (SELECT url FROM downloads)");
}
return where.Generate();
}
public static string GenerateWhereClause(this DownloadItemFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.IncludeStatuses != null) {
where.AddCondition("status IN (" + filter.IncludeStatuses.In() + ")");
}
if (filter.ExcludeStatuses != null) {
where.AddCondition("status NOT IN (" + filter.ExcludeStatuses.In() + ")");
}
return where.Generate();
}
private static string In(this ISet<DownloadStatus> statuses) {
return string.Join(",", statuses.Select(static status => (int) status));
}
}
}

View File

@@ -1,50 +0,0 @@
using System;
using System.Collections.Generic;
using DHT.Server.Data.Filters;
namespace DHT.Server.Database.Sqlite {
static class SqliteMessageFilter {
public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
if (tableAlias != null) {
tableAlias += ".";
}
List<string> conditions = new();
if (filter.StartDate != null) {
conditions.Add(tableAlias + "timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
}
if (filter.EndDate != null) {
conditions.Add(tableAlias + "timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
}
if (filter.ChannelIds != null) {
conditions.Add(tableAlias + "channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
}
if (filter.UserIds != null) {
conditions.Add(tableAlias + "sender_id IN (" + string.Join(",", filter.UserIds) + ")");
}
if (filter.MessageIds != null) {
conditions.Add(tableAlias + "message_id IN (" + string.Join(",", filter.MessageIds) + ")");
}
if (conditions.Count == 0) {
return "";
}
if (invert) {
return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
}
else {
return " WHERE " + string.Join(" AND ", conditions);
}
}
}
}

View File

@@ -13,6 +13,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
}
private readonly object monitor = new ();
private readonly Random rand = new ();
private volatile bool isDisposed;
private readonly BlockingCollection<PooledConnection> free = new (new ConcurrentStack<PooledConnection>());
@@ -49,7 +50,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
while (conn == null) {
ThrowIfDisposed();
lock (monitor) {
if (free.TryTake(out conn, TimeSpan.FromMilliseconds(100))) {
if (free.TryTake(out conn, TimeSpan.FromMilliseconds(rand.Next(100, 200)))) {
used.Add(conn);
break;
}

View File

@@ -56,6 +56,10 @@ namespace DHT.Server.Database.Sqlite.Utils {
}
}
public static void AddAndSet(this SqliteCommand cmd, string key, SqliteType type, object? value) {
cmd.Parameters.Add(key, type).Value = value ?? DBNull.Value;
}
public static void Set(this SqliteCommand cmd, string key, object? value) {
cmd.Parameters[key].Value = value ?? DBNull.Value;
}

View File

@@ -0,0 +1,31 @@
using System.Collections.Generic;
namespace DHT.Server.Database.Sqlite.Utils {
sealed class SqliteWhereGenerator {
private readonly string? tableAlias;
private readonly bool invert;
private readonly List<string> conditions = new ();
public SqliteWhereGenerator(string? tableAlias, bool invert) {
this.tableAlias = tableAlias;
this.invert = invert;
}
public void AddCondition(string condition) {
conditions.Add(tableAlias == null ? condition : tableAlias + '.' + condition);
}
public string Generate() {
if (conditions.Count == 0) {
return "";
}
if (invert) {
return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
}
else {
return " WHERE " + string.Join(" AND ", conditions);
}
}
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Concurrent;
using System.Net;
using System.Threading;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Models;
namespace DHT.Server.Download {
public sealed class BackgroundDownloadThread : BaseModel {
private static readonly Log Log = Log.ForType<BackgroundDownloadThread>();
public event EventHandler<DownloadItem>? OnItemFinished {
add => parameters.OnItemFinished += value;
remove => parameters.OnItemFinished -= value;
}
public event EventHandler? OnServerStopped {
add => parameters.OnServerStopped += value;
remove => parameters.OnServerStopped -= value;
}
private readonly CancellationTokenSource cancellationTokenSource;
private readonly ThreadInstance.Parameters parameters;
public BackgroundDownloadThread(IDatabaseFile db) {
this.cancellationTokenSource = new CancellationTokenSource();
this.parameters = new ThreadInstance.Parameters(db, cancellationTokenSource);
var thread = new Thread(new ThreadInstance().Work) {
Name = "DHT download thread"
};
thread.Start(parameters);
}
public void StopThread() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {
Log.Warn("Attempted to stop background download thread after the cancellation token has been disposed.");
}
}
private sealed class ThreadInstance {
private const int QueueSize = 32;
public sealed class Parameters {
public event EventHandler<DownloadItem>? OnItemFinished;
public event EventHandler? OnServerStopped;
public IDatabaseFile Db { get; }
public CancellationTokenSource CancellationTokenSource { get; }
public Parameters(IDatabaseFile db, CancellationTokenSource cancellationTokenSource) {
Db = db;
CancellationTokenSource = cancellationTokenSource;
}
public void FireOnItemFinished(DownloadItem item) {
OnItemFinished?.Invoke(null, item);
}
public void FireOnServerStopped() {
OnServerStopped?.Invoke(null, EventArgs.Empty);
}
}
private readonly WebClient client = new ();
public ThreadInstance() {
client.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36";
}
public void Work(object? obj) {
var parameters = (Parameters) obj!;
var cancellationTokenSource = parameters.CancellationTokenSource;
var cancellationToken = cancellationTokenSource.Token;
var db = parameters.Db;
var queue = new ConcurrentQueue<DownloadItem>();
cancellationToken.Register(client.CancelAsync);
try {
while (!cancellationToken.IsCancellationRequested) {
FillQueue(db, queue, cancellationToken);
while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var item)) {
var url = item.Url;
Log.Debug("Downloading " + url + "...");
try {
db.AddDownloads(new [] { Data.Download.NewSuccess(url, client.DownloadData(url)) });
} catch (WebException e) {
db.AddDownloads(new [] { Data.Download.NewFailure(url, e.Response is HttpWebResponse response ? response.StatusCode : null, item.Size) });
Log.Error(e);
} finally {
parameters.FireOnItemFinished(item);
}
}
}
} catch (OperationCanceledException) {
//
} catch (ObjectDisposedException) {
//
} finally {
cancellationTokenSource.Dispose();
parameters.FireOnServerStopped();
}
}
private static void FillQueue(IDatabaseFile db, ConcurrentQueue<DownloadItem> queue, CancellationToken cancellationToken) {
while (!cancellationToken.IsCancellationRequested && queue.IsEmpty) {
var newItems = db.GetEnqueuedDownloadItems(QueueSize);
if (newItems.Count == 0) {
Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
else {
foreach (var item in newItems) {
queue.Enqueue(item);
}
}
}
}
}
}
}

View File

@@ -0,0 +1,6 @@
namespace DHT.Server.Download {
public readonly struct DownloadItem {
public string Url { get; init; }
public ulong Size { get; init; }
}
}

View File

@@ -20,7 +20,7 @@
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.2" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />

View File

@@ -11,8 +11,10 @@ namespace DHT.Utils.Tasks {
private readonly object stateLock = new ();
private CancellationTokenSource? currentCancellationTokenSource;
private Func<CancellationToken, TValue>? currentComputeFunction;
private SoftHardCancellationToken? currentCancellationTokenSource;
private bool wasHardCancelled = false;
private Func<TValue>? currentComputeFunction;
private bool hasComputeFunctionChanged = false;
private AsyncValueComputer(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler, bool processOutdatedResults) {
@@ -21,12 +23,21 @@ namespace DHT.Utils.Tasks {
this.processOutdatedResults = processOutdatedResults;
}
public void Compute(Func<CancellationToken, TValue> func) {
public void Cancel() {
lock (stateLock) {
wasHardCancelled = true;
currentCancellationTokenSource?.RequestHardCancellation();
}
}
public void Compute(Func<TValue> func) {
lock (stateLock) {
wasHardCancelled = false;
if (currentComputeFunction != null) {
currentComputeFunction = func;
hasComputeFunctionChanged = true;
currentCancellationTokenSource?.Cancel();
currentCancellationTokenSource?.RequestSoftCancellation();
}
else {
EnqueueComputation(func);
@@ -35,19 +46,18 @@ namespace DHT.Utils.Tasks {
}
[SuppressMessage("ReSharper", "MethodSupportsCancellation")]
private void EnqueueComputation(Func<CancellationToken, TValue> func) {
var cancellationTokenSource = new CancellationTokenSource();
var cancellationToken = cancellationTokenSource.Token;
private void EnqueueComputation(Func<TValue> func) {
var cancellationTokenSource = new SoftHardCancellationToken();
currentCancellationTokenSource?.Cancel();
currentCancellationTokenSource?.RequestSoftCancellation();
currentCancellationTokenSource = cancellationTokenSource;
currentComputeFunction = func;
hasComputeFunctionChanged = false;
var task = Task.Run(() => func(cancellationToken));
var task = Task.Run(func);
task.ContinueWith(t => {
if (processOutdatedResults || !cancellationToken.IsCancellationRequested) {
if (!cancellationTokenSource.IsCancelled(processOutdatedResults)) {
resultProcessor(t.Result);
}
}, CancellationToken.None, TaskContinuationOptions.NotOnFaulted, resultTaskScheduler);
@@ -60,11 +70,12 @@ namespace DHT.Utils.Tasks {
currentCancellationTokenSource = null;
}
if (hasComputeFunctionChanged) {
if (hasComputeFunctionChanged && !wasHardCancelled) {
EnqueueComputation(currentComputeFunction);
}
else {
currentComputeFunction = null;
hasComputeFunctionChanged = false;
}
}
});
@@ -72,9 +83,9 @@ namespace DHT.Utils.Tasks {
public sealed class Single {
private readonly AsyncValueComputer<TValue> baseComputer;
private readonly Func<CancellationToken, TValue> resultComputer;
private readonly Func<TValue> resultComputer;
internal Single(AsyncValueComputer<TValue> baseComputer, Func<CancellationToken, TValue> resultComputer) {
internal Single(AsyncValueComputer<TValue> baseComputer, Func<TValue> resultComputer) {
this.baseComputer = baseComputer;
this.resultComputer = resultComputer;
}
@@ -107,7 +118,7 @@ namespace DHT.Utils.Tasks {
return new AsyncValueComputer<TValue>(resultProcessor, resultTaskScheduler, processOutdatedResults);
}
public Single BuildWithComputer(Func<CancellationToken, TValue> resultComputer) {
public Single BuildWithComputer(Func<TValue> resultComputer) {
return new Single(Build(), resultComputer);
}
}

View File

@@ -0,0 +1,39 @@
using System;
using System.Threading;
namespace DHT.Utils.Tasks {
/// <summary>
/// Manages a pair of cancellation tokens that follow these rules:
/// <list type="number">
/// <item><description>If the soft token is cancelled, the hard token remains uncancelled.</description></item>
/// <item><description>If the hard token is cancelled, the soft token is also cancelled.</description></item>
/// </list>
/// </summary>
sealed class SoftHardCancellationToken : IDisposable {
private readonly CancellationTokenSource soft;
private readonly CancellationTokenSource hard;
public SoftHardCancellationToken() {
this.soft = new CancellationTokenSource();
this.hard = new CancellationTokenSource();
}
public bool IsCancelled(bool onlyHardCancellation) {
return (onlyHardCancellation ? hard : soft).IsCancellationRequested;
}
public void RequestSoftCancellation() {
soft.Cancel();
}
public void RequestHardCancellation() {
soft.Cancel();
hard.Cancel();
}
public void Dispose() {
soft.Dispose();
hard.Dispose();
}
}
}

View File

@@ -7,6 +7,6 @@ using DHT.Utils;
namespace DHT.Utils {
static class Version {
public const string Tag = "35.3.0.0";
public const string Tag = "36.0.0.0";
}
}

Binary file not shown.