1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-08-17 19:31:42 +02:00

22 Commits

Author SHA1 Message Date
3d435d0165 Release v36 (beta) 2022-05-29 12:40:13 +02:00
3e8151e1f3 Fix empty space between DHT tracker and app after a recent Discord update 2022-05-29 12:14:27 +02:00
9f98eba9c1 Update Avalonia to 0.10.14 2022-05-29 12:10:34 +02:00
6b54a80be1 Implement attachment downloads in desktop app 2022-05-29 12:09:47 +02:00
1e6e5c6f92 Refactor generating SQL "WHERE" clauses 2022-05-28 21:40:13 +02:00
2459c8ee1a Add randomization of SQLite connection pool delays to reduce chance of livelocks 2022-05-28 21:40:13 +02:00
d129a60d1c Add BytesValueConverter 2022-05-28 21:40:13 +02:00
65ecb0177c Fix wrong plural in the Viewer tab if the total amount of messages is zero 2022-05-28 21:40:13 +02:00
d51dcb0a84 Fix Viewer tab statistics (not computing in constructor, not cancelling after resetting filter) & tweak code 2022-05-28 21:40:12 +02:00
b13b85dedd Add support for cancelling async value computation 2022-05-28 21:38:42 +02:00
15e8b9da63 Tweak wording in the Advanced tab to reduce text length 2022-05-28 11:22:55 +02:00
9572f0f002 Rename MessageFilterRemovalMode to FilterRemovalMode 2022-05-24 22:02:36 +02:00
2f3b8b974c Rename MessageFilterPanel to FilterPanel 2022-05-24 22:02:36 +02:00
bff86b09c7 Update SQLite version to 3.35.5 2022-05-22 16:19:19 +02:00
5ca7cf09e8 Clarify and fix instructions for platform-portable releases
Closes #182
2022-05-21 22:20:26 +02:00
a1c93232d0 Cleanup temporary files when DHT is closed 2022-05-21 21:32:32 +02:00
db5f9d65db Change default viewer file name to the name of the database file 2022-05-21 20:11:09 +02:00
4cbf387e2a Optimize viewer export to support exporting large databases 2022-05-21 20:11:09 +02:00
64cf3c9fbb Calculate amount of exported messages asynchronously 2022-05-21 20:11:09 +02:00
a4ebd5eed6 Replace message statistics thread with new async value computer 2022-05-21 03:27:57 +02:00
06716330d6 Add utility for asynchronous value computation 2022-05-21 03:03:32 +02:00
1a6346677e Improve performance of check box dialogs by using ItemsRepeater instead of ItemsControl 2022-05-19 22:07:38 +02:00
50 changed files with 1551 additions and 381 deletions

View File

@@ -8,11 +8,13 @@
<entry key="Desktop/Dialogs/Message/MessageDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/FilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/AttachmentFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/AdvancedPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/AttachmentsPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/DatabasePage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/DebugPage.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Pages/TrackingPage.axaml" value="Desktop/Desktop.csproj" />

View File

@@ -7,6 +7,7 @@
<Application.Styles>
<FluentTheme Mode="Light" />
<StyleInclude Source="avares://Avalonia.Controls.DataGrid/Themes/Default.xaml"/>
<Style Selector="Button, CheckBox, RadioButton, Expander /template/ ToggleButton#ExpanderHeader">
<Setter Property="Cursor" Value="Hand" />
@@ -105,6 +106,7 @@
<Application.Resources>
<common:NumberValueConverter x:Key="NumberValueConverter" />
<common:BytesValueConverter x:Key="BytesValueConverter" />
<system:Double x:Key="ControlContentThemeFontSize">14</system:Double>
<CornerRadius x:Key="ControlCornerRadius">0</CornerRadius>
@@ -154,7 +156,7 @@
<SolidColorBrush x:Key="TextControlPlaceholderForegroundDisabled" Color="#AAAAAA" />
<Thickness x:Key="ExpanderHeaderPadding">15,0</Thickness>
<Thickness x:Key="ExpanderContentPadding">15</Thickness>
<Thickness x:Key="ExpanderContentPadding">12</Thickness>
<SolidColorBrush x:Key="ExpanderBorderBrush" Color="#697DAB" />
<SolidColorBrush x:Key="ExpanderBackground" Color="#697DAB" />
<SolidColorBrush x:Key="ExpanderForeground" Color="#FFFFFF" />

View File

@@ -0,0 +1,45 @@
using System;
using System.Globalization;
using Avalonia.Data.Converters;
namespace DHT.Desktop.Common {
sealed class BytesValueConverter : IValueConverter {
private static readonly string[] Units = {
"B",
"kB",
"MB",
"GB",
"TB"
};
private const int Scale = 1000;
private static string Convert(ulong size) {
int power = size == 0L ? 0 : (int) Math.Log(size, Scale);
int unit = power >= Units.Length ? Units.Length - 1 : power;
if (unit == 0) {
return string.Format(Program.Culture, "{0:n0}", size) + " " + Units[unit];
}
else {
double humanReadableSize = size / Math.Pow(Scale, unit);
return string.Format(Program.Culture, "{0:n0}", humanReadableSize) + " " + Units[unit];
}
}
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) {
if (value is long size and >= 0L) {
return Convert((ulong) size);
}
else if (value is ulong usize) {
return Convert(usize);
}
else {
return "-";
}
}
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) {
throw new NotSupportedException();
}
}
}

View File

@@ -21,9 +21,10 @@
<DebugType>none</DebugType>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Avalonia" Version="0.10.13" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.13" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.13" Condition=" '$(Configuration)' == 'Debug' " />
<PackageReference Include="Avalonia" Version="0.10.14" />
<PackageReference Include="Avalonia.Controls.DataGrid" Version="0.10.14" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.14" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.14" Condition=" '$(Configuration)' == 'Debug' " />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Server\Server.csproj" />

View File

@@ -34,8 +34,8 @@
<StackPanel Margin="20">
<ScrollViewer MaxHeight="400">
<ItemsControl Items="{Binding Items}">
<ItemsControl.ItemTemplate>
<ItemsRepeater Items="{Binding Items}">
<ItemsRepeater.ItemTemplate>
<DataTemplate>
<CheckBox IsChecked="{Binding Checked}">
<Label>
@@ -43,8 +43,8 @@
</Label>
</CheckBox>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
</ItemsRepeater.ItemTemplate>
</ItemsRepeater>
</ScrollViewer>
<Panel Classes="buttons">
<WrapPanel Classes="left">

View File

@@ -0,0 +1,51 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.AttachmentFilterPanel">
<Design.DataContext>
<controls:AttachmentFilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="ComboBox">
<Setter Property="Margin" Value="8 0 0 0" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<StackPanel>
<TextBlock Text="{Binding FilterStatisticsText}" />
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding LimitSize}">Limit Size</CheckBox>
<StackPanel Orientation="Horizontal">
<TextBox Text="{Binding MaximumSize}" IsEnabled="{Binding LimitSize}" HorizontalContentAlignment="Right" />
<ComboBox IsEnabled="{Binding LimitSize}" Items="{Binding Units}" SelectedItem="{Binding MaximumSizeUnit}">
<ComboBox.ItemTemplate>
<DataTemplate>
<TextBlock Text="{Binding Name}" />
</DataTemplate>
</ComboBox.ItemTemplate>
</ComboBox>
</StackPanel>
</StackPanel>
</WrapPanel>
</StackPanel>
</UserControl>

View File

@@ -0,0 +1,17 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Controls {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class AttachmentFilterPanel : UserControl {
public AttachmentFilterPanel() {
InitializeComponent();
}
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
}
}

View File

@@ -0,0 +1,125 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using DHT.Desktop.Common;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls {
sealed class AttachmentFilterPanelModel : BaseModel, IDisposable {
public sealed record Unit(string Name, int Scale);
private static readonly Unit[] AllUnits = {
new ("B", 1),
new ("kB", 1024),
new ("MB", 1024 * 1024)
};
private static readonly HashSet<string> FilterProperties = new () {
nameof(LimitSize),
nameof(MaximumSize),
nameof(MaximumSizeUnit)
};
public string FilterStatisticsText { get; private set; } = "";
private bool limitSize = false;
private int maximumSize = 0;
private Unit maximumSizeUnit = AllUnits[0];
public bool LimitSize {
get => limitSize;
set => Change(ref limitSize, value);
}
public int MaximumSize {
get => maximumSize;
set => Change(ref maximumSize, value);
}
public Unit MaximumSizeUnit {
get => maximumSizeUnit;
set => Change(ref maximumSizeUnit, value);
}
public IEnumerable<Unit> Units => AllUnits;
private readonly IDatabaseFile db;
private readonly string verb;
private readonly AsyncValueComputer<long> matchingAttachmentCountComputer;
private long? matchingAttachmentCount;
private long? totalAttachmentCount;
[Obsolete("Designer")]
public AttachmentFilterPanelModel() : this(DummyDatabaseFile.Instance) {}
public AttachmentFilterPanelModel(IDatabaseFile db, string verb = "Matches") {
this.db = db;
this.verb = verb;
this.matchingAttachmentCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetAttachmentCounts).Build();
UpdateFilterStatistics();
PropertyChanged += OnPropertyChanged;
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
}
private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
UpdateFilterStatistics();
}
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
totalAttachmentCount = db.Statistics.TotalAttachments;
UpdateFilterStatistics();
}
}
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
matchingAttachmentCountComputer.Cancel();
matchingAttachmentCount = totalAttachmentCount;
UpdateFilterStatisticsText();
}
else {
matchingAttachmentCount = null;
UpdateFilterStatisticsText();
matchingAttachmentCountComputer.Compute(() => db.CountAttachments(filter));
}
}
private void SetAttachmentCounts(long matchingAttachmentCount) {
this.matchingAttachmentCount = matchingAttachmentCount;
UpdateFilterStatisticsText();
}
private void UpdateFilterStatisticsText() {
var matchingAttachmentCountStr = matchingAttachmentCount?.Format() ?? "(...)";
var totalAttachmentCountStr = totalAttachmentCount?.Format() ?? "(...)";
FilterStatisticsText = verb + " " + matchingAttachmentCountStr + " out of " + totalAttachmentCountStr + " attachment" + (totalAttachmentCount is null or 1 ? "." : "s.");
OnPropertyChanged(nameof(FilterStatisticsText));
}
public AttachmentFilter CreateFilter() {
AttachmentFilter filter = new();
if (LimitSize) {
filter.MaxBytes = maximumSize * maximumSizeUnit.Scale;
}
return filter;
}
}
}

View File

@@ -1,60 +0,0 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.FilterPanel">
<Design.DataContext>
<controls:FilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="Grid > CalendarDatePicker">
<Setter Property="CornerRadius" Value="0" />
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="HorizontalContentAlignment" Value="Center" />
<Setter Property="VerticalAlignment" Value="Center" />
<Setter Property="VerticalContentAlignment" Value="Center" />
<Setter Property="IsTodayHighlighted" Value="True" />
<Setter Property="SelectedDateFormat" Value="Short" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
<Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
<Label Grid.Row="0" Grid.Column="0">From:</Label>
<CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
<Label Grid.Row="2" Grid.Column="0">To:</Label>
<CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
</Grid>
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
<Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
<TextBlock Text="{Binding ChannelFilterLabel}" />
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
<Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
<TextBlock Text="{Binding UserFilterLabel}" />
</StackPanel>
</WrapPanel>
</UserControl>

View File

@@ -0,0 +1,63 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.MessageFilterPanel">
<Design.DataContext>
<controls:MessageFilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="Grid > CalendarDatePicker">
<Setter Property="CornerRadius" Value="0" />
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="HorizontalContentAlignment" Value="Center" />
<Setter Property="VerticalAlignment" Value="Center" />
<Setter Property="VerticalContentAlignment" Value="Center" />
<Setter Property="IsTodayHighlighted" Value="True" />
<Setter Property="SelectedDateFormat" Value="Short" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<StackPanel>
<TextBlock Text="{Binding FilterStatisticsText}" />
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
<Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
<Label Grid.Row="0" Grid.Column="0">From:</Label>
<CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
<Label Grid.Row="2" Grid.Column="0">To:</Label>
<CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
</Grid>
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
<Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
<TextBlock Text="{Binding ChannelFilterLabel}" />
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
<Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
<TextBlock Text="{Binding UserFilterLabel}" />
</StackPanel>
</WrapPanel>
</StackPanel>
</UserControl>

View File

@@ -4,11 +4,11 @@ using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Controls {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class FilterPanel : UserControl {
public sealed class MessageFilterPanel : UserControl {
private CalendarDatePicker StartDatePicker => this.FindControl<CalendarDatePicker>("StartDatePicker");
private CalendarDatePicker EndDatePicker => this.FindControl<CalendarDatePicker>("EndDatePicker");
public FilterPanel() {
public MessageFilterPanel() {
InitializeComponent();
}
@@ -25,7 +25,7 @@ namespace DHT.Desktop.Main.Controls {
}
public void CalendarDatePicker_OnSelectedDateChanged(object? sender, SelectionChangedEventArgs e) {
if (DataContext is FilterPanelModel model) {
if (DataContext is MessageFilterPanelModel model) {
model.StartDate = StartDatePicker.SelectedDate;
model.EndDate = EndDatePicker.SelectedDate;
}

View File

@@ -12,9 +12,10 @@ using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls {
sealed class FilterPanelModel : BaseModel, IDisposable {
sealed class MessageFilterPanelModel : BaseModel, IDisposable {
private static readonly HashSet<string> FilterProperties = new () {
nameof(FilterByDate),
nameof(StartDate),
@@ -25,6 +26,8 @@ namespace DHT.Desktop.Main.Controls {
nameof(IncludedUsers)
};
public string FilterStatisticsText { get; private set; } = "";
public event PropertyChangedEventHandler? FilterPropertyChanged;
public bool HasAnyFilters => FilterByDate || FilterByChannel || FilterByUser;
@@ -88,14 +91,23 @@ namespace DHT.Desktop.Main.Controls {
private readonly Window window;
private readonly IDatabaseFile db;
private readonly string verb;
private readonly AsyncValueComputer<long> exportedMessageCountComputer;
private long? exportedMessageCount;
private long? totalMessageCount;
[Obsolete("Designer")]
public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
public MessageFilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
public FilterPanelModel(Window window, IDatabaseFile db) {
public MessageFilterPanelModel(Window window, IDatabaseFile db, string verb = "Matches") {
this.window = window;
this.db = db;
this.verb = verb;
this.exportedMessageCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetExportedMessageCount).Build();
UpdateFilterStatistics();
UpdateChannelFilterLabel();
UpdateUserFilterLabel();
@@ -109,6 +121,7 @@ namespace DHT.Desktop.Main.Controls {
private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
UpdateFilterStatistics();
FilterPropertyChanged?.Invoke(sender, e);
}
@@ -121,7 +134,11 @@ namespace DHT.Desktop.Main.Controls {
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
totalMessageCount = db.Statistics.TotalMessages;
UpdateFilterStatistics();
}
else if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
UpdateChannelFilterLabel();
}
else if (e.PropertyName == nameof(DatabaseStatistics.TotalUsers)) {
@@ -129,6 +146,33 @@ namespace DHT.Desktop.Main.Controls {
}
}
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
exportedMessageCountComputer.Cancel();
exportedMessageCount = totalMessageCount;
UpdateFilterStatisticsText();
}
else {
exportedMessageCount = null;
UpdateFilterStatisticsText();
exportedMessageCountComputer.Compute(() => db.CountMessages(filter));
}
}
private void SetExportedMessageCount(long exportedMessageCount) {
this.exportedMessageCount = exportedMessageCount;
UpdateFilterStatisticsText();
}
private void UpdateFilterStatisticsText() {
var exportedMessageCountStr = exportedMessageCount?.Format() ?? "(...)";
var totalMessageCountStr = totalMessageCount?.Format() ?? "(...)";
FilterStatisticsText = verb + " " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or 0 ? "." : "s.");
OnPropertyChanged(nameof(FilterStatisticsText));
}
public async void OpenChannelFilterDialog() {
var servers = db.GetAllServers().ToDictionary(static server => server.Id);
var items = new List<CheckBoxItem<ulong>>();

View File

@@ -24,7 +24,7 @@
<StackPanel>
<Button Command="{Binding OnClickToggleServerButton}" Content="{Binding ToggleServerButtonText}" IsEnabled="{Binding IsToggleServerButtonEnabled}" />
<TextBlock TextWrapping="Wrap" Margin="0 15">
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy and apply the tracking script again.
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy/paste the tracking script again.
</TextBlock>
<WrapPanel>
<StackPanel>

View File

@@ -8,7 +8,7 @@
Title="{Binding Title}"
Icon="avares://DiscordHistoryTracker/Resources/icon.ico"
Width="800" Height="500"
MinWidth="500" MinHeight="275"
MinWidth="520" MinHeight="300"
WindowStartupLocation="CenterScreen"
Closed="OnClosed">

View File

@@ -1,8 +1,10 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Avalonia;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
using DHT.Desktop.Main.Pages;
using JetBrains.Annotations;
namespace DHT.Desktop.Main {
@@ -30,6 +32,14 @@ namespace DHT.Desktop.Main {
if (DataContext is IDisposable disposable) {
disposable.Dispose();
}
foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {
try {
File.Delete(temporaryFile);
} catch (Exception) {
// ignored
}
}
}
}
}

View File

@@ -0,0 +1,54 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="DHT.Desktop.Main.Pages.AttachmentsPage">
<Design.DataContext>
<pages:AttachmentsPageModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="Expander">
<Setter Property="Margin" Value="0 5 0 0" />
</Style>
<Style Selector="DataGridColumnHeader">
<Setter Property="FontWeight" Value="Medium" />
</Style>
<Style Selector="DataGridColumnHeader:nth-child(2)">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
<Style Selector="DataGridColumnHeader:nth-child(3)">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
<Style Selector="DataGridCell.right">
<Setter Property="HorizontalContentAlignment" Value="Right" />
</Style>
</UserControl.Styles>
<StackPanel Orientation="Vertical" Spacing="20">
<DockPanel>
<Button Command="{Binding OnClickToggleDownload}" Content="{Binding ToggleDownloadButtonText}" IsEnabled="{Binding IsToggleDownloadButtonEnabled}" DockPanel.Dock="Left" />
<TextBlock Text="{Binding DownloadMessage}" Margin="10 0 0 0" VerticalAlignment="Center" DockPanel.Dock="Left" />
<ProgressBar Value="{Binding DownloadProgress}" IsVisible="{Binding IsDownloading}" Margin="15 0" VerticalAlignment="Center" DockPanel.Dock="Right" />
</DockPanel>
<controls:AttachmentFilterPanel DataContext="{Binding FilterModel}" IsEnabled="{Binding !DataContext.IsDownloading, RelativeSource={RelativeSource AncestorType=UserControl}}" />
<StackPanel Orientation="Vertical" Spacing="12">
<Expander Header="Download Status" IsExpanded="True">
<DataGrid Items="{Binding StatisticsRows}" AutoGenerateColumns="False" CanUserReorderColumns="False" CanUserResizeColumns="False" CanUserSortColumns="False" IsReadOnly="True">
<DataGrid.Columns>
<DataGridTextColumn Header="State" Binding="{Binding State}" Width="*" />
<DataGridTextColumn Header="Attachments" Binding="{Binding Items, Converter={StaticResource NumberValueConverter}}" Width="*" CellStyleClasses="right" />
<DataGridTextColumn Header="Size" Binding="{Binding Size, Converter={StaticResource BytesValueConverter}}" Width="*" CellStyleClasses="right" />
</DataGrid.Columns>
</DataGrid>
</Expander>
<StackPanel Orientation="Horizontal" Spacing="10">
<Button Command="{Binding OnClickRetryFailedDownloads}" IsEnabled="{Binding HasFailedDownloads}">Retry Failed Downloads</Button>
</StackPanel>
</StackPanel>
</StackPanel>
</UserControl>

View File

@@ -0,0 +1,16 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Pages {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class AttachmentsPage : UserControl {
public AttachmentsPage() {
InitializeComponent();
}
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
}
}

View File

@@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using DHT.Desktop.Common;
using DHT.Desktop.Main.Controls;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Server.Download;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Pages {
sealed class AttachmentsPageModel : BaseModel, IDisposable {
private static readonly DownloadItemFilter EnqueuedItemFilter = new() {
IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued
}
};
private bool isThreadDownloadButtonEnabled = true;
public string ToggleDownloadButtonText => downloadThread == null ? "Start Downloading" : "Stop Downloading";
public bool IsToggleDownloadButtonEnabled {
get => isThreadDownloadButtonEnabled;
set => Change(ref isThreadDownloadButtonEnabled, value);
}
public string DownloadMessage { get; set; } = "";
public double DownloadProgress => allItemsCount is null or 0 ? 0.0 : 100.0 * doneItemsCount / allItemsCount.Value;
public AttachmentFilterPanelModel FilterModel { get; }
private readonly StatisticsRow statisticsEnqueued = new ("Enqueued");
private readonly StatisticsRow statisticsDownloaded = new ("Downloaded");
private readonly StatisticsRow statisticsFailed = new ("Failed");
private readonly StatisticsRow statisticsSkipped = new ("Skipped");
public List<StatisticsRow> StatisticsRows {
get {
return new List<StatisticsRow> {
statisticsEnqueued,
statisticsDownloaded,
statisticsFailed,
statisticsSkipped
};
}
}
public bool IsDownloading => downloadThread != null;
public bool HasFailedDownloads => statisticsFailed.Items > 0;
private readonly IDatabaseFile db;
private readonly AsyncValueComputer<DownloadStatusStatistics>.Single downloadStatisticsComputer;
private BackgroundDownloadThread? downloadThread;
private int doneItemsCount;
private int? allItemsCount;
public AttachmentsPageModel() : this(DummyDatabaseFile.Instance) {}
public AttachmentsPageModel(IDatabaseFile db) {
this.db = db;
this.FilterModel = new AttachmentFilterPanelModel(db);
this.downloadStatisticsComputer = AsyncValueComputer<DownloadStatusStatistics>.WithResultProcessor(UpdateStatistics).WithOutdatedResults().BuildWithComputer(db.GetDownloadStatusStatistics);
this.downloadStatisticsComputer.Recompute();
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose();
DisposeDownloadThread();
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalAttachments)) {
if (IsDownloading) {
EnqueueDownloadItems();
}
else {
downloadStatisticsComputer.Recompute();
}
}
}
private void EnqueueDownloadItems() {
var filter = FilterModel.CreateFilter();
filter.DownloadItemRule = AttachmentFilter.DownloadItemRules.OnlyNotPresent;
db.EnqueueDownloadItems(filter);
downloadStatisticsComputer.Recompute();
}
private void UpdateStatistics(DownloadStatusStatistics statusStatistics) {
var hadFailedDownloads = HasFailedDownloads;
statisticsEnqueued.Items = statusStatistics.EnqueuedCount;
statisticsEnqueued.Size = statusStatistics.EnqueuedSize;
statisticsDownloaded.Items = statusStatistics.SuccessfulCount;
statisticsDownloaded.Size = statusStatistics.SuccessfulSize;
statisticsFailed.Items = statusStatistics.FailedCount;
statisticsFailed.Size = statusStatistics.FailedSize;
statisticsSkipped.Items = statusStatistics.SkippedCount;
statisticsSkipped.Size = statusStatistics.SkippedSize;
OnPropertyChanged(nameof(StatisticsRows));
if (hadFailedDownloads != HasFailedDownloads) {
OnPropertyChanged(nameof(HasFailedDownloads));
}
allItemsCount = doneItemsCount + statisticsEnqueued.Items;
UpdateDownloadMessage();
}
private void UpdateDownloadMessage() {
DownloadMessage = IsDownloading ? doneItemsCount.Format() + " / " + (allItemsCount?.Format() ?? "?") : "";
OnPropertyChanged(nameof(DownloadMessage));
OnPropertyChanged(nameof(DownloadProgress));
}
private void DownloadThreadOnOnItemFinished(object? sender, DownloadItem e) {
++doneItemsCount;
UpdateDownloadMessage();
downloadStatisticsComputer.Recompute();
}
private void DownloadThreadOnOnServerStopped(object? sender, EventArgs e) {
downloadStatisticsComputer.Recompute();
IsToggleDownloadButtonEnabled = true;
}
public void OnClickToggleDownload() {
if (downloadThread == null) {
EnqueueDownloadItems();
downloadThread = new BackgroundDownloadThread(db);
downloadThread.OnItemFinished += DownloadThreadOnOnItemFinished;
downloadThread.OnServerStopped += DownloadThreadOnOnServerStopped;
}
else {
IsToggleDownloadButtonEnabled = false;
DisposeDownloadThread();
db.RemoveDownloadItems(EnqueuedItemFilter, FilterRemovalMode.RemoveMatching);
doneItemsCount = 0;
allItemsCount = null;
UpdateDownloadMessage();
}
OnPropertyChanged(nameof(ToggleDownloadButtonText));
OnPropertyChanged(nameof(IsDownloading));
}
public void OnClickRetryFailedDownloads() {
var allExceptFailedFilter = new DownloadItemFilter {
IncludeStatuses = new HashSet<DownloadStatus> {
DownloadStatus.Enqueued,
DownloadStatus.Success
}
};
db.RemoveDownloadItems(allExceptFailedFilter, FilterRemovalMode.KeepMatching);
downloadStatisticsComputer.Recompute();
if (IsDownloading) {
EnqueueDownloadItems();
}
}
private void DisposeDownloadThread() {
if (downloadThread != null) {
downloadThread.OnItemFinished -= DownloadThreadOnOnItemFinished;
downloadThread.StopThread();
}
downloadThread = null;
}
public sealed class StatisticsRow {
public string State { get; }
public int Items { get; set; }
public ulong? Size { get; set; }
public StatisticsRow(string state) {
State = state;
}
}
}
}

View File

@@ -5,9 +5,7 @@
xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="DHT.Desktop.Main.Pages.ViewerPage"
AttachedToVisualTree="OnAttachedToVisualTree"
DetachedFromVisualTree="OnDetachedFromVisualTree">
x:Class="DHT.Desktop.Main.Pages.ViewerPage">
<Design.DataContext>
<pages:ViewerPageModel />
@@ -15,17 +13,16 @@
<UserControl.Styles>
<Style Selector="Expander">
<Setter Property="Margin" Value="0 25 0 0" />
<Setter Property="Margin" Value="0 5 0 0" />
</Style>
</UserControl.Styles>
<StackPanel Orientation="Vertical">
<StackPanel Orientation="Vertical" Spacing="20">
<StackPanel Orientation="Horizontal" VerticalAlignment="Top">
<Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
<Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
</StackPanel>
<TextBlock Text="{Binding ExportedMessageText}" Margin="0 20 0 0" />
<controls:FilterPanel DataContext="{Binding FilterModel}" />
<controls:MessageFilterPanel DataContext="{Binding FilterModel}" />
<Expander Header="Database Tools">
<StackPanel Orientation="Vertical" Spacing="10">
<StackPanel Orientation="Vertical" Spacing="4">

View File

@@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
@@ -13,17 +12,5 @@ namespace DHT.Desktop.Main.Pages {
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
public void OnAttachedToVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(true);
}
}
public void OnDetachedFromVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(false);
}
}
}
}

View File

@@ -1,8 +1,10 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Avalonia.Controls;
@@ -17,8 +19,8 @@ using static DHT.Desktop.Program;
namespace DHT.Desktop.Main.Pages {
sealed class ViewerPageModel : BaseModel, IDisposable {
public string ExportedMessageText { get; private set; } = "";
public static readonly ConcurrentBag<string> TemporaryFiles = new ();
public bool DatabaseToolFilterModeKeep { get; set; } = true;
public bool DatabaseToolFilterModeRemove { get; set; } = false;
@@ -29,13 +31,11 @@ namespace DHT.Desktop.Main.Pages {
set => Change(ref hasFilters, value);
}
private FilterPanelModel FilterModel { get; }
private MessageFilterPanelModel FilterModel { get; }
private readonly Window window;
private readonly IDatabaseFile db;
private bool isPageVisible = false;
[Obsolete("Designer")]
public ViewerPageModel() : this(null!, DummyDatabaseFile.Instance) {}
@@ -43,51 +43,53 @@ namespace DHT.Desktop.Main.Pages {
this.window = window;
this.db = db;
FilterModel = new FilterPanelModel(window, db);
FilterModel = new MessageFilterPanelModel(window, db, "Will export");
FilterModel.FilterPropertyChanged += OnFilterPropertyChanged;
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose();
}
public void SetPageVisible(bool isPageVisible) {
this.isPageVisible = isPageVisible;
if (isPageVisible) {
UpdateStatistics();
}
}
private void OnFilterPropertyChanged(object? sender, PropertyChangedEventArgs e) {
UpdateStatistics();
HasFilters = FilterModel.HasAnyFilters;
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (isPageVisible && e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
UpdateStatistics();
private async Task WriteViewerFile(string path) {
const string ArchiveTag = "/*[ARCHIVE]*/";
string indexFile = await Resources.ReadTextAsync("Viewer/index.html");
string viewerTemplate = indexFile.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'));
int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag);
int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length;
string jsonTempFile = path + ".tmp";
await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) {
await ViewerJsonExport.Generate(jsonStream, db, FilterModel.CreateFilter());
char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)];
jsonStream.Position = 0;
await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) {
await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]);
using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) {
int readBytes;
while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) {
string jsonChunk = new string(jsonBuffer, 0, readBytes);
await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk));
}
}
await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]);
}
}
}
private void UpdateStatistics() {
var filter = FilterModel.CreateFilter();
var allMessagesCount = db.Statistics.TotalMessages?.Format() ?? "?";
var filteredMessagesCount = filter.IsEmpty ? allMessagesCount : db.CountMessages(filter).Format();
ExportedMessageText = "Will export " + filteredMessagesCount + " out of " + allMessagesCount + " message(s).";
OnPropertyChanged(nameof(ExportedMessageText));
}
private async Task<string> GenerateViewerContents() {
string json = ViewerJsonExport.Generate(db, FilterModel.CreateFilter());
string index = await Resources.ReadTextAsync("Viewer/index.html");
string viewer = index.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'))
.Replace("/*[ARCHIVE]*/", HttpUtility.JavaScriptStringEncode(json));
return viewer;
File.Delete(jsonTempFile);
}
public async void OnClickOpenViewer() {
@@ -101,8 +103,10 @@ namespace DHT.Desktop.Main.Pages {
fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html");
}
TemporaryFiles.Add(fullPath);
Directory.CreateDirectory(rootPath);
await File.WriteAllTextAsync(fullPath, await GenerateViewerContents());
await WriteViewerFile(fullPath);
Process.Start(new ProcessStartInfo(fullPath) { UseShellExecute = true });
}
@@ -110,7 +114,7 @@ namespace DHT.Desktop.Main.Pages {
public async void OnClickSaveViewer() {
var dialog = new SaveFileDialog {
Title = "Save Viewer",
InitialFileName = "archive.html",
InitialFileName = Path.GetFileNameWithoutExtension(db.Path) + ".html",
Directory = Path.GetDirectoryName(db.Path),
Filters = new List<FileDialogFilter> {
new() {
@@ -122,7 +126,7 @@ namespace DHT.Desktop.Main.Pages {
string? path = await dialog;
if (!string.IsNullOrEmpty(path)) {
await File.WriteAllTextAsync(path, await GenerateViewerContents());
await WriteViewerFile(path);
}
}
@@ -131,12 +135,12 @@ namespace DHT.Desktop.Main.Pages {
if (DatabaseToolFilterModeKeep) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.KeepMatching);
db.RemoveMessages(filter, FilterRemovalMode.KeepMatching);
}
}
else if (DatabaseToolFilterModeRemove) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.RemoveMatching);
db.RemoveMessages(filter, FilterRemovalMode.RemoveMatching);
}
}
}

View File

@@ -73,14 +73,14 @@
<DockPanel>
<Border Classes="statusBar" DockPanel.Dock="Bottom">
<DockPanel>
<TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Advanced</TextBlock>
<TextBlock Classes="invisibleTabItem" DockPanel.Dock="Left">Attachments</TextBlock>
<controls:StatusBar DataContext="{Binding StatusBarModel}" DockPanel.Dock="Right" />
</DockPanel>
</Border>
<TabControl x:Name="TabControl" TabStripPlacement="Left" DockPanel.Dock="Top">
<TabControl.ItemsPanel>
<ItemsPanelTemplate>
<Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,*,Auto,Auto" />
<Grid ColumnDefinitions="Auto" RowDefinitions="Auto,Auto,Auto,Auto,*,Auto,Auto" />
</ItemsPanelTemplate>
</TabControl.ItemsPanel>
<TabItem x:Name="TabDatabase" Header="Database" Classes="first" Grid.Row="0">
@@ -93,17 +93,22 @@
<ContentPresenter Content="{Binding TrackingPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="2">
<TabItem x:Name="TabAttachments" Header="Attachments" Grid.Row="2">
<ScrollViewer>
<ContentPresenter Content="{Binding AttachmentsPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabViewer" Header="Viewer" Grid.Row="3">
<ScrollViewer>
<ContentPresenter Content="{Binding ViewerPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabAdvanced" Header="Advanced" Grid.Row="4">
<TabItem x:Name="TabAdvanced" Header="Advanced" Grid.Row="5">
<ScrollViewer>
<ContentPresenter Content="{Binding AdvancedPage}" Classes="page" />
</ScrollViewer>
</TabItem>
<TabItem x:Name="TabDebug" Header="Debug" Grid.Row="5" IsVisible="{Binding HasDebugPage}">
<TabItem x:Name="TabDebug" Header="Debug" Grid.Row="6" IsVisible="{Binding HasDebugPage}">
<ScrollViewer>
<ContentPresenter Content="{Binding DebugPage}" Classes="page" />
</ScrollViewer>

View File

@@ -19,6 +19,9 @@ namespace DHT.Desktop.Main.Screens {
public TrackingPage TrackingPage { get; }
private TrackingPageModel TrackingPageModel { get; }
public AttachmentsPage AttachmentsPage { get; }
private AttachmentsPageModel AttachmentsPageModel { get; }
public ViewerPage ViewerPage { get; }
private ViewerPageModel ViewerPageModel { get; }
@@ -63,6 +66,9 @@ namespace DHT.Desktop.Main.Screens {
TrackingPageModel = new TrackingPageModel(window);
TrackingPage = new TrackingPage { DataContext = TrackingPageModel };
AttachmentsPageModel = new AttachmentsPageModel(db);
AttachmentsPage = new AttachmentsPage { DataContext = AttachmentsPageModel };
ViewerPageModel = new ViewerPageModel(window, db);
ViewerPage = new ViewerPage { DataContext = ViewerPageModel };
@@ -92,6 +98,7 @@ namespace DHT.Desktop.Main.Screens {
public void Dispose() {
ServerLauncher.ServerManagementExceptionCaught -= ServerLauncherOnServerManagementExceptionCaught;
AttachmentsPageModel.Dispose();
ViewerPageModel.Dispose();
serverManager.Dispose();
}

View File

@@ -2,6 +2,10 @@
margin-bottom: 48px !important;
}
#app-mount div[class*="app-"] > div[class*="app-"] {
margin-bottom: 0 !important;
}
#dht-ctrl {
position: absolute;
bottom: 0;

View File

@@ -0,0 +1,15 @@
namespace DHT.Server.Data.Aggregations {
public sealed class DownloadStatusStatistics {
public int EnqueuedCount { get; internal set; }
public ulong EnqueuedSize { get; internal set; }
public int SuccessfulCount { get; internal set; }
public ulong SuccessfulSize { get; internal set; }
public int FailedCount { get; internal set; }
public ulong FailedSize { get; internal set; }
public int SkippedCount { get; internal set; }
public ulong SkippedSize { get; internal set; }
}
}

View File

@@ -0,0 +1,30 @@
using System;
using System.Net;
namespace DHT.Server.Data {
public readonly struct Download {
internal static Download NewEnqueued(string url, ulong size) {
return new Download(url, DownloadStatus.Enqueued, size);
}
internal static Download NewSuccess(string url, byte[] data) {
return new Download(url, DownloadStatus.Success, (ulong) Math.Max(data.LongLength, 0), data);
}
internal static Download NewFailure(string url, HttpStatusCode? statusCode, ulong size) {
return new Download(url, statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError, size);
}
public string Url { get; }
public DownloadStatus Status { get; }
public ulong Size { get; }
public byte[]? Data { get; }
private Download(string url, DownloadStatus status, ulong size, byte[]? data = null) {
Url = url;
Status = status;
Size = size;
Data = data;
}
}
}

View File

@@ -0,0 +1,12 @@
using System.Net;
namespace DHT.Server.Data {
/// <summary>
/// Extends <see cref="HttpStatusCode"/> with custom status codes in the range 0-99.
/// </summary>
public enum DownloadStatus {
Enqueued = 0,
GenericError = 1,
Success = HttpStatusCode.OK
}
}

View File

@@ -0,0 +1,15 @@
namespace DHT.Server.Data.Filters {
public sealed class AttachmentFilter {
public long? MaxBytes { get; set; } = null;
public DownloadItemRules? DownloadItemRule { get; set; } = null;
public bool IsEmpty => MaxBytes == null &&
DownloadItemRule == null;
public enum DownloadItemRules {
OnlyNotPresent,
OnlyPresent
}
}
}

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
namespace DHT.Server.Data.Filters {
public sealed class DownloadItemFilter {
public HashSet<DownloadStatus>? IncludeStatuses { get; set; } = null;
public HashSet<DownloadStatus>? ExcludeStatuses { get; set; } = null;
public bool IsEmpty => IncludeStatuses == null && ExcludeStatuses == null;
}
}

View File

@@ -1,5 +1,5 @@
namespace DHT.Server.Data.Filters {
public enum MessageFilterRemovalMode {
public enum FilterRemovalMode {
KeepMatching,
RemoveMatching
}

View File

@@ -6,6 +6,7 @@ namespace DHT.Server.Database {
private long totalChannels;
private long totalUsers;
private long? totalMessages;
private long? totalAttachments;
public long TotalServers {
get => totalServers;
@@ -27,12 +28,18 @@ namespace DHT.Server.Database {
internal set => Change(ref totalMessages, value);
}
public long? TotalAttachments {
get => totalAttachments;
internal set => Change(ref totalAttachments, value);
}
public DatabaseStatistics Clone() {
return new DatabaseStatistics {
totalServers = totalServers,
totalChannels = totalChannels,
totalUsers = TotalUsers,
totalMessages = totalMessages
totalMessages = totalMessages,
totalAttachments = totalAttachments
};
}
}

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Download;
namespace DHT.Server.Database {
public sealed class DummyDatabaseFile : IDatabaseFile {
@@ -39,7 +41,25 @@ namespace DHT.Server.Database {
return new();
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {}
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {}
public int CountAttachments(AttachmentFilter? filter = null) {
return new();
}
public void AddDownloads(IEnumerable<Data.Download> downloads) {}
public void EnqueueDownloadItems(AttachmentFilter? filter = null) {}
public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
return new();
}
public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {}
public DownloadStatusStatistics GetDownloadStatusStatistics() {
return new();
}
public void Vacuum() {}

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Utils.Logging;
@@ -9,7 +11,7 @@ namespace DHT.Server.Database.Export {
public static class ViewerJsonExport {
private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
public static string Generate(IDatabaseFile db, MessageFilter? filter = null) {
public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) {
var perf = Log.Start();
var includedUserIds = new HashSet<ulong>();
@@ -37,17 +39,20 @@ namespace DHT.Server.Database.Export {
perf.Step("Collect database data");
var value = new {
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
};
perf.Step("Generate value object");
var opts = new JsonSerializerOptions();
opts.Converters.Add(new ViewerJsonSnowflakeSerializer());
var json = JsonSerializer.Serialize(new {
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
}, opts);
await JsonSerializer.SerializeAsync(stream, value, opts);
perf.Step("Serialize to JSON");
perf.End();
return json;
}
private static object GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds, out List<string> userindex, out Dictionary<ulong, object> userIndices) {
@@ -159,8 +164,8 @@ namespace DHT.Server.Database.Export {
}
if (!message.Attachments.IsEmpty) {
obj["a"] = message.Attachments.Select(static attachment => new {
url = attachment.Url
obj["a"] = message.Attachments.Select(static attachment => new Dictionary<string, object> {
{ "url", attachment.Url }
}).ToArray();
}

View File

@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Download;
namespace DHT.Server.Database {
public interface IDatabaseFile : IDisposable {
@@ -20,7 +22,15 @@ namespace DHT.Server.Database {
void AddMessages(Message[] messages);
int CountMessages(MessageFilter? filter = null);
List<Message> GetMessages(MessageFilter? filter = null);
void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode);
void RemoveMessages(MessageFilter filter, FilterRemovalMode mode);
int CountAttachments(AttachmentFilter? filter = null);
void AddDownloads(IEnumerable<Data.Download> downloads);
void EnqueueDownloadItems(AttachmentFilter? filter = null);
List<DownloadItem> GetEnqueuedDownloadItems(int count);
void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode);
DownloadStatusStatistics GetDownloadStatusStatistics();
void Vacuum();
}

View File

@@ -6,7 +6,7 @@ using DHT.Utils.Logging;
namespace DHT.Server.Database.Sqlite {
sealed class Schema {
internal const int Version = 3;
internal const int Version = 4;
private static readonly Log Log = Log.ForType<Schema>();
@@ -94,6 +94,7 @@ namespace DHT.Server.Database.Sqlite {
CreateMessageEditTimestampTable();
CreateMessageRepliedToTable();
CreateDownloadsTable();
Execute("CREATE INDEX attachments_message_ix ON attachments(message_id)");
Execute("CREATE INDEX embeds_message_ix ON embeds(message_id)");
@@ -114,6 +115,14 @@ namespace DHT.Server.Database.Sqlite {
replied_to_id INTEGER NOT NULL)");
}
private void CreateDownloadsTable() {
Execute(@"CREATE TABLE downloads (
url TEXT NOT NULL PRIMARY KEY,
status INTEGER NOT NULL,
size INTEGER NOT NULL,
blob BLOB)");
}
private void UpgradeSchemas(int dbVersion) {
var perf = Log.Start("from version " + dbVersion);
@@ -145,6 +154,11 @@ namespace DHT.Server.Database.Sqlite {
perf.Step("Vacuum");
}
if (dbVersion <= 3) {
CreateDownloadsTable();
perf.Step("Upgrade to version 4");
}
perf.End();
}
}

View File

@@ -4,10 +4,13 @@ using System.Collections.Immutable;
using System.Text;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Server.Download;
using DHT.Utils.Collections;
using DHT.Utils.Logging;
using DHT.Utils.Tasks;
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite {
@@ -36,12 +39,15 @@ namespace DHT.Server.Database.Sqlite {
private readonly Log log;
private readonly SqliteConnectionPool pool;
private readonly SqliteMessageStatisticsThread messageStatisticsThread;
private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
private readonly AsyncValueComputer<long>.Single totalAttachmentsComputer;
private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
this.pool = pool;
this.messageStatisticsThread = new SqliteMessageStatisticsThread(pool, UpdateMessageStatistics);
this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
this.totalAttachmentsComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateAttachmentStatistics).WithOutdatedResults().BuildWithComputer(ComputeAttachmentStatistics);
this.Path = path;
this.Statistics = new DatabaseStatistics();
@@ -52,11 +58,11 @@ namespace DHT.Server.Database.Sqlite {
UpdateUserStatistics(conn);
}
messageStatisticsThread.RequestUpdate();
totalMessagesComputer.Recompute();
totalAttachmentsComputer.Recompute();
}
public void Dispose() {
messageStatisticsThread.Dispose();
pool.Dispose();
}
@@ -192,120 +198,130 @@ namespace DHT.Server.Database.Sqlite {
cmd.Set(":message_id", id);
cmd.ExecuteNonQuery();
}
bool addedAttachments = false;
using var conn = pool.Take();
using var tx = conn.BeginTransaction();
using (var conn = pool.Take()) {
using var tx = conn.BeginTransaction();
using var messageCmd = conn.Upsert("messages", new[] {
("message_id", SqliteType.Integer),
("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer),
("text", SqliteType.Text),
("timestamp", SqliteType.Integer)
});
using var messageCmd = conn.Upsert("messages", new[] {
("message_id", SqliteType.Integer),
("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer),
("text", SqliteType.Text),
("timestamp", SqliteType.Integer)
});
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
});
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
});
using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
});
using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
});
using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer),
("name", SqliteType.Text),
("type", SqliteType.Text),
("url", SqliteType.Text),
("size", SqliteType.Integer)
});
using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer),
("name", SqliteType.Text),
("type", SqliteType.Text),
("url", SqliteType.Text),
("size", SqliteType.Integer)
});
using var embedCmd = conn.Insert("embeds", new[] {
("message_id", SqliteType.Integer),
("json", SqliteType.Text)
});
using var embedCmd = conn.Insert("embeds", new[] {
("message_id", SqliteType.Integer),
("json", SqliteType.Text)
});
using var reactionCmd = conn.Insert("reactions", new[] {
("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer)
});
using var reactionCmd = conn.Insert("reactions", new[] {
("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer)
});
foreach (var message in messages) {
object messageId = message.Id;
foreach (var message in messages) {
object messageId = message.Id;
messageCmd.Set(":message_id", messageId);
messageCmd.Set(":sender_id", message.Sender);
messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.ExecuteNonQuery();
messageCmd.Set(":message_id", messageId);
messageCmd.Set(":sender_id", message.Sender);
messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.ExecuteNonQuery();
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery();
}
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery();
}
if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":url", attachment.Url);
attachmentCmd.Set(":size", attachment.Size);
attachmentCmd.ExecuteNonQuery();
if (!message.Attachments.IsEmpty) {
addedAttachments = true;
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":url", attachment.Url);
attachmentCmd.Set(":size", attachment.Size);
attachmentCmd.ExecuteNonQuery();
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
}
tx.Commit();
}
tx.Commit();
messageStatisticsThread.RequestUpdate();
totalMessagesComputer.Recompute();
if (addedAttachments) {
totalAttachmentsComputer.Recompute();
}
}
public int CountMessages(MessageFilter? filter = null) {
@@ -353,26 +369,125 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
return list;
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == MessageFilterRemovalMode.KeepMatching);
if (string.IsNullOrEmpty(whereClause)) {
return;
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
if (!string.IsNullOrEmpty(whereClause)) {
var perf = log.Start();
DeleteFromTable("messages", whereClause);
totalMessagesComputer.Recompute();
perf.End();
}
}
public int CountAttachments(AttachmentFilter? filter = null) {
using var conn = pool.Take();
using var cmd = conn.Command("SELECT COUNT(*) FROM attachments a" + filter.GenerateWhereClause("a"));
using var reader = cmd.ExecuteReader();
return reader.Read() ? reader.GetInt32(0) : 0;
}
public void AddDownloads(IEnumerable<Data.Download> downloads) {
using var conn = pool.Take();
using var tx = conn.BeginTransaction();
using var cmd = conn.Upsert("downloads", new[] {
("url", SqliteType.Text),
("status", SqliteType.Integer),
("size", SqliteType.Integer),
("blob", SqliteType.Blob)
});
foreach (var download in downloads) {
cmd.Set(":url", download.Url);
cmd.Set(":status", (int) download.Status);
cmd.Set(":size", download.Size);
cmd.Set(":blob", download.Data);
cmd.ExecuteNonQuery();
}
var perf = log.Start();
tx.Commit();
}
// Rider is being stupid...
StringBuilder build = new StringBuilder()
.Append("DELETE ")
.Append("FROM messages")
.Append(whereClause);
public void EnqueueDownloadItems(AttachmentFilter? filter = null) {
using var conn = pool.Take();
using var cmd = conn.Command("INSERT INTO downloads (url, status, size) SELECT a.url, :enqueued, a.size FROM attachments a" + filter.GenerateWhereClause("a"));
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.ExecuteNonQuery();
}
public List<DownloadItem> GetEnqueuedDownloadItems(int count) {
var list = new List<DownloadItem>();
using var conn = pool.Take();
using var cmd = conn.Command("SELECT url, size FROM downloads WHERE status = :enqueued LIMIT :limit");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":limit", SqliteType.Integer, Math.Max(0, count));
using var reader = cmd.ExecuteReader();
while (reader.Read()) {
list.Add(new DownloadItem {
Url = reader.GetString(0),
Size = reader.GetUint64(1)
});
}
return list;
}
public void RemoveDownloadItems(DownloadItemFilter? filter, FilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
if (!string.IsNullOrEmpty(whereClause)) {
DeleteFromTable("downloads", whereClause);
}
}
public DownloadStatusStatistics GetDownloadStatusStatistics() {
static void LoadUndownloadedStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
using var cmd = conn.Command("SELECT IFNULL(COUNT(filtered.size), 0), IFNULL(SUM(filtered.size), 0) FROM (SELECT DISTINCT a.url, a.size FROM attachments a WHERE a.url NOT IN (SELECT d.url FROM downloads d)) filtered");
using var reader = cmd.ExecuteReader();
if (reader.Read()) {
result.SkippedCount = reader.GetInt32(0);
result.SkippedSize = reader.GetUint64(1);
}
}
static void LoadSuccessStatistics(ISqliteConnection conn, DownloadStatusStatistics result) {
using var cmd = conn.Command(@"SELECT
IFNULL(SUM(CASE WHEN status = :enqueued THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :enqueued THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status = :success THEN size ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN 1 ELSE 0 END), 0),
IFNULL(SUM(CASE WHEN status != :enqueued AND status != :success THEN size ELSE 0 END), 0)
FROM downloads");
cmd.AddAndSet(":enqueued", SqliteType.Integer, (int) DownloadStatus.Enqueued);
cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
using var reader = cmd.ExecuteReader();
if (reader.Read()) {
result.EnqueuedCount = reader.GetInt32(0);
result.EnqueuedSize = reader.GetUint64(1);
result.SuccessfulCount = reader.GetInt32(2);
result.SuccessfulSize = reader.GetUint64(3);
result.FailedCount = reader.GetInt32(4);
result.FailedSize = reader.GetUint64(5);
}
}
var result = new DownloadStatusStatistics();
using var conn = pool.Take();
using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery();
UpdateMessageStatistics(conn);
perf.End();
LoadUndownloadedStatistics(conn, result);
LoadSuccessStatistics(conn, result);
return result;
}
private MultiDictionary<ulong, Attachment> GetAllAttachments() {
@@ -436,6 +551,19 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
return dict;
}
private void DeleteFromTable(string table, string whereClause) {
// Rider is being stupid...
StringBuilder build = new StringBuilder()
.Append("DELETE ")
.Append("FROM ")
.Append(table)
.Append(whereClause);
using var conn = pool.Take();
using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery();
}
public void Vacuum() {
using var conn = pool.Take();
using var cmd = conn.Command("VACUUM");
@@ -454,8 +582,22 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0;
}
private void UpdateMessageStatistics(ISqliteConnection conn) {
Statistics.TotalMessages = conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
private long ComputeMessageStatistics() {
using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
}
private void UpdateMessageStatistics(long totalMessages) {
Statistics.TotalMessages = totalMessages;
}
private long ComputeAttachmentStatistics() {
using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM attachments") as long? ?? 0L;
}
private void UpdateAttachmentStatistics(long totalAttachments) {
Statistics.TotalAttachments = totalAttachments;
}
}
}

View File

@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Linq;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite {
static class SqliteFilters {
public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.StartDate != null) {
where.AddCondition("timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
}
if (filter.EndDate != null) {
where.AddCondition("timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
}
if (filter.ChannelIds != null) {
where.AddCondition("channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
}
if (filter.UserIds != null) {
where.AddCondition("sender_id IN (" + string.Join(",", filter.UserIds) + ")");
}
if (filter.MessageIds != null) {
where.AddCondition("message_id IN (" + string.Join(",", filter.MessageIds) + ")");
}
return where.Generate();
}
public static string GenerateWhereClause(this AttachmentFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.MaxBytes != null) {
where.AddCondition("size <= " + filter.MaxBytes);
}
if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyNotPresent) {
where.AddCondition("url NOT IN (SELECT url FROM downloads)");
}
else if (filter.DownloadItemRule == AttachmentFilter.DownloadItemRules.OnlyPresent) {
where.AddCondition("url IN (SELECT url FROM downloads)");
}
return where.Generate();
}
public static string GenerateWhereClause(this DownloadItemFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
var where = new SqliteWhereGenerator(tableAlias, invert);
if (filter.IncludeStatuses != null) {
where.AddCondition("status IN (" + filter.IncludeStatuses.In() + ")");
}
if (filter.ExcludeStatuses != null) {
where.AddCondition("status NOT IN (" + filter.ExcludeStatuses.In() + ")");
}
return where.Generate();
}
private static string In(this ISet<DownloadStatus> statuses) {
return string.Join(",", statuses.Select(static status => (int) status));
}
}
}

View File

@@ -1,50 +0,0 @@
using System;
using System.Collections.Generic;
using DHT.Server.Data.Filters;
namespace DHT.Server.Database.Sqlite {
static class SqliteMessageFilter {
public static string GenerateWhereClause(this MessageFilter? filter, string? tableAlias = null, bool invert = false) {
if (filter == null) {
return "";
}
if (tableAlias != null) {
tableAlias += ".";
}
List<string> conditions = new();
if (filter.StartDate != null) {
conditions.Add(tableAlias + "timestamp >= " + new DateTimeOffset(filter.StartDate.Value).ToUnixTimeMilliseconds());
}
if (filter.EndDate != null) {
conditions.Add(tableAlias + "timestamp <= " + new DateTimeOffset(filter.EndDate.Value).ToUnixTimeMilliseconds());
}
if (filter.ChannelIds != null) {
conditions.Add(tableAlias + "channel_id IN (" + string.Join(",", filter.ChannelIds) + ")");
}
if (filter.UserIds != null) {
conditions.Add(tableAlias + "sender_id IN (" + string.Join(",", filter.UserIds) + ")");
}
if (filter.MessageIds != null) {
conditions.Add(tableAlias + "message_id IN (" + string.Join(",", filter.MessageIds) + ")");
}
if (conditions.Count == 0) {
return "";
}
if (invert) {
return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
}
else {
return " WHERE " + string.Join(" AND ", conditions);
}
}
}
}

View File

@@ -1,54 +0,0 @@
using System;
using System.Threading;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite {
sealed class SqliteMessageStatisticsThread : IDisposable {
private readonly SqliteConnectionPool pool;
private readonly Action<ISqliteConnection> action;
private readonly CancellationTokenSource cancellationTokenSource = new();
private readonly CancellationToken cancellationToken;
private readonly AutoResetEvent requestEvent = new (false);
public SqliteMessageStatisticsThread(SqliteConnectionPool pool, Action<ISqliteConnection> action) {
this.pool = pool;
this.action = action;
this.cancellationToken = cancellationTokenSource.Token;
var thread = new Thread(RunThread) {
Name = "DHT message statistics thread",
IsBackground = true
};
thread.Start();
}
public void Dispose() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
}
public void RequestUpdate() {
try {
requestEvent.Set();
} catch (ObjectDisposedException) {}
}
private void RunThread() {
try {
while (!cancellationToken.IsCancellationRequested) {
if (requestEvent.WaitOne(TimeSpan.FromMilliseconds(100))) {
using var conn = pool.Take();
action(conn);
}
}
} finally {
cancellationTokenSource.Dispose();
requestEvent.Dispose();
}
}
}
}

View File

@@ -13,6 +13,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
}
private readonly object monitor = new ();
private readonly Random rand = new ();
private volatile bool isDisposed;
private readonly BlockingCollection<PooledConnection> free = new (new ConcurrentStack<PooledConnection>());
@@ -49,7 +50,7 @@ namespace DHT.Server.Database.Sqlite.Utils {
while (conn == null) {
ThrowIfDisposed();
lock (monitor) {
if (free.TryTake(out conn, TimeSpan.FromMilliseconds(100))) {
if (free.TryTake(out conn, TimeSpan.FromMilliseconds(rand.Next(100, 200)))) {
used.Add(conn);
break;
}

View File

@@ -56,6 +56,10 @@ namespace DHT.Server.Database.Sqlite.Utils {
}
}
public static void AddAndSet(this SqliteCommand cmd, string key, SqliteType type, object? value) {
cmd.Parameters.Add(key, type).Value = value ?? DBNull.Value;
}
public static void Set(this SqliteCommand cmd, string key, object? value) {
cmd.Parameters[key].Value = value ?? DBNull.Value;
}

View File

@@ -0,0 +1,31 @@
using System.Collections.Generic;
namespace DHT.Server.Database.Sqlite.Utils {
sealed class SqliteWhereGenerator {
private readonly string? tableAlias;
private readonly bool invert;
private readonly List<string> conditions = new ();
public SqliteWhereGenerator(string? tableAlias, bool invert) {
this.tableAlias = tableAlias;
this.invert = invert;
}
public void AddCondition(string condition) {
conditions.Add(tableAlias == null ? condition : tableAlias + '.' + condition);
}
public string Generate() {
if (conditions.Count == 0) {
return "";
}
if (invert) {
return " WHERE NOT (" + string.Join(" AND ", conditions) + ")";
}
else {
return " WHERE " + string.Join(" AND ", conditions);
}
}
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Concurrent;
using System.Net;
using System.Threading;
using DHT.Server.Database;
using DHT.Utils.Logging;
using DHT.Utils.Models;
namespace DHT.Server.Download {
public sealed class BackgroundDownloadThread : BaseModel {
private static readonly Log Log = Log.ForType<BackgroundDownloadThread>();
public event EventHandler<DownloadItem>? OnItemFinished {
add => parameters.OnItemFinished += value;
remove => parameters.OnItemFinished -= value;
}
public event EventHandler? OnServerStopped {
add => parameters.OnServerStopped += value;
remove => parameters.OnServerStopped -= value;
}
private readonly CancellationTokenSource cancellationTokenSource;
private readonly ThreadInstance.Parameters parameters;
public BackgroundDownloadThread(IDatabaseFile db) {
this.cancellationTokenSource = new CancellationTokenSource();
this.parameters = new ThreadInstance.Parameters(db, cancellationTokenSource);
var thread = new Thread(new ThreadInstance().Work) {
Name = "DHT download thread"
};
thread.Start(parameters);
}
public void StopThread() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {
Log.Warn("Attempted to stop background download thread after the cancellation token has been disposed.");
}
}
private sealed class ThreadInstance {
private const int QueueSize = 32;
public sealed class Parameters {
public event EventHandler<DownloadItem>? OnItemFinished;
public event EventHandler? OnServerStopped;
public IDatabaseFile Db { get; }
public CancellationTokenSource CancellationTokenSource { get; }
public Parameters(IDatabaseFile db, CancellationTokenSource cancellationTokenSource) {
Db = db;
CancellationTokenSource = cancellationTokenSource;
}
public void FireOnItemFinished(DownloadItem item) {
OnItemFinished?.Invoke(null, item);
}
public void FireOnServerStopped() {
OnServerStopped?.Invoke(null, EventArgs.Empty);
}
}
private readonly WebClient client = new ();
public ThreadInstance() {
client.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36";
}
public void Work(object? obj) {
var parameters = (Parameters) obj!;
var cancellationTokenSource = parameters.CancellationTokenSource;
var cancellationToken = cancellationTokenSource.Token;
var db = parameters.Db;
var queue = new ConcurrentQueue<DownloadItem>();
cancellationToken.Register(client.CancelAsync);
try {
while (!cancellationToken.IsCancellationRequested) {
FillQueue(db, queue, cancellationToken);
while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var item)) {
var url = item.Url;
Log.Debug("Downloading " + url + "...");
try {
db.AddDownloads(new [] { Data.Download.NewSuccess(url, client.DownloadData(url)) });
} catch (WebException e) {
db.AddDownloads(new [] { Data.Download.NewFailure(url, e.Response is HttpWebResponse response ? response.StatusCode : null, item.Size) });
Log.Error(e);
} finally {
parameters.FireOnItemFinished(item);
}
}
}
} catch (OperationCanceledException) {
//
} catch (ObjectDisposedException) {
//
} finally {
cancellationTokenSource.Dispose();
parameters.FireOnServerStopped();
}
}
private static void FillQueue(IDatabaseFile db, ConcurrentQueue<DownloadItem> queue, CancellationToken cancellationToken) {
while (!cancellationToken.IsCancellationRequested && queue.IsEmpty) {
var newItems = db.GetEnqueuedDownloadItems(QueueSize);
if (newItems.Count == 0) {
Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
else {
foreach (var item in newItems) {
queue.Enqueue(item);
}
}
}
}
}
}
}

View File

@@ -0,0 +1,6 @@
namespace DHT.Server.Download {
public readonly struct DownloadItem {
public string Url { get; init; }
public ulong Size { get; init; }
}
}

View File

@@ -20,7 +20,7 @@
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.2" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />

View File

@@ -0,0 +1,126 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
namespace DHT.Utils.Tasks {
public sealed class AsyncValueComputer<TValue> {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private readonly bool processOutdatedResults;
private readonly object stateLock = new ();
private SoftHardCancellationToken? currentCancellationTokenSource;
private bool wasHardCancelled = false;
private Func<TValue>? currentComputeFunction;
private bool hasComputeFunctionChanged = false;
private AsyncValueComputer(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler, bool processOutdatedResults) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
this.processOutdatedResults = processOutdatedResults;
}
public void Cancel() {
lock (stateLock) {
wasHardCancelled = true;
currentCancellationTokenSource?.RequestHardCancellation();
}
}
public void Compute(Func<TValue> func) {
lock (stateLock) {
wasHardCancelled = false;
if (currentComputeFunction != null) {
currentComputeFunction = func;
hasComputeFunctionChanged = true;
currentCancellationTokenSource?.RequestSoftCancellation();
}
else {
EnqueueComputation(func);
}
}
}
[SuppressMessage("ReSharper", "MethodSupportsCancellation")]
private void EnqueueComputation(Func<TValue> func) {
var cancellationTokenSource = new SoftHardCancellationToken();
currentCancellationTokenSource?.RequestSoftCancellation();
currentCancellationTokenSource = cancellationTokenSource;
currentComputeFunction = func;
hasComputeFunctionChanged = false;
var task = Task.Run(func);
task.ContinueWith(t => {
if (!cancellationTokenSource.IsCancelled(processOutdatedResults)) {
resultProcessor(t.Result);
}
}, CancellationToken.None, TaskContinuationOptions.NotOnFaulted, resultTaskScheduler);
task.ContinueWith(_ => {
lock (stateLock) {
cancellationTokenSource.Dispose();
if (currentCancellationTokenSource == cancellationTokenSource) {
currentCancellationTokenSource = null;
}
if (hasComputeFunctionChanged && !wasHardCancelled) {
EnqueueComputation(currentComputeFunction);
}
else {
currentComputeFunction = null;
hasComputeFunctionChanged = false;
}
}
});
}
public sealed class Single {
private readonly AsyncValueComputer<TValue> baseComputer;
private readonly Func<TValue> resultComputer;
internal Single(AsyncValueComputer<TValue> baseComputer, Func<TValue> resultComputer) {
this.baseComputer = baseComputer;
this.resultComputer = resultComputer;
}
public void Recompute() {
baseComputer.Compute(resultComputer);
}
}
public static Builder WithResultProcessor(Action<TValue> resultProcessor, TaskScheduler? scheduler = null) {
return new Builder(resultProcessor, scheduler ?? TaskScheduler.FromCurrentSynchronizationContext());
}
public sealed class Builder {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private bool processOutdatedResults;
internal Builder(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
}
public Builder WithOutdatedResults() {
this.processOutdatedResults = true;
return this;
}
public AsyncValueComputer<TValue> Build() {
return new AsyncValueComputer<TValue>(resultProcessor, resultTaskScheduler, processOutdatedResults);
}
public Single BuildWithComputer(Func<TValue> resultComputer) {
return new Single(Build(), resultComputer);
}
}
}
}

View File

@@ -0,0 +1,39 @@
using System;
using System.Threading;
namespace DHT.Utils.Tasks {
/// <summary>
/// Manages a pair of cancellation tokens that follow these rules:
/// <list type="number">
/// <item><description>If the soft token is cancelled, the hard token remains uncancelled.</description></item>
/// <item><description>If the hard token is cancelled, the soft token is also cancelled.</description></item>
/// </list>
/// </summary>
sealed class SoftHardCancellationToken : IDisposable {
private readonly CancellationTokenSource soft;
private readonly CancellationTokenSource hard;
public SoftHardCancellationToken() {
this.soft = new CancellationTokenSource();
this.hard = new CancellationTokenSource();
}
public bool IsCancelled(bool onlyHardCancellation) {
return (onlyHardCancellation ? hard : soft).IsCancellationRequested;
}
public void RequestSoftCancellation() {
soft.Cancel();
}
public void RequestHardCancellation() {
soft.Cancel();
hard.Cancel();
}
public void Dispose() {
soft.Dispose();
hard.Dispose();
}
}
}

View File

@@ -7,6 +7,6 @@ using DHT.Utils;
namespace DHT.Utils {
static class Version {
public const string Tag = "35.3.0.0";
public const string Tag = "36.0.0.0";
}
}

Binary file not shown.

View File

@@ -53,11 +53,11 @@ define('LATEST_VERSION', $version === false ? '_' : $version);
<svg class="icon">
<use href="#icon-globe" />
</svg>
<span class="platform">Portable</span>
<span class="platform">Other</span>
</a>
</div>
<p>To launch the three OS-specific versions, extract the <strong>DiscordHistoryTracker</strong> executable, and double-click it.</p>
<p>To launch the <strong>Portable</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0/runtime" rel="nofollow noopener">.NET 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p>
<p>To launch the <strong>Other</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0" rel="nofollow noopener">ASP.NET Core Runtime 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p>
<h3>How to Track Messages</h3>
<p>The app saves messages into a database file stored on your computer. When you open the app, you are given the option to create a new database file, or open an existing one.</p>