1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-09-15 19:32:09 +02:00

12 Commits

20 changed files with 424 additions and 307 deletions

View File

@@ -8,7 +8,7 @@
<entry key="Desktop/Dialogs/Message/MessageDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Dialogs/Progress/ProgressDialog.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/AboutWindow.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/FilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/MessageFilterPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/ServerConfigurationPanel.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/Controls/StatusBar.axaml" value="Desktop/Desktop.csproj" />
<entry key="Desktop/Main/MainWindow.axaml" value="Desktop/Desktop.csproj" />

View File

@@ -34,8 +34,8 @@
<StackPanel Margin="20">
<ScrollViewer MaxHeight="400">
<ItemsControl Items="{Binding Items}">
<ItemsControl.ItemTemplate>
<ItemsRepeater Items="{Binding Items}">
<ItemsRepeater.ItemTemplate>
<DataTemplate>
<CheckBox IsChecked="{Binding Checked}">
<Label>
@@ -43,8 +43,8 @@
</Label>
</CheckBox>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
</ItemsRepeater.ItemTemplate>
</ItemsRepeater>
</ScrollViewer>
<Panel Classes="buttons">
<WrapPanel Classes="left">

View File

@@ -1,60 +0,0 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.FilterPanel">
<Design.DataContext>
<controls:FilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="Grid > CalendarDatePicker">
<Setter Property="CornerRadius" Value="0" />
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="HorizontalContentAlignment" Value="Center" />
<Setter Property="VerticalAlignment" Value="Center" />
<Setter Property="VerticalContentAlignment" Value="Center" />
<Setter Property="IsTodayHighlighted" Value="True" />
<Setter Property="SelectedDateFormat" Value="Short" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
<Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
<Label Grid.Row="0" Grid.Column="0">From:</Label>
<CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
<Label Grid.Row="2" Grid.Column="0">To:</Label>
<CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
</Grid>
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
<Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
<TextBlock Text="{Binding ChannelFilterLabel}" />
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
<Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
<TextBlock Text="{Binding UserFilterLabel}" />
</StackPanel>
</WrapPanel>
</UserControl>

View File

@@ -0,0 +1,63 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d"
x:Class="DHT.Desktop.Main.Controls.MessageFilterPanel">
<Design.DataContext>
<controls:MessageFilterPanelModel />
</Design.DataContext>
<UserControl.Styles>
<Style Selector="WrapPanel > StackPanel">
<Setter Property="Margin" Value="0 20 40 0" />
<Setter Property="Spacing" Value="4" />
</Style>
<Style Selector="WrapPanel > StackPanel:nth-last-child(1)">
<Setter Property="Margin" Value="0 20 0 0" />
</Style>
<Style Selector="Grid > Label">
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style Selector="Grid > CalendarDatePicker">
<Setter Property="CornerRadius" Value="0" />
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="HorizontalContentAlignment" Value="Center" />
<Setter Property="VerticalAlignment" Value="Center" />
<Setter Property="VerticalContentAlignment" Value="Center" />
<Setter Property="IsTodayHighlighted" Value="True" />
<Setter Property="SelectedDateFormat" Value="Short" />
</Style>
<Style Selector="Button">
<Setter Property="Margin" Value="0 0 0 8" />
</Style>
</UserControl.Styles>
<StackPanel>
<TextBlock Text="{Binding FilterStatisticsText}" />
<WrapPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
<Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
<Label Grid.Row="0" Grid.Column="0">From:</Label>
<CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
<Label Grid.Row="2" Grid.Column="0">To:</Label>
<CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
</Grid>
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
<Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
<TextBlock Text="{Binding ChannelFilterLabel}" />
</StackPanel>
<StackPanel>
<CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
<Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
<TextBlock Text="{Binding UserFilterLabel}" />
</StackPanel>
</WrapPanel>
</StackPanel>
</UserControl>

View File

@@ -4,11 +4,11 @@ using Avalonia.Markup.Xaml;
namespace DHT.Desktop.Main.Controls {
[SuppressMessage("ReSharper", "MemberCanBeInternal")]
public sealed class FilterPanel : UserControl {
public sealed class MessageFilterPanel : UserControl {
private CalendarDatePicker StartDatePicker => this.FindControl<CalendarDatePicker>("StartDatePicker");
private CalendarDatePicker EndDatePicker => this.FindControl<CalendarDatePicker>("EndDatePicker");
public FilterPanel() {
public MessageFilterPanel() {
InitializeComponent();
}
@@ -25,7 +25,7 @@ namespace DHT.Desktop.Main.Controls {
}
public void CalendarDatePicker_OnSelectedDateChanged(object? sender, SelectionChangedEventArgs e) {
if (DataContext is FilterPanelModel model) {
if (DataContext is MessageFilterPanelModel model) {
model.StartDate = StartDatePicker.SelectedDate;
model.EndDate = EndDatePicker.SelectedDate;
}

View File

@@ -12,9 +12,10 @@ using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database;
using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls {
sealed class FilterPanelModel : BaseModel, IDisposable {
sealed class MessageFilterPanelModel : BaseModel, IDisposable {
private static readonly HashSet<string> FilterProperties = new () {
nameof(FilterByDate),
nameof(StartDate),
@@ -25,6 +26,8 @@ namespace DHT.Desktop.Main.Controls {
nameof(IncludedUsers)
};
public string FilterStatisticsText { get; private set; } = "";
public event PropertyChangedEventHandler? FilterPropertyChanged;
public bool HasAnyFilters => FilterByDate || FilterByChannel || FilterByUser;
@@ -89,13 +92,20 @@ namespace DHT.Desktop.Main.Controls {
private readonly Window window;
private readonly IDatabaseFile db;
[Obsolete("Designer")]
public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
private readonly AsyncValueComputer<long> exportedMessageCountComputer;
private long? exportedMessageCount;
private long? totalMessageCount;
public FilterPanelModel(Window window, IDatabaseFile db) {
[Obsolete("Designer")]
public MessageFilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
public MessageFilterPanelModel(Window window, IDatabaseFile db) {
this.window = window;
this.db = db;
this.exportedMessageCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetExportedMessageCount).Build();
UpdateFilterStatisticsText();
UpdateChannelFilterLabel();
UpdateUserFilterLabel();
@@ -109,6 +119,7 @@ namespace DHT.Desktop.Main.Controls {
private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
UpdateFilterStatistics();
FilterPropertyChanged?.Invoke(sender, e);
}
@@ -121,7 +132,11 @@ namespace DHT.Desktop.Main.Controls {
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
totalMessageCount = db.Statistics.TotalMessages;
UpdateFilterStatistics();
}
else if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
UpdateChannelFilterLabel();
}
else if (e.PropertyName == nameof(DatabaseStatistics.TotalUsers)) {
@@ -129,6 +144,32 @@ namespace DHT.Desktop.Main.Controls {
}
}
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
exportedMessageCount = totalMessageCount;
UpdateFilterStatisticsText();
}
else {
exportedMessageCount = null;
UpdateFilterStatisticsText();
exportedMessageCountComputer.Compute(_ => db.CountMessages(filter));
}
}
private void SetExportedMessageCount(long exportedMessageCount) {
this.exportedMessageCount = exportedMessageCount;
UpdateFilterStatisticsText();
}
private void UpdateFilterStatisticsText() {
var exportedMessageCountStr = exportedMessageCount?.Format() ?? "(...)";
var totalMessageCountStr = totalMessageCount?.Format() ?? "(...)";
FilterStatisticsText = "Will export " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or > 0 ? "s." : ".");
OnPropertyChanged(nameof(FilterStatisticsText));
}
public async void OpenChannelFilterDialog() {
var servers = db.GetAllServers().ToDictionary(static server => server.Id);
var items = new List<CheckBoxItem<ulong>>();

View File

@@ -24,7 +24,7 @@
<StackPanel>
<Button Command="{Binding OnClickToggleServerButton}" Content="{Binding ToggleServerButtonText}" IsEnabled="{Binding IsToggleServerButtonEnabled}" />
<TextBlock TextWrapping="Wrap" Margin="0 15">
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy and apply the tracking script again.
The following settings determine how the tracking script communicates with this application. If you change them, you will have to copy/paste the tracking script again.
</TextBlock>
<WrapPanel>
<StackPanel>

View File

@@ -1,8 +1,10 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Avalonia;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
using DHT.Desktop.Main.Pages;
using JetBrains.Annotations;
namespace DHT.Desktop.Main {
@@ -30,6 +32,14 @@ namespace DHT.Desktop.Main {
if (DataContext is IDisposable disposable) {
disposable.Dispose();
}
foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {
try {
File.Delete(temporaryFile);
} catch (Exception) {
// ignored
}
}
}
}
}

View File

@@ -5,9 +5,7 @@
xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="DHT.Desktop.Main.Pages.ViewerPage"
AttachedToVisualTree="OnAttachedToVisualTree"
DetachedFromVisualTree="OnDetachedFromVisualTree">
x:Class="DHT.Desktop.Main.Pages.ViewerPage">
<Design.DataContext>
<pages:ViewerPageModel />
@@ -24,8 +22,7 @@
<Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
<Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
</StackPanel>
<TextBlock Text="{Binding ExportedMessageText}" Margin="0 20 0 0" />
<controls:FilterPanel DataContext="{Binding FilterModel}" />
<controls:MessageFilterPanel DataContext="{Binding FilterModel}" Margin="0 20 0 0" />
<Expander Header="Database Tools">
<StackPanel Orientation="Vertical" Spacing="10">
<StackPanel Orientation="Vertical" Spacing="4">

View File

@@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using Avalonia;
using Avalonia.Controls;
using Avalonia.Markup.Xaml;
@@ -13,17 +12,5 @@ namespace DHT.Desktop.Main.Pages {
private void InitializeComponent() {
AvaloniaXamlLoader.Load(this);
}
public void OnAttachedToVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(true);
}
}
public void OnDetachedFromVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(false);
}
}
}
}

View File

@@ -1,8 +1,10 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Avalonia.Controls;
@@ -17,8 +19,8 @@ using static DHT.Desktop.Program;
namespace DHT.Desktop.Main.Pages {
sealed class ViewerPageModel : BaseModel, IDisposable {
public string ExportedMessageText { get; private set; } = "";
public static readonly ConcurrentBag<string> TemporaryFiles = new ();
public bool DatabaseToolFilterModeKeep { get; set; } = true;
public bool DatabaseToolFilterModeRemove { get; set; } = false;
@@ -29,13 +31,11 @@ namespace DHT.Desktop.Main.Pages {
set => Change(ref hasFilters, value);
}
private FilterPanelModel FilterModel { get; }
private MessageFilterPanelModel FilterModel { get; }
private readonly Window window;
private readonly IDatabaseFile db;
private bool isPageVisible = false;
[Obsolete("Designer")]
public ViewerPageModel() : this(null!, DummyDatabaseFile.Instance) {}
@@ -43,51 +43,53 @@ namespace DHT.Desktop.Main.Pages {
this.window = window;
this.db = db;
FilterModel = new FilterPanelModel(window, db);
FilterModel = new MessageFilterPanelModel(window, db);
FilterModel.FilterPropertyChanged += OnFilterPropertyChanged;
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
}
public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose();
}
public void SetPageVisible(bool isPageVisible) {
this.isPageVisible = isPageVisible;
if (isPageVisible) {
UpdateStatistics();
}
}
private void OnFilterPropertyChanged(object? sender, PropertyChangedEventArgs e) {
UpdateStatistics();
HasFilters = FilterModel.HasAnyFilters;
}
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (isPageVisible && e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
UpdateStatistics();
private async Task WriteViewerFile(string path) {
const string ArchiveTag = "/*[ARCHIVE]*/";
string indexFile = await Resources.ReadTextAsync("Viewer/index.html");
string viewerTemplate = indexFile.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'));
int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag);
int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length;
string jsonTempFile = path + ".tmp";
await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) {
await ViewerJsonExport.Generate(jsonStream, db, FilterModel.CreateFilter());
char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)];
jsonStream.Position = 0;
await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) {
await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]);
using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) {
int readBytes;
while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) {
string jsonChunk = new string(jsonBuffer, 0, readBytes);
await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk));
}
}
await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]);
}
}
}
private void UpdateStatistics() {
var filter = FilterModel.CreateFilter();
var allMessagesCount = db.Statistics.TotalMessages?.Format() ?? "?";
var filteredMessagesCount = filter.IsEmpty ? allMessagesCount : db.CountMessages(filter).Format();
ExportedMessageText = "Will export " + filteredMessagesCount + " out of " + allMessagesCount + " message(s).";
OnPropertyChanged(nameof(ExportedMessageText));
}
private async Task<string> GenerateViewerContents() {
string json = ViewerJsonExport.Generate(db, FilterModel.CreateFilter());
string index = await Resources.ReadTextAsync("Viewer/index.html");
string viewer = index.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'))
.Replace("/*[ARCHIVE]*/", HttpUtility.JavaScriptStringEncode(json));
return viewer;
File.Delete(jsonTempFile);
}
public async void OnClickOpenViewer() {
@@ -101,8 +103,10 @@ namespace DHT.Desktop.Main.Pages {
fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html");
}
TemporaryFiles.Add(fullPath);
Directory.CreateDirectory(rootPath);
await File.WriteAllTextAsync(fullPath, await GenerateViewerContents());
await WriteViewerFile(fullPath);
Process.Start(new ProcessStartInfo(fullPath) { UseShellExecute = true });
}
@@ -110,7 +114,7 @@ namespace DHT.Desktop.Main.Pages {
public async void OnClickSaveViewer() {
var dialog = new SaveFileDialog {
Title = "Save Viewer",
InitialFileName = "archive.html",
InitialFileName = Path.GetFileNameWithoutExtension(db.Path) + ".html",
Directory = Path.GetDirectoryName(db.Path),
Filters = new List<FileDialogFilter> {
new() {
@@ -122,7 +126,7 @@ namespace DHT.Desktop.Main.Pages {
string? path = await dialog;
if (!string.IsNullOrEmpty(path)) {
await File.WriteAllTextAsync(path, await GenerateViewerContents());
await WriteViewerFile(path);
}
}
@@ -131,12 +135,12 @@ namespace DHT.Desktop.Main.Pages {
if (DatabaseToolFilterModeKeep) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Keep Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be kept, and the rest will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.KeepMatching);
db.RemoveMessages(filter, FilterRemovalMode.KeepMatching);
}
}
else if (DatabaseToolFilterModeRemove) {
if (DialogResult.YesNo.Yes == await Dialog.ShowYesNo(window, "Remove Matching Messages in This Database", db.CountMessages(filter).Pluralize("message") + " will be removed from this database. This action cannot be undone. Proceed?")) {
db.RemoveMessages(filter, MessageFilterRemovalMode.RemoveMatching);
db.RemoveMessages(filter, FilterRemovalMode.RemoveMatching);
}
}
}

View File

@@ -1,5 +1,5 @@
namespace DHT.Server.Data.Filters {
public enum MessageFilterRemovalMode {
public enum FilterRemovalMode {
KeepMatching,
RemoveMatching
}

View File

@@ -39,7 +39,7 @@ namespace DHT.Server.Database {
return new();
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {}
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {}
public void Vacuum() {}

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Utils.Logging;
@@ -9,7 +11,7 @@ namespace DHT.Server.Database.Export {
public static class ViewerJsonExport {
private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
public static string Generate(IDatabaseFile db, MessageFilter? filter = null) {
public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) {
var perf = Log.Start();
var includedUserIds = new HashSet<ulong>();
@@ -37,17 +39,20 @@ namespace DHT.Server.Database.Export {
perf.Step("Collect database data");
var value = new {
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
};
perf.Step("Generate value object");
var opts = new JsonSerializerOptions();
opts.Converters.Add(new ViewerJsonSnowflakeSerializer());
var json = JsonSerializer.Serialize(new {
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
}, opts);
await JsonSerializer.SerializeAsync(stream, value, opts);
perf.Step("Serialize to JSON");
perf.End();
return json;
}
private static object GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds, out List<string> userindex, out Dictionary<ulong, object> userIndices) {
@@ -159,8 +164,8 @@ namespace DHT.Server.Database.Export {
}
if (!message.Attachments.IsEmpty) {
obj["a"] = message.Attachments.Select(static attachment => new {
url = attachment.Url
obj["a"] = message.Attachments.Select(static attachment => new Dictionary<string, object> {
{ "url", attachment.Url }
}).ToArray();
}

View File

@@ -20,7 +20,7 @@ namespace DHT.Server.Database {
void AddMessages(Message[] messages);
int CountMessages(MessageFilter? filter = null);
List<Message> GetMessages(MessageFilter? filter = null);
void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode);
void RemoveMessages(MessageFilter filter, FilterRemovalMode mode);
void Vacuum();
}

View File

@@ -2,12 +2,14 @@ using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Collections;
using DHT.Utils.Logging;
using DHT.Utils.Tasks;
using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite {
@@ -36,12 +38,12 @@ namespace DHT.Server.Database.Sqlite {
private readonly Log log;
private readonly SqliteConnectionPool pool;
private readonly SqliteMessageStatisticsThread messageStatisticsThread;
private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
this.pool = pool;
this.messageStatisticsThread = new SqliteMessageStatisticsThread(pool, UpdateMessageStatistics);
this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
this.Path = path;
this.Statistics = new DatabaseStatistics();
@@ -52,11 +54,10 @@ namespace DHT.Server.Database.Sqlite {
UpdateUserStatistics(conn);
}
messageStatisticsThread.RequestUpdate();
totalMessagesComputer.Recompute();
}
public void Dispose() {
messageStatisticsThread.Dispose();
pool.Dispose();
}
@@ -193,119 +194,121 @@ namespace DHT.Server.Database.Sqlite {
cmd.ExecuteNonQuery();
}
using var conn = pool.Take();
using var tx = conn.BeginTransaction();
using (var conn = pool.Take()) {
using var tx = conn.BeginTransaction();
using var messageCmd = conn.Upsert("messages", new[] {
("message_id", SqliteType.Integer),
("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer),
("text", SqliteType.Text),
("timestamp", SqliteType.Integer)
});
using var messageCmd = conn.Upsert("messages", new[] {
("message_id", SqliteType.Integer),
("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer),
("text", SqliteType.Text),
("timestamp", SqliteType.Integer)
});
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
});
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer)
});
using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
});
using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer)
});
using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer),
("name", SqliteType.Text),
("type", SqliteType.Text),
("url", SqliteType.Text),
("size", SqliteType.Integer)
});
using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer),
("name", SqliteType.Text),
("type", SqliteType.Text),
("url", SqliteType.Text),
("size", SqliteType.Integer)
});
using var embedCmd = conn.Insert("embeds", new[] {
("message_id", SqliteType.Integer),
("json", SqliteType.Text)
});
using var embedCmd = conn.Insert("embeds", new[] {
("message_id", SqliteType.Integer),
("json", SqliteType.Text)
});
using var reactionCmd = conn.Insert("reactions", new[] {
("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer)
});
using var reactionCmd = conn.Insert("reactions", new[] {
("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer)
});
foreach (var message in messages) {
object messageId = message.Id;
foreach (var message in messages) {
object messageId = message.Id;
messageCmd.Set(":message_id", messageId);
messageCmd.Set(":sender_id", message.Sender);
messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.ExecuteNonQuery();
messageCmd.Set(":message_id", messageId);
messageCmd.Set(":sender_id", message.Sender);
messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.ExecuteNonQuery();
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery();
}
if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery();
}
if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery();
}
if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":url", attachment.Url);
attachmentCmd.Set(":size", attachment.Size);
attachmentCmd.ExecuteNonQuery();
if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":url", attachment.Url);
attachmentCmd.Set(":size", attachment.Size);
attachmentCmd.ExecuteNonQuery();
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
}
tx.Commit();
}
tx.Commit();
messageStatisticsThread.RequestUpdate();
totalMessagesComputer.Recompute();
}
public int CountMessages(MessageFilter? filter = null) {
@@ -353,8 +356,8 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
return list;
}
public void RemoveMessages(MessageFilter filter, MessageFilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == MessageFilterRemovalMode.KeepMatching);
public void RemoveMessages(MessageFilter filter, FilterRemovalMode mode) {
var whereClause = filter.GenerateWhereClause(invert: mode == FilterRemovalMode.KeepMatching);
if (string.IsNullOrEmpty(whereClause)) {
return;
}
@@ -367,11 +370,12 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
.Append("FROM messages")
.Append(whereClause);
using var conn = pool.Take();
using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery();
using (var conn = pool.Take()) {
using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery();
}
UpdateMessageStatistics(conn);
totalMessagesComputer.Recompute();
perf.End();
}
@@ -454,8 +458,13 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0;
}
private void UpdateMessageStatistics(ISqliteConnection conn) {
Statistics.TotalMessages = conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
private long ComputeMessageStatistics(CancellationToken token) {
using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
}
private void UpdateMessageStatistics(long totalMessages) {
Statistics.TotalMessages = totalMessages;
}
}
}

View File

@@ -1,54 +0,0 @@
using System;
using System.Threading;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite {
sealed class SqliteMessageStatisticsThread : IDisposable {
private readonly SqliteConnectionPool pool;
private readonly Action<ISqliteConnection> action;
private readonly CancellationTokenSource cancellationTokenSource = new();
private readonly CancellationToken cancellationToken;
private readonly AutoResetEvent requestEvent = new (false);
public SqliteMessageStatisticsThread(SqliteConnectionPool pool, Action<ISqliteConnection> action) {
this.pool = pool;
this.action = action;
this.cancellationToken = cancellationTokenSource.Token;
var thread = new Thread(RunThread) {
Name = "DHT message statistics thread",
IsBackground = true
};
thread.Start();
}
public void Dispose() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
}
public void RequestUpdate() {
try {
requestEvent.Set();
} catch (ObjectDisposedException) {}
}
private void RunThread() {
try {
while (!cancellationToken.IsCancellationRequested) {
if (requestEvent.WaitOne(TimeSpan.FromMilliseconds(100))) {
using var conn = pool.Take();
action(conn);
}
}
} finally {
cancellationTokenSource.Dispose();
requestEvent.Dispose();
}
}
}
}

View File

@@ -20,7 +20,7 @@
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.2" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.5" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />

View File

@@ -0,0 +1,115 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
namespace DHT.Utils.Tasks {
public sealed class AsyncValueComputer<TValue> {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private readonly bool processOutdatedResults;
private readonly object stateLock = new ();
private CancellationTokenSource? currentCancellationTokenSource;
private Func<CancellationToken, TValue>? currentComputeFunction;
private bool hasComputeFunctionChanged = false;
private AsyncValueComputer(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler, bool processOutdatedResults) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
this.processOutdatedResults = processOutdatedResults;
}
public void Compute(Func<CancellationToken, TValue> func) {
lock (stateLock) {
if (currentComputeFunction != null) {
currentComputeFunction = func;
hasComputeFunctionChanged = true;
currentCancellationTokenSource?.Cancel();
}
else {
EnqueueComputation(func);
}
}
}
[SuppressMessage("ReSharper", "MethodSupportsCancellation")]
private void EnqueueComputation(Func<CancellationToken, TValue> func) {
var cancellationTokenSource = new CancellationTokenSource();
var cancellationToken = cancellationTokenSource.Token;
currentCancellationTokenSource?.Cancel();
currentCancellationTokenSource = cancellationTokenSource;
currentComputeFunction = func;
hasComputeFunctionChanged = false;
var task = Task.Run(() => func(cancellationToken));
task.ContinueWith(t => {
if (processOutdatedResults || !cancellationToken.IsCancellationRequested) {
resultProcessor(t.Result);
}
}, CancellationToken.None, TaskContinuationOptions.NotOnFaulted, resultTaskScheduler);
task.ContinueWith(_ => {
lock (stateLock) {
cancellationTokenSource.Dispose();
if (currentCancellationTokenSource == cancellationTokenSource) {
currentCancellationTokenSource = null;
}
if (hasComputeFunctionChanged) {
EnqueueComputation(currentComputeFunction);
}
else {
currentComputeFunction = null;
}
}
});
}
public sealed class Single {
private readonly AsyncValueComputer<TValue> baseComputer;
private readonly Func<CancellationToken, TValue> resultComputer;
internal Single(AsyncValueComputer<TValue> baseComputer, Func<CancellationToken, TValue> resultComputer) {
this.baseComputer = baseComputer;
this.resultComputer = resultComputer;
}
public void Recompute() {
baseComputer.Compute(resultComputer);
}
}
public static Builder WithResultProcessor(Action<TValue> resultProcessor, TaskScheduler? scheduler = null) {
return new Builder(resultProcessor, scheduler ?? TaskScheduler.FromCurrentSynchronizationContext());
}
public sealed class Builder {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private bool processOutdatedResults;
internal Builder(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
}
public Builder WithOutdatedResults() {
this.processOutdatedResults = true;
return this;
}
public AsyncValueComputer<TValue> Build() {
return new AsyncValueComputer<TValue>(resultProcessor, resultTaskScheduler, processOutdatedResults);
}
public Single BuildWithComputer(Func<CancellationToken, TValue> resultComputer) {
return new Single(Build(), resultComputer);
}
}
}
}

View File

@@ -53,11 +53,11 @@ define('LATEST_VERSION', $version === false ? '_' : $version);
<svg class="icon">
<use href="#icon-globe" />
</svg>
<span class="platform">Portable</span>
<span class="platform">Other</span>
</a>
</div>
<p>To launch the three OS-specific versions, extract the <strong>DiscordHistoryTracker</strong> executable, and double-click it.</p>
<p>To launch the <strong>Portable</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0/runtime" rel="nofollow noopener">.NET 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p>
<p>To launch the <strong>Other</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0" rel="nofollow noopener">ASP.NET Core Runtime 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p>
<h3>How to Track Messages</h3>
<p>The app saves messages into a database file stored on your computer. When you open the app, you are given the option to create a new database file, or open an existing one.</p>