1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-10-17 17:42:51 +02:00

Compare commits

...

7 Commits

11 changed files with 362 additions and 245 deletions

View File

@ -35,26 +35,29 @@
</Style> </Style>
</UserControl.Styles> </UserControl.Styles>
<WrapPanel> <StackPanel>
<StackPanel> <TextBlock Text="{Binding FilterStatisticsText}" />
<CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox> <WrapPanel>
<Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0"> <StackPanel>
<Label Grid.Row="0" Grid.Column="0">From:</Label> <CheckBox IsChecked="{Binding FilterByDate}">Filter by Date</CheckBox>
<CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" /> <Grid ColumnDefinitions="Auto, 4, 125" RowDefinitions="Auto, 4, Auto" Margin="4 0">
<Label Grid.Row="2" Grid.Column="0">To:</Label> <Label Grid.Row="0" Grid.Column="0">From:</Label>
<CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" /> <CalendarDatePicker Grid.Row="0" Grid.Column="2" x:Name="StartDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
</Grid> <Label Grid.Row="2" Grid.Column="0">To:</Label>
</StackPanel> <CalendarDatePicker Grid.Row="2" Grid.Column="2" x:Name="EndDatePicker" IsEnabled="{Binding FilterByDate}" SelectedDateChanged="CalendarDatePicker_OnSelectedDateChanged" />
<StackPanel> </Grid>
<CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox> </StackPanel>
<Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button> <StackPanel>
<TextBlock Text="{Binding ChannelFilterLabel}" /> <CheckBox IsChecked="{Binding FilterByChannel}">Filter by Channel</CheckBox>
</StackPanel> <Button Command="{Binding OpenChannelFilterDialog}" IsEnabled="{Binding FilterByChannel}">Select Channels...</Button>
<StackPanel> <TextBlock Text="{Binding ChannelFilterLabel}" />
<CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox> </StackPanel>
<Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button> <StackPanel>
<TextBlock Text="{Binding UserFilterLabel}" /> <CheckBox IsChecked="{Binding FilterByUser}">Filter by User</CheckBox>
</StackPanel> <Button Command="{Binding OpenUserFilterDialog}" IsEnabled="{Binding FilterByUser}">Select Users...</Button>
</WrapPanel> <TextBlock Text="{Binding UserFilterLabel}" />
</StackPanel>
</WrapPanel>
</StackPanel>
</UserControl> </UserControl>

View File

@ -12,6 +12,7 @@ using DHT.Server.Data;
using DHT.Server.Data.Filters; using DHT.Server.Data.Filters;
using DHT.Server.Database; using DHT.Server.Database;
using DHT.Utils.Models; using DHT.Utils.Models;
using DHT.Utils.Tasks;
namespace DHT.Desktop.Main.Controls { namespace DHT.Desktop.Main.Controls {
sealed class FilterPanelModel : BaseModel, IDisposable { sealed class FilterPanelModel : BaseModel, IDisposable {
@ -25,6 +26,8 @@ namespace DHT.Desktop.Main.Controls {
nameof(IncludedUsers) nameof(IncludedUsers)
}; };
public string FilterStatisticsText { get; private set; } = "";
public event PropertyChangedEventHandler? FilterPropertyChanged; public event PropertyChangedEventHandler? FilterPropertyChanged;
public bool HasAnyFilters => FilterByDate || FilterByChannel || FilterByUser; public bool HasAnyFilters => FilterByDate || FilterByChannel || FilterByUser;
@ -89,6 +92,10 @@ namespace DHT.Desktop.Main.Controls {
private readonly Window window; private readonly Window window;
private readonly IDatabaseFile db; private readonly IDatabaseFile db;
private readonly AsyncValueComputer<long> exportedMessageCountComputer;
private long? exportedMessageCount;
private long? totalMessageCount;
[Obsolete("Designer")] [Obsolete("Designer")]
public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {} public FilterPanelModel() : this(null!, DummyDatabaseFile.Instance) {}
@ -96,6 +103,9 @@ namespace DHT.Desktop.Main.Controls {
this.window = window; this.window = window;
this.db = db; this.db = db;
this.exportedMessageCountComputer = AsyncValueComputer<long>.WithResultProcessor(SetExportedMessageCount).Build();
UpdateFilterStatisticsText();
UpdateChannelFilterLabel(); UpdateChannelFilterLabel();
UpdateUserFilterLabel(); UpdateUserFilterLabel();
@ -109,6 +119,7 @@ namespace DHT.Desktop.Main.Controls {
private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) { private void OnPropertyChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) { if (e.PropertyName != null && FilterProperties.Contains(e.PropertyName)) {
UpdateFilterStatistics();
FilterPropertyChanged?.Invoke(sender, e); FilterPropertyChanged?.Invoke(sender, e);
} }
@ -121,7 +132,11 @@ namespace DHT.Desktop.Main.Controls {
} }
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) { private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) {
if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) { if (e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) {
totalMessageCount = db.Statistics.TotalMessages;
UpdateFilterStatistics();
}
else if (e.PropertyName == nameof(DatabaseStatistics.TotalChannels)) {
UpdateChannelFilterLabel(); UpdateChannelFilterLabel();
} }
else if (e.PropertyName == nameof(DatabaseStatistics.TotalUsers)) { else if (e.PropertyName == nameof(DatabaseStatistics.TotalUsers)) {
@ -129,6 +144,32 @@ namespace DHT.Desktop.Main.Controls {
} }
} }
private void UpdateFilterStatistics() {
var filter = CreateFilter();
if (filter.IsEmpty) {
exportedMessageCount = totalMessageCount;
UpdateFilterStatisticsText();
}
else {
exportedMessageCount = null;
UpdateFilterStatisticsText();
exportedMessageCountComputer.Compute(_ => db.CountMessages(filter));
}
}
private void SetExportedMessageCount(long exportedMessageCount) {
this.exportedMessageCount = exportedMessageCount;
UpdateFilterStatisticsText();
}
private void UpdateFilterStatisticsText() {
var exportedMessageCountStr = exportedMessageCount?.Format() ?? "(...)";
var totalMessageCountStr = totalMessageCount?.Format() ?? "(...)";
FilterStatisticsText = "Will export " + exportedMessageCountStr + " out of " + totalMessageCountStr + " message" + (totalMessageCount is null or > 0 ? "s." : ".");
OnPropertyChanged(nameof(FilterStatisticsText));
}
public async void OpenChannelFilterDialog() { public async void OpenChannelFilterDialog() {
var servers = db.GetAllServers().ToDictionary(static server => server.Id); var servers = db.GetAllServers().ToDictionary(static server => server.Id);
var items = new List<CheckBoxItem<ulong>>(); var items = new List<CheckBoxItem<ulong>>();

View File

@ -1,8 +1,10 @@
using System; using System;
using System.Diagnostics.CodeAnalysis; using System.Diagnostics.CodeAnalysis;
using System.IO;
using Avalonia; using Avalonia;
using Avalonia.Controls; using Avalonia.Controls;
using Avalonia.Markup.Xaml; using Avalonia.Markup.Xaml;
using DHT.Desktop.Main.Pages;
using JetBrains.Annotations; using JetBrains.Annotations;
namespace DHT.Desktop.Main { namespace DHT.Desktop.Main {
@ -30,6 +32,14 @@ namespace DHT.Desktop.Main {
if (DataContext is IDisposable disposable) { if (DataContext is IDisposable disposable) {
disposable.Dispose(); disposable.Dispose();
} }
foreach (var temporaryFile in ViewerPageModel.TemporaryFiles) {
try {
File.Delete(temporaryFile);
} catch (Exception) {
// ignored
}
}
} }
} }
} }

View File

@ -5,9 +5,7 @@
xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages" xmlns:pages="clr-namespace:DHT.Desktop.Main.Pages"
xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls" xmlns:controls="clr-namespace:DHT.Desktop.Main.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450" mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="DHT.Desktop.Main.Pages.ViewerPage" x:Class="DHT.Desktop.Main.Pages.ViewerPage">
AttachedToVisualTree="OnAttachedToVisualTree"
DetachedFromVisualTree="OnDetachedFromVisualTree">
<Design.DataContext> <Design.DataContext>
<pages:ViewerPageModel /> <pages:ViewerPageModel />
@ -24,8 +22,7 @@
<Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button> <Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button>
<Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button> <Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button>
</StackPanel> </StackPanel>
<TextBlock Text="{Binding ExportedMessageText}" Margin="0 20 0 0" /> <controls:FilterPanel DataContext="{Binding FilterModel}" Margin="0 20 0 0" />
<controls:FilterPanel DataContext="{Binding FilterModel}" />
<Expander Header="Database Tools"> <Expander Header="Database Tools">
<StackPanel Orientation="Vertical" Spacing="10"> <StackPanel Orientation="Vertical" Spacing="10">
<StackPanel Orientation="Vertical" Spacing="4"> <StackPanel Orientation="Vertical" Spacing="4">

View File

@ -1,5 +1,4 @@
using System.Diagnostics.CodeAnalysis; using System.Diagnostics.CodeAnalysis;
using Avalonia;
using Avalonia.Controls; using Avalonia.Controls;
using Avalonia.Markup.Xaml; using Avalonia.Markup.Xaml;
@ -13,17 +12,5 @@ namespace DHT.Desktop.Main.Pages {
private void InitializeComponent() { private void InitializeComponent() {
AvaloniaXamlLoader.Load(this); AvaloniaXamlLoader.Load(this);
} }
public void OnAttachedToVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(true);
}
}
public void OnDetachedFromVisualTree(object? sender, VisualTreeAttachmentEventArgs e) {
if (DataContext is ViewerPageModel model) {
model.SetPageVisible(false);
}
}
} }
} }

View File

@ -1,8 +1,10 @@
using System; using System;
using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Web; using System.Web;
using Avalonia.Controls; using Avalonia.Controls;
@ -17,7 +19,7 @@ using static DHT.Desktop.Program;
namespace DHT.Desktop.Main.Pages { namespace DHT.Desktop.Main.Pages {
sealed class ViewerPageModel : BaseModel, IDisposable { sealed class ViewerPageModel : BaseModel, IDisposable {
public string ExportedMessageText { get; private set; } = ""; public static readonly ConcurrentBag<string> TemporaryFiles = new ();
public bool DatabaseToolFilterModeKeep { get; set; } = true; public bool DatabaseToolFilterModeKeep { get; set; } = true;
public bool DatabaseToolFilterModeRemove { get; set; } = false; public bool DatabaseToolFilterModeRemove { get; set; } = false;
@ -34,8 +36,6 @@ namespace DHT.Desktop.Main.Pages {
private readonly Window window; private readonly Window window;
private readonly IDatabaseFile db; private readonly IDatabaseFile db;
private bool isPageVisible = false;
[Obsolete("Designer")] [Obsolete("Designer")]
public ViewerPageModel() : this(null!, DummyDatabaseFile.Instance) {} public ViewerPageModel() : this(null!, DummyDatabaseFile.Instance) {}
@ -45,49 +45,51 @@ namespace DHT.Desktop.Main.Pages {
FilterModel = new FilterPanelModel(window, db); FilterModel = new FilterPanelModel(window, db);
FilterModel.FilterPropertyChanged += OnFilterPropertyChanged; FilterModel.FilterPropertyChanged += OnFilterPropertyChanged;
db.Statistics.PropertyChanged += OnDbStatisticsChanged;
} }
public void Dispose() { public void Dispose() {
db.Statistics.PropertyChanged -= OnDbStatisticsChanged;
FilterModel.Dispose(); FilterModel.Dispose();
} }
public void SetPageVisible(bool isPageVisible) {
this.isPageVisible = isPageVisible;
if (isPageVisible) {
UpdateStatistics();
}
}
private void OnFilterPropertyChanged(object? sender, PropertyChangedEventArgs e) { private void OnFilterPropertyChanged(object? sender, PropertyChangedEventArgs e) {
UpdateStatistics();
HasFilters = FilterModel.HasAnyFilters; HasFilters = FilterModel.HasAnyFilters;
} }
private void OnDbStatisticsChanged(object? sender, PropertyChangedEventArgs e) { private async Task WriteViewerFile(string path) {
if (isPageVisible && e.PropertyName == nameof(DatabaseStatistics.TotalMessages)) { const string ArchiveTag = "/*[ARCHIVE]*/";
UpdateStatistics();
string indexFile = await Resources.ReadTextAsync("Viewer/index.html");
string viewerTemplate = indexFile.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'));
int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag);
int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length;
string jsonTempFile = path + ".tmp";
await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) {
await ViewerJsonExport.Generate(jsonStream, db, FilterModel.CreateFilter());
char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)];
jsonStream.Position = 0;
await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) {
await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]);
using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) {
int readBytes;
while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) {
string jsonChunk = new string(jsonBuffer, 0, readBytes);
await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk));
}
}
await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]);
}
} }
}
private void UpdateStatistics() { File.Delete(jsonTempFile);
var filter = FilterModel.CreateFilter();
var allMessagesCount = db.Statistics.TotalMessages?.Format() ?? "?";
var filteredMessagesCount = filter.IsEmpty ? allMessagesCount : db.CountMessages(filter).Format();
ExportedMessageText = "Will export " + filteredMessagesCount + " out of " + allMessagesCount + " message(s).";
OnPropertyChanged(nameof(ExportedMessageText));
}
private async Task<string> GenerateViewerContents() {
string json = ViewerJsonExport.Generate(db, FilterModel.CreateFilter());
string index = await Resources.ReadTextAsync("Viewer/index.html");
string viewer = index.Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n'))
.Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n'))
.Replace("/*[ARCHIVE]*/", HttpUtility.JavaScriptStringEncode(json));
return viewer;
} }
public async void OnClickOpenViewer() { public async void OnClickOpenViewer() {
@ -101,8 +103,10 @@ namespace DHT.Desktop.Main.Pages {
fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html"); fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html");
} }
TemporaryFiles.Add(fullPath);
Directory.CreateDirectory(rootPath); Directory.CreateDirectory(rootPath);
await File.WriteAllTextAsync(fullPath, await GenerateViewerContents()); await WriteViewerFile(fullPath);
Process.Start(new ProcessStartInfo(fullPath) { UseShellExecute = true }); Process.Start(new ProcessStartInfo(fullPath) { UseShellExecute = true });
} }
@ -110,7 +114,7 @@ namespace DHT.Desktop.Main.Pages {
public async void OnClickSaveViewer() { public async void OnClickSaveViewer() {
var dialog = new SaveFileDialog { var dialog = new SaveFileDialog {
Title = "Save Viewer", Title = "Save Viewer",
InitialFileName = "archive.html", InitialFileName = Path.GetFileNameWithoutExtension(db.Path) + ".html",
Directory = Path.GetDirectoryName(db.Path), Directory = Path.GetDirectoryName(db.Path),
Filters = new List<FileDialogFilter> { Filters = new List<FileDialogFilter> {
new() { new() {
@ -122,7 +126,7 @@ namespace DHT.Desktop.Main.Pages {
string? path = await dialog; string? path = await dialog;
if (!string.IsNullOrEmpty(path)) { if (!string.IsNullOrEmpty(path)) {
await File.WriteAllTextAsync(path, await GenerateViewerContents()); await WriteViewerFile(path);
} }
} }

View File

@ -1,6 +1,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Linq; using System.Linq;
using System.Text.Json; using System.Text.Json;
using System.Threading.Tasks;
using DHT.Server.Data; using DHT.Server.Data;
using DHT.Server.Data.Filters; using DHT.Server.Data.Filters;
using DHT.Utils.Logging; using DHT.Utils.Logging;
@ -9,7 +11,7 @@ namespace DHT.Server.Database.Export {
public static class ViewerJsonExport { public static class ViewerJsonExport {
private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport)); private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
public static string Generate(IDatabaseFile db, MessageFilter? filter = null) { public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) {
var perf = Log.Start(); var perf = Log.Start();
var includedUserIds = new HashSet<ulong>(); var includedUserIds = new HashSet<ulong>();
@ -37,17 +39,20 @@ namespace DHT.Server.Database.Export {
perf.Step("Collect database data"); perf.Step("Collect database data");
var value = new {
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
};
perf.Step("Generate value object");
var opts = new JsonSerializerOptions(); var opts = new JsonSerializerOptions();
opts.Converters.Add(new ViewerJsonSnowflakeSerializer()); opts.Converters.Add(new ViewerJsonSnowflakeSerializer());
var json = JsonSerializer.Serialize(new { await JsonSerializer.SerializeAsync(stream, value, opts);
meta = new { users, userindex, servers, channels },
data = GenerateMessageList(includedMessages, userIndices)
}, opts);
perf.Step("Serialize to JSON"); perf.Step("Serialize to JSON");
perf.End(); perf.End();
return json;
} }
private static object GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds, out List<string> userindex, out Dictionary<ulong, object> userIndices) { private static object GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds, out List<string> userindex, out Dictionary<ulong, object> userIndices) {
@ -159,8 +164,8 @@ namespace DHT.Server.Database.Export {
} }
if (!message.Attachments.IsEmpty) { if (!message.Attachments.IsEmpty) {
obj["a"] = message.Attachments.Select(static attachment => new { obj["a"] = message.Attachments.Select(static attachment => new Dictionary<string, object> {
url = attachment.Url { "url", attachment.Url }
}).ToArray(); }).ToArray();
} }

View File

@ -2,12 +2,14 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Data; using DHT.Server.Data;
using DHT.Server.Data.Filters; using DHT.Server.Data.Filters;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Utils.Collections; using DHT.Utils.Collections;
using DHT.Utils.Logging; using DHT.Utils.Logging;
using DHT.Utils.Tasks;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite { namespace DHT.Server.Database.Sqlite {
@ -36,12 +38,12 @@ namespace DHT.Server.Database.Sqlite {
private readonly Log log; private readonly Log log;
private readonly SqliteConnectionPool pool; private readonly SqliteConnectionPool pool;
private readonly SqliteMessageStatisticsThread messageStatisticsThread; private readonly AsyncValueComputer<long>.Single totalMessagesComputer;
private SqliteDatabaseFile(string path, SqliteConnectionPool pool) { private SqliteDatabaseFile(string path, SqliteConnectionPool pool) {
this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path)); this.log = Log.ForType(typeof(SqliteDatabaseFile), System.IO.Path.GetFileName(path));
this.pool = pool; this.pool = pool;
this.messageStatisticsThread = new SqliteMessageStatisticsThread(pool, UpdateMessageStatistics); this.totalMessagesComputer = AsyncValueComputer<long>.WithResultProcessor(UpdateMessageStatistics).WithOutdatedResults().BuildWithComputer(ComputeMessageStatistics);
this.Path = path; this.Path = path;
this.Statistics = new DatabaseStatistics(); this.Statistics = new DatabaseStatistics();
@ -52,11 +54,10 @@ namespace DHT.Server.Database.Sqlite {
UpdateUserStatistics(conn); UpdateUserStatistics(conn);
} }
messageStatisticsThread.RequestUpdate(); totalMessagesComputer.Recompute();
} }
public void Dispose() { public void Dispose() {
messageStatisticsThread.Dispose();
pool.Dispose(); pool.Dispose();
} }
@ -193,119 +194,121 @@ namespace DHT.Server.Database.Sqlite {
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
} }
using var conn = pool.Take(); using (var conn = pool.Take()) {
using var tx = conn.BeginTransaction(); using var tx = conn.BeginTransaction();
using var messageCmd = conn.Upsert("messages", new[] { using var messageCmd = conn.Upsert("messages", new[] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("sender_id", SqliteType.Integer), ("sender_id", SqliteType.Integer),
("channel_id", SqliteType.Integer), ("channel_id", SqliteType.Integer),
("text", SqliteType.Text), ("text", SqliteType.Text),
("timestamp", SqliteType.Integer) ("timestamp", SqliteType.Integer)
}); });
using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps"); using var deleteEditTimestampCmd = DeleteByMessageId(conn, "edit_timestamps");
using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to"); using var deleteRepliedToCmd = DeleteByMessageId(conn, "replied_to");
using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments"); using var deleteAttachmentsCmd = DeleteByMessageId(conn, "attachments");
using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds"); using var deleteEmbedsCmd = DeleteByMessageId(conn, "embeds");
using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions"); using var deleteReactionsCmd = DeleteByMessageId(conn, "reactions");
using var editTimestampCmd = conn.Insert("edit_timestamps", new [] { using var editTimestampCmd = conn.Insert("edit_timestamps", new [] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("edit_timestamp", SqliteType.Integer) ("edit_timestamp", SqliteType.Integer)
}); });
using var repliedToCmd = conn.Insert("replied_to", new [] { using var repliedToCmd = conn.Insert("replied_to", new [] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("replied_to_id", SqliteType.Integer) ("replied_to_id", SqliteType.Integer)
}); });
using var attachmentCmd = conn.Insert("attachments", new[] { using var attachmentCmd = conn.Insert("attachments", new[] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("attachment_id", SqliteType.Integer), ("attachment_id", SqliteType.Integer),
("name", SqliteType.Text), ("name", SqliteType.Text),
("type", SqliteType.Text), ("type", SqliteType.Text),
("url", SqliteType.Text), ("url", SqliteType.Text),
("size", SqliteType.Integer) ("size", SqliteType.Integer)
}); });
using var embedCmd = conn.Insert("embeds", new[] { using var embedCmd = conn.Insert("embeds", new[] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("json", SqliteType.Text) ("json", SqliteType.Text)
}); });
using var reactionCmd = conn.Insert("reactions", new[] { using var reactionCmd = conn.Insert("reactions", new[] {
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
("emoji_id", SqliteType.Integer), ("emoji_id", SqliteType.Integer),
("emoji_name", SqliteType.Text), ("emoji_name", SqliteType.Text),
("emoji_flags", SqliteType.Integer), ("emoji_flags", SqliteType.Integer),
("count", SqliteType.Integer) ("count", SqliteType.Integer)
}); });
foreach (var message in messages) { foreach (var message in messages) {
object messageId = message.Id; object messageId = message.Id;
messageCmd.Set(":message_id", messageId); messageCmd.Set(":message_id", messageId);
messageCmd.Set(":sender_id", message.Sender); messageCmd.Set(":sender_id", message.Sender);
messageCmd.Set(":channel_id", message.Channel); messageCmd.Set(":channel_id", message.Channel);
messageCmd.Set(":text", message.Text); messageCmd.Set(":text", message.Text);
messageCmd.Set(":timestamp", message.Timestamp); messageCmd.Set(":timestamp", message.Timestamp);
messageCmd.ExecuteNonQuery(); messageCmd.ExecuteNonQuery();
ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId); ExecuteDeleteByMessageId(deleteEditTimestampCmd, messageId);
ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId); ExecuteDeleteByMessageId(deleteRepliedToCmd, messageId);
ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId); ExecuteDeleteByMessageId(deleteAttachmentsCmd, messageId);
ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId); ExecuteDeleteByMessageId(deleteEmbedsCmd, messageId);
ExecuteDeleteByMessageId(deleteReactionsCmd, messageId); ExecuteDeleteByMessageId(deleteReactionsCmd, messageId);
if (message.EditTimestamp is {} timestamp) { if (message.EditTimestamp is {} timestamp) {
editTimestampCmd.Set(":message_id", messageId); editTimestampCmd.Set(":message_id", messageId);
editTimestampCmd.Set(":edit_timestamp", timestamp); editTimestampCmd.Set(":edit_timestamp", timestamp);
editTimestampCmd.ExecuteNonQuery(); editTimestampCmd.ExecuteNonQuery();
} }
if (message.RepliedToId is {} repliedToId) { if (message.RepliedToId is {} repliedToId) {
repliedToCmd.Set(":message_id", messageId); repliedToCmd.Set(":message_id", messageId);
repliedToCmd.Set(":replied_to_id", repliedToId); repliedToCmd.Set(":replied_to_id", repliedToId);
repliedToCmd.ExecuteNonQuery(); repliedToCmd.ExecuteNonQuery();
} }
if (!message.Attachments.IsEmpty) { if (!message.Attachments.IsEmpty) {
foreach (var attachment in message.Attachments) { foreach (var attachment in message.Attachments) {
attachmentCmd.Set(":message_id", messageId); attachmentCmd.Set(":message_id", messageId);
attachmentCmd.Set(":attachment_id", attachment.Id); attachmentCmd.Set(":attachment_id", attachment.Id);
attachmentCmd.Set(":name", attachment.Name); attachmentCmd.Set(":name", attachment.Name);
attachmentCmd.Set(":type", attachment.Type); attachmentCmd.Set(":type", attachment.Type);
attachmentCmd.Set(":url", attachment.Url); attachmentCmd.Set(":url", attachment.Url);
attachmentCmd.Set(":size", attachment.Size); attachmentCmd.Set(":size", attachment.Size);
attachmentCmd.ExecuteNonQuery(); attachmentCmd.ExecuteNonQuery();
}
}
if (!message.Embeds.IsEmpty) {
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
} }
} }
if (!message.Embeds.IsEmpty) { tx.Commit();
foreach (var embed in message.Embeds) {
embedCmd.Set(":message_id", messageId);
embedCmd.Set(":json", embed.Json);
embedCmd.ExecuteNonQuery();
}
}
if (!message.Reactions.IsEmpty) {
foreach (var reaction in message.Reactions) {
reactionCmd.Set(":message_id", messageId);
reactionCmd.Set(":emoji_id", reaction.EmojiId);
reactionCmd.Set(":emoji_name", reaction.EmojiName);
reactionCmd.Set(":emoji_flags", (int) reaction.EmojiFlags);
reactionCmd.Set(":count", reaction.Count);
reactionCmd.ExecuteNonQuery();
}
}
} }
tx.Commit(); totalMessagesComputer.Recompute();
messageStatisticsThread.RequestUpdate();
} }
public int CountMessages(MessageFilter? filter = null) { public int CountMessages(MessageFilter? filter = null) {
@ -367,11 +370,12 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
.Append("FROM messages") .Append("FROM messages")
.Append(whereClause); .Append(whereClause);
using var conn = pool.Take(); using (var conn = pool.Take()) {
using var cmd = conn.Command(build.ToString()); using var cmd = conn.Command(build.ToString());
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
}
UpdateMessageStatistics(conn); totalMessagesComputer.Recompute();
perf.End(); perf.End();
} }
@ -454,8 +458,13 @@ LEFT JOIN replied_to rt ON m.message_id = rt.message_id" + filter.GenerateWhereC
Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0; Statistics.TotalUsers = conn.SelectScalar("SELECT COUNT(*) FROM users") as long? ?? 0;
} }
private void UpdateMessageStatistics(ISqliteConnection conn) { private long ComputeMessageStatistics(CancellationToken token) {
Statistics.TotalMessages = conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L; using var conn = pool.Take();
return conn.SelectScalar("SELECT COUNT(*) FROM messages") as long? ?? 0L;
}
private void UpdateMessageStatistics(long totalMessages) {
Statistics.TotalMessages = totalMessages;
} }
} }
} }

View File

@ -1,54 +0,0 @@
using System;
using System.Threading;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite {
sealed class SqliteMessageStatisticsThread : IDisposable {
private readonly SqliteConnectionPool pool;
private readonly Action<ISqliteConnection> action;
private readonly CancellationTokenSource cancellationTokenSource = new();
private readonly CancellationToken cancellationToken;
private readonly AutoResetEvent requestEvent = new (false);
public SqliteMessageStatisticsThread(SqliteConnectionPool pool, Action<ISqliteConnection> action) {
this.pool = pool;
this.action = action;
this.cancellationToken = cancellationTokenSource.Token;
var thread = new Thread(RunThread) {
Name = "DHT message statistics thread",
IsBackground = true
};
thread.Start();
}
public void Dispose() {
try {
cancellationTokenSource.Cancel();
} catch (ObjectDisposedException) {}
}
public void RequestUpdate() {
try {
requestEvent.Set();
} catch (ObjectDisposedException) {}
}
private void RunThread() {
try {
while (!cancellationToken.IsCancellationRequested) {
if (requestEvent.WaitOne(TimeSpan.FromMilliseconds(100))) {
using var conn = pool.Take();
action(conn);
}
}
} finally {
cancellationTokenSource.Dispose();
requestEvent.Dispose();
}
}
}
}

View File

@ -0,0 +1,115 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
namespace DHT.Utils.Tasks {
public sealed class AsyncValueComputer<TValue> {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private readonly bool processOutdatedResults;
private readonly object stateLock = new ();
private CancellationTokenSource? currentCancellationTokenSource;
private Func<CancellationToken, TValue>? currentComputeFunction;
private bool hasComputeFunctionChanged = false;
private AsyncValueComputer(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler, bool processOutdatedResults) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
this.processOutdatedResults = processOutdatedResults;
}
public void Compute(Func<CancellationToken, TValue> func) {
lock (stateLock) {
if (currentComputeFunction != null) {
currentComputeFunction = func;
hasComputeFunctionChanged = true;
currentCancellationTokenSource?.Cancel();
}
else {
EnqueueComputation(func);
}
}
}
[SuppressMessage("ReSharper", "MethodSupportsCancellation")]
private void EnqueueComputation(Func<CancellationToken, TValue> func) {
var cancellationTokenSource = new CancellationTokenSource();
var cancellationToken = cancellationTokenSource.Token;
currentCancellationTokenSource?.Cancel();
currentCancellationTokenSource = cancellationTokenSource;
currentComputeFunction = func;
hasComputeFunctionChanged = false;
var task = Task.Run(() => func(cancellationToken));
task.ContinueWith(t => {
if (processOutdatedResults || !cancellationToken.IsCancellationRequested) {
resultProcessor(t.Result);
}
}, CancellationToken.None, TaskContinuationOptions.NotOnFaulted, resultTaskScheduler);
task.ContinueWith(_ => {
lock (stateLock) {
cancellationTokenSource.Dispose();
if (currentCancellationTokenSource == cancellationTokenSource) {
currentCancellationTokenSource = null;
}
if (hasComputeFunctionChanged) {
EnqueueComputation(currentComputeFunction);
}
else {
currentComputeFunction = null;
}
}
});
}
public sealed class Single {
private readonly AsyncValueComputer<TValue> baseComputer;
private readonly Func<CancellationToken, TValue> resultComputer;
internal Single(AsyncValueComputer<TValue> baseComputer, Func<CancellationToken, TValue> resultComputer) {
this.baseComputer = baseComputer;
this.resultComputer = resultComputer;
}
public void Recompute() {
baseComputer.Compute(resultComputer);
}
}
public static Builder WithResultProcessor(Action<TValue> resultProcessor, TaskScheduler? scheduler = null) {
return new Builder(resultProcessor, scheduler ?? TaskScheduler.FromCurrentSynchronizationContext());
}
public sealed class Builder {
private readonly Action<TValue> resultProcessor;
private readonly TaskScheduler resultTaskScheduler;
private bool processOutdatedResults;
internal Builder(Action<TValue> resultProcessor, TaskScheduler resultTaskScheduler) {
this.resultProcessor = resultProcessor;
this.resultTaskScheduler = resultTaskScheduler;
}
public Builder WithOutdatedResults() {
this.processOutdatedResults = true;
return this;
}
public AsyncValueComputer<TValue> Build() {
return new AsyncValueComputer<TValue>(resultProcessor, resultTaskScheduler, processOutdatedResults);
}
public Single BuildWithComputer(Func<CancellationToken, TValue> resultComputer) {
return new Single(Build(), resultComputer);
}
}
}
}

View File

@ -53,11 +53,11 @@ define('LATEST_VERSION', $version === false ? '_' : $version);
<svg class="icon"> <svg class="icon">
<use href="#icon-globe" /> <use href="#icon-globe" />
</svg> </svg>
<span class="platform">Portable</span> <span class="platform">Other</span>
</a> </a>
</div> </div>
<p>To launch the three OS-specific versions, extract the <strong>DiscordHistoryTracker</strong> executable, and double-click it.</p> <p>To launch the three OS-specific versions, extract the <strong>DiscordHistoryTracker</strong> executable, and double-click it.</p>
<p>To launch the <strong>Portable</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0/runtime" rel="nofollow noopener">.NET 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p> <p>To launch the <strong>Other</strong> version, which works on other operating systems including 32-bit versions, you must install <a href="https://dotnet.microsoft.com/download/dotnet/5.0" rel="nofollow noopener">ASP.NET Core Runtime 5</a>. Then extract the downloaded archive into a folder, open the folder in a terminal, and type: <code>dotnet DiscordHistoryTracker.dll</code></p>
<h3>How to Track Messages</h3> <h3>How to Track Messages</h3>
<p>The app saves messages into a database file stored on your computer. When you open the app, you are given the option to create a new database file, or open an existing one.</p> <p>The app saves messages into a database file stored on your computer. When you open the app, you are given the option to create a new database file, or open an existing one.</p>