mirror of
https://github.com/chylex/Discord-History-Tracker.git
synced 2025-09-15 19:32:09 +02:00
Compare commits
4 Commits
v41.0
...
07615de87a
Author | SHA1 | Date | |
---|---|---|---|
07615de87a
|
|||
7fdc19880e
|
|||
67b9c12843
|
|||
9030a2f010
|
@@ -15,6 +15,7 @@ sealed class Arguments {
|
||||
public string? DatabaseFile { get; }
|
||||
public ushort? ServerPort { get; }
|
||||
public string? ServerToken { get; }
|
||||
public byte? ConcurrentDownloads { get; }
|
||||
|
||||
public Arguments(IReadOnlyList<string> args) {
|
||||
for (int i = FirstArgument; i < args.Count; i++) {
|
||||
@@ -50,11 +51,11 @@ sealed class Arguments {
|
||||
continue;
|
||||
|
||||
case "-port": {
|
||||
if (ushort.TryParse(value, out var port)) {
|
||||
ServerPort = port;
|
||||
if (!ushort.TryParse(value, out var port)) {
|
||||
Log.Warn("Invalid port number: " + value);
|
||||
}
|
||||
else {
|
||||
Log.Warn("Invalid port number: " + value);
|
||||
ServerPort = port;
|
||||
}
|
||||
|
||||
continue;
|
||||
@@ -64,6 +65,20 @@ sealed class Arguments {
|
||||
ServerToken = value;
|
||||
continue;
|
||||
|
||||
case "-concurrentdownloads":
|
||||
if (!ulong.TryParse(value, out var concurrentDownloads) || concurrentDownloads == 0) {
|
||||
Log.Warn("Invalid concurrent downloads count: " + value);
|
||||
}
|
||||
else if (concurrentDownloads > 10) {
|
||||
Log.Warn("Limiting concurrent downloads to 10");
|
||||
ConcurrentDownloads = 10;
|
||||
}
|
||||
else {
|
||||
ConcurrentDownloads = (byte) concurrentDownloads;
|
||||
}
|
||||
|
||||
continue;
|
||||
|
||||
default:
|
||||
Log.Warn("Unknown command line argument: " + key);
|
||||
break;
|
||||
|
@@ -30,6 +30,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
|
||||
private MainContentScreenModel? mainContentScreenModel;
|
||||
|
||||
private readonly Window window;
|
||||
private readonly int? concurrentDownloads;
|
||||
|
||||
private State? state;
|
||||
|
||||
@@ -73,6 +74,8 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
|
||||
if (args.ServerToken != null) {
|
||||
ServerConfiguration.Token = args.ServerToken;
|
||||
}
|
||||
|
||||
concurrentDownloads = args.ConcurrentDownloads;
|
||||
}
|
||||
|
||||
private async void OnDatabaseSelected(object? sender, IDatabaseFile db) {
|
||||
@@ -80,7 +83,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable {
|
||||
|
||||
await DisposeState();
|
||||
|
||||
state = new State(db);
|
||||
state = new State(db, concurrentDownloads);
|
||||
|
||||
try {
|
||||
await state.Server.Start(ServerConfiguration.Port, ServerConfiguration.Token);
|
||||
|
@@ -48,6 +48,16 @@ const STATE = (function() {
|
||||
});
|
||||
};
|
||||
|
||||
const getDate = function(date) {
|
||||
if (date instanceof Date) {
|
||||
return date;
|
||||
}
|
||||
else {
|
||||
// noinspection JSUnresolvedReference
|
||||
return date.toDate();
|
||||
}
|
||||
};
|
||||
|
||||
const trackingStateChangedListeners = [];
|
||||
let isTracking = false;
|
||||
|
||||
@@ -69,8 +79,8 @@ const STATE = (function() {
|
||||
* @property {String} channel_id
|
||||
* @property {DiscordUser} author
|
||||
* @property {String} content
|
||||
* @property {Timestamp} timestamp
|
||||
* @property {Timestamp|null} editedTimestamp
|
||||
* @property {Date} timestamp
|
||||
* @property {Date|null} editedTimestamp
|
||||
* @property {DiscordAttachment[]} attachments
|
||||
* @property {Object[]} embeds
|
||||
* @property {DiscordMessageReaction[]} [reactions]
|
||||
@@ -106,11 +116,6 @@ const STATE = (function() {
|
||||
* @property {Boolean} animated
|
||||
*/
|
||||
|
||||
/**
|
||||
* @name Timestamp
|
||||
* @property {Function} toDate
|
||||
*/
|
||||
|
||||
return {
|
||||
setup(port, token) {
|
||||
serverPort = port;
|
||||
@@ -223,12 +228,12 @@ const STATE = (function() {
|
||||
sender: msg.author.id,
|
||||
channel: msg.channel_id,
|
||||
text: msg.content,
|
||||
timestamp: msg.timestamp.toDate().getTime()
|
||||
timestamp: getDate(msg.timestamp).getTime()
|
||||
};
|
||||
|
||||
if (msg.editedTimestamp !== null) {
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
obj.editTimestamp = msg.editedTimestamp.toDate().getTime();
|
||||
obj.editTimestamp = getDate(msg.editedTimestamp).getTime();
|
||||
}
|
||||
|
||||
if (msg.messageReference !== null) {
|
||||
|
@@ -11,16 +11,18 @@ public sealed class Downloader {
|
||||
public bool IsDownloading => current != null;
|
||||
|
||||
private readonly IDatabaseFile db;
|
||||
private readonly int? concurrentDownloads;
|
||||
private readonly SemaphoreSlim semaphore = new (1, 1);
|
||||
|
||||
internal Downloader(IDatabaseFile db) {
|
||||
internal Downloader(IDatabaseFile db, int? concurrentDownloads) {
|
||||
this.db = db;
|
||||
this.concurrentDownloads = concurrentDownloads;
|
||||
}
|
||||
|
||||
public async Task<IObservable<DownloadItem>> Start(DownloadItemFilter filter) {
|
||||
await semaphore.WaitAsync();
|
||||
try {
|
||||
current ??= new DownloaderTask(db, filter);
|
||||
current ??= new DownloaderTask(db, filter, concurrentDownloads);
|
||||
return current.FinishedItems;
|
||||
} finally {
|
||||
semaphore.Release();
|
||||
|
@@ -15,10 +15,14 @@ namespace DHT.Server.Download;
|
||||
sealed class DownloaderTask : IAsyncDisposable {
|
||||
private static readonly Log Log = Log.ForType<DownloaderTask>();
|
||||
|
||||
private const int DownloadTasks = 4;
|
||||
private const int DefaultConcurrentDownloads = 4;
|
||||
private const int QueueSize = 25;
|
||||
private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
||||
|
||||
private static int GetDownloadTaskCount(int? concurrentDownloads) {
|
||||
return Math.Max(1, concurrentDownloads ?? DefaultConcurrentDownloads);
|
||||
}
|
||||
|
||||
private readonly Channel<DownloadItem> downloadQueue = Channel.CreateBounded<DownloadItem>(new BoundedChannelOptions(QueueSize) {
|
||||
SingleReader = false,
|
||||
SingleWriter = true,
|
||||
@@ -38,12 +42,12 @@ sealed class DownloaderTask : IAsyncDisposable {
|
||||
|
||||
public IObservable<DownloadItem> FinishedItems => finishedItemPublisher;
|
||||
|
||||
internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter) {
|
||||
internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter, int? concurrentDownloads) {
|
||||
this.db = db;
|
||||
this.filter = filter;
|
||||
this.cancellationToken = cancellationTokenSource.Token;
|
||||
this.queueWriterTask = Task.Run(RunQueueWriterTask);
|
||||
this.downloadTasks = Enumerable.Range(1, DownloadTasks).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray();
|
||||
this.downloadTasks = Enumerable.Range(1, GetDownloadTaskCount(concurrentDownloads)).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray();
|
||||
}
|
||||
|
||||
private async Task RunQueueWriterTask() {
|
||||
@@ -74,8 +78,12 @@ sealed class DownloaderTask : IAsyncDisposable {
|
||||
try {
|
||||
var downloadedBytes = await client.GetByteArrayAsync(item.DownloadUrl, cancellationToken);
|
||||
await db.Downloads.AddDownload(item.ToSuccess(downloadedBytes));
|
||||
} catch (OperationCanceledException) {
|
||||
} catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) {
|
||||
// Ignore.
|
||||
} catch (TaskCanceledException e) {
|
||||
// HttpClient request timed out.
|
||||
await db.Downloads.AddDownload(item.ToFailure());
|
||||
log.Error(e.Message);
|
||||
} catch (HttpRequestException e) {
|
||||
await db.Downloads.AddDownload(item.ToFailure(e.StatusCode));
|
||||
log.Error(e);
|
||||
|
@@ -6,18 +6,12 @@ using DHT.Server.Service;
|
||||
|
||||
namespace DHT.Server;
|
||||
|
||||
public sealed class State : IAsyncDisposable {
|
||||
public static State Dummy { get; } = new (DummyDatabaseFile.Instance);
|
||||
public sealed class State(IDatabaseFile db, int? concurrentDownloads) : IAsyncDisposable {
|
||||
public static State Dummy { get; } = new (DummyDatabaseFile.Instance, null);
|
||||
|
||||
public IDatabaseFile Db { get; }
|
||||
public Downloader Downloader { get; }
|
||||
public ServerManager Server { get; }
|
||||
|
||||
public State(IDatabaseFile db) {
|
||||
Db = db;
|
||||
Downloader = new Downloader(db);
|
||||
Server = new ServerManager(db);
|
||||
}
|
||||
public IDatabaseFile Db { get; } = db;
|
||||
public Downloader Downloader { get; } = new (db, concurrentDownloads);
|
||||
public ServerManager Server { get; } = new (db);
|
||||
|
||||
public async ValueTask DisposeAsync() {
|
||||
await Downloader.Stop();
|
||||
|
@@ -8,5 +8,5 @@ using DHT.Utils;
|
||||
namespace DHT.Utils;
|
||||
|
||||
static class Version {
|
||||
public const string Tag = "41.0.0.0";
|
||||
public const string Tag = "41.1.0.0";
|
||||
}
|
||||
|
Reference in New Issue
Block a user