1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2024-11-25 05:42:45 +01:00

Compare commits

..

No commits in common. "d4da64a5edf2b549276b6c3703be9496c5d66c0c" and "07615de87ad369d6ae877891ee466867612daf66" have entirely different histories.

28 changed files with 415 additions and 440 deletions

View File

@ -10,164 +10,164 @@ using DHT.Server;
using DHT.Server.Data; using DHT.Server.Data;
using DHT.Server.Service; using DHT.Server.Service;
namespace DHT.Desktop.Main.Pages; namespace DHT.Desktop.Main.Pages {
sealed class DebugPageModel {
public string GenerateChannels { get; set; } = "0";
public string GenerateUsers { get; set; } = "0";
public string GenerateMessages { get; set; } = "0";
sealed class DebugPageModel { private readonly Window window;
public string GenerateChannels { get; set; } = "0"; private readonly State state;
public string GenerateUsers { get; set; } = "0";
public string GenerateMessages { get; set; } = "0";
private readonly Window window; [Obsolete("Designer")]
private readonly State state; public DebugPageModel() : this(null!, State.Dummy) {}
[Obsolete("Designer")] public DebugPageModel(Window window, State state) {
public DebugPageModel() : this(null!, State.Dummy) {} this.window = window;
this.state = state;
public DebugPageModel(Window window, State state) {
this.window = window;
this.state = state;
}
public async void OnClickAddRandomDataToDatabase() {
if (!int.TryParse(GenerateChannels, out int channels) || channels < 1) {
await Dialog.ShowOk(window, "Generate Random Data", "Amount of channels must be at least 1!");
return;
} }
if (!int.TryParse(GenerateUsers, out int users) || users < 1) { public async void OnClickAddRandomDataToDatabase() {
await Dialog.ShowOk(window, "Generate Random Data", "Amount of users must be at least 1!"); if (!int.TryParse(GenerateChannels, out int channels) || channels < 1) {
return; await Dialog.ShowOk(window, "Generate Random Data", "Amount of channels must be at least 1!");
return;
}
if (!int.TryParse(GenerateUsers, out int users) || users < 1) {
await Dialog.ShowOk(window, "Generate Random Data", "Amount of users must be at least 1!");
return;
}
if (!int.TryParse(GenerateMessages, out int messages) || messages < 1) {
await Dialog.ShowOk(window, "Generate Random Data", "Amount of messages must be at least 1!");
return;
}
await ProgressDialog.Show(window, "Generating Random Data", async (_, callback) => await GenerateRandomData(channels, users, messages, callback));
} }
if (!int.TryParse(GenerateMessages, out int messages) || messages < 1) { private const int BatchSize = 500;
await Dialog.ShowOk(window, "Generate Random Data", "Amount of messages must be at least 1!");
return;
}
await ProgressDialog.Show(window, "Generating Random Data", async (_, callback) => await GenerateRandomData(channels, users, messages, callback)); private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) {
} int batchCount = (messageCount + BatchSize - 1) / BatchSize;
await callback.Update("Adding messages in batches of " + BatchSize, 0, batchCount);
private const int BatchSize = 500; var rand = new Random();
var server = new DHT.Server.Data.Server {
Id = RandomId(rand),
Name = RandomName("s"),
Type = ServerType.Server,
};
private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) { var channels = Enumerable.Range(0, channelCount).Select(i => new Channel {
int batchCount = (messageCount + BatchSize - 1) / BatchSize; Id = RandomId(rand),
await callback.Update("Adding messages in batches of " + BatchSize, 0, batchCount); Server = server.Id,
Name = RandomName("c"),
var rand = new Random(); ParentId = null,
var server = new DHT.Server.Data.Server { Position = i,
Id = RandomId(rand), Topic = RandomText(rand, 10),
Name = RandomName("s"), Nsfw = rand.Next(4) == 0,
Type = ServerType.Server,
};
var channels = Enumerable.Range(0, channelCount).Select(i => new Channel {
Id = RandomId(rand),
Server = server.Id,
Name = RandomName("c"),
ParentId = null,
Position = i,
Topic = RandomText(rand, 10),
Nsfw = rand.Next(4) == 0,
}).ToArray();
var users = Enumerable.Range(0, userCount).Select(_ => new User {
Id = RandomId(rand),
Name = RandomName("u"),
AvatarUrl = null,
Discriminator = rand.Next(0, 9999).ToString(),
}).ToArray();
await state.Db.Users.Add(users);
await state.Db.Servers.Add([server]);
await state.Db.Channels.Add(channels);
var now = DateTimeOffset.Now;
int batchIndex = 0;
while (messageCount > 0) {
int hourOffset = batchIndex;
var messages = Enumerable.Range(0, Math.Min(messageCount, BatchSize)).Select(i => {
DateTimeOffset time = now.AddHours(hourOffset).AddMinutes(i * 60.0 / BatchSize);
DateTimeOffset? edit = rand.Next(100) == 0 ? time.AddSeconds(rand.Next(1, 60)) : null;
var timeMillis = time.ToUnixTimeMilliseconds();
var editMillis = edit?.ToUnixTimeMilliseconds();
return new Message {
Id = (ulong) timeMillis,
Sender = RandomBiasedIndex(rand, users).Id,
Channel = RandomBiasedIndex(rand, channels).Id,
Text = RandomText(rand, 100),
Timestamp = timeMillis,
EditTimestamp = editMillis,
RepliedToId = null,
Attachments = ImmutableList<Attachment>.Empty,
Embeds = ImmutableList<Embed>.Empty,
Reactions = ImmutableList<Reaction>.Empty,
};
}).ToArray(); }).ToArray();
await state.Db.Messages.Add(messages); var users = Enumerable.Range(0, userCount).Select(_ => new User {
Id = RandomId(rand),
Name = RandomName("u"),
AvatarUrl = null,
Discriminator = rand.Next(0, 9999).ToString(),
}).ToArray();
messageCount -= BatchSize; await state.Db.Users.Add(users);
await callback.Update("Adding messages in batches of " + BatchSize, ++batchIndex, batchCount); await state.Db.Servers.Add([server]);
await state.Db.Channels.Add(channels);
var now = DateTimeOffset.Now;
int batchIndex = 0;
while (messageCount > 0) {
int hourOffset = batchIndex;
var messages = Enumerable.Range(0, Math.Min(messageCount, BatchSize)).Select(i => {
DateTimeOffset time = now.AddHours(hourOffset).AddMinutes(i * 60.0 / BatchSize);
DateTimeOffset? edit = rand.Next(100) == 0 ? time.AddSeconds(rand.Next(1, 60)) : null;
var timeMillis = time.ToUnixTimeMilliseconds();
var editMillis = edit?.ToUnixTimeMilliseconds();
return new Message {
Id = (ulong) timeMillis,
Sender = RandomBiasedIndex(rand, users).Id,
Channel = RandomBiasedIndex(rand, channels).Id,
Text = RandomText(rand, 100),
Timestamp = timeMillis,
EditTimestamp = editMillis,
RepliedToId = null,
Attachments = ImmutableList<Attachment>.Empty,
Embeds = ImmutableList<Embed>.Empty,
Reactions = ImmutableList<Reaction>.Empty,
};
}).ToArray();
await state.Db.Messages.Add(messages);
messageCount -= BatchSize;
await callback.Update("Adding messages in batches of " + BatchSize, ++batchIndex, batchCount);
}
} }
}
private static ulong RandomId(Random rand) { private static ulong RandomId(Random rand) {
ulong h = unchecked((ulong) rand.Next()); ulong h = unchecked((ulong) rand.Next());
ulong l = unchecked((ulong) rand.Next()); ulong l = unchecked((ulong) rand.Next());
return (h << 32) | l; return (h << 32) | l;
} }
private static string RandomName(string prefix) { private static string RandomName(string prefix) {
return prefix + "-" + ServerUtils.GenerateRandomToken(5); return prefix + "-" + ServerUtils.GenerateRandomToken(5);
} }
private static T RandomBiasedIndex<T>(Random rand, T[] options) { private static T RandomBiasedIndex<T>(Random rand, T[] options) {
return options[(int) Math.Floor(options.Length * rand.NextDouble() * rand.NextDouble())]; return options[(int) Math.Floor(options.Length * rand.NextDouble() * rand.NextDouble())];
} }
private static readonly string[] RandomWords = [ private static readonly string[] RandomWords = [
"apple", "apricot", "artichoke", "arugula", "asparagus", "avocado", "apple", "apricot", "artichoke", "arugula", "asparagus", "avocado",
"banana", "bean", "beechnut", "beet", "blackberry", "blackcurrant", "blueberry", "boysenberry", "bramble", "broccoli", "banana", "bean", "beechnut", "beet", "blackberry", "blackcurrant", "blueberry", "boysenberry", "bramble", "broccoli",
"cabbage", "cacao", "cantaloupe", "caper", "carambola", "carrot", "cauliflower", "celery", "chard", "cherry", "chokeberry", "citron", "clementine", "coconut", "corn", "crabapple", "cranberry", "cucumber", "currant", "cabbage", "cacao", "cantaloupe", "caper", "carambola", "carrot", "cauliflower", "celery", "chard", "cherry", "chokeberry", "citron", "clementine", "coconut", "corn", "crabapple", "cranberry", "cucumber", "currant",
"daikon", "date", "dewberry", "durian", "daikon", "date", "dewberry", "durian",
"edamame", "eggplant", "elderberry", "endive", "edamame", "eggplant", "elderberry", "endive",
"fig", "fig",
"garlic", "ginger", "gooseberry", "grape", "grapefruit", "guava", "garlic", "ginger", "gooseberry", "grape", "grapefruit", "guava",
"honeysuckle", "horseradish", "huckleberry", "honeysuckle", "horseradish", "huckleberry",
"jackfruit", "jicama", "jackfruit", "jicama",
"kale", "kiwi", "kohlrabi", "kumquat", "kale", "kiwi", "kohlrabi", "kumquat",
"leek", "lemon", "lentil", "lettuce", "lime", "leek", "lemon", "lentil", "lettuce", "lime",
"mandarin", "mango", "mushroom", "myrtle", "mandarin", "mango", "mushroom", "myrtle",
"nectarine", "nut", "nectarine", "nut",
"olive", "okra", "onion", "orange", "olive", "okra", "onion", "orange",
"papaya", "parsnip", "pawpaw", "peach", "pear", "pea", "pepper", "persimmon", "pineapple", "plum", "plantain", "pomegranate", "pomelo", "potato", "prune", "pumpkin", "papaya", "parsnip", "pawpaw", "peach", "pear", "pea", "pepper", "persimmon", "pineapple", "plum", "plantain", "pomegranate", "pomelo", "potato", "prune", "pumpkin",
"quandong", "quinoa", "quandong", "quinoa",
"radicchio", "radish", "raisin", "raspberry", "redcurrant", "rhubarb", "rutabaga", "radicchio", "radish", "raisin", "raspberry", "redcurrant", "rhubarb", "rutabaga",
"spinach", "strawberry", "squash", "spinach", "strawberry", "squash",
"tamarind", "tangerine", "tomatillo", "tomato", "turnip", "tamarind", "tangerine", "tomatillo", "tomato", "turnip",
"vanilla", "vanilla",
"watercress", "watermelon", "watercress", "watermelon",
"yam", "yam",
"zucchini" "zucchini"
]; ];
private static string RandomText(Random rand, int maxWords) { private static string RandomText(Random rand, int maxWords) {
int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3)); int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3));
return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)])); return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)]));
}
} }
} }
#else #else
namespace DHT.Desktop.Main.Pages; namespace DHT.Desktop.Main.Pages {
sealed class DebugPageModel {
public string GenerateChannels { get; set; } = "0";
public string GenerateUsers { get; set; } = "0";
public string GenerateMessages { get; set; } = "0";
sealed class DebugPageModel { public void OnClickAddRandomDataToDatabase() {}
public string GenerateChannels { get; set; } = "0"; }
public string GenerateUsers { get; set; } = "0";
public string GenerateMessages { get; set; } = "0";
public void OnClickAddRandomDataToDatabase() {}
} }
#endif #endif

View File

@ -0,0 +1,3 @@
namespace DHT.Server.Data;
public readonly record struct DownloadWithData(Download Download, byte[]? Data);

View File

@ -26,9 +26,7 @@ public static class DatabaseExtensions {
await target.Messages.Add(batchedMessages); await target.Messages.Add(batchedMessages);
await foreach (var download in source.Downloads.Get()) { await foreach (var download in source.Downloads.Get()) {
if (download.Status != DownloadStatus.Success || !await source.Downloads.GetDownloadData(download.NormalizedUrl, stream => target.Downloads.AddDownload(download, stream))) { await target.Downloads.AddDownload(await source.Downloads.HydrateWithData(download));
await target.Downloads.AddDownload(download, stream: null);
}
} }
} }
} }

View File

@ -161,7 +161,7 @@ public static class LegacyArchiveImport {
var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null; var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null;
if (messagesObj == null) { if (messagesObj == null) {
return []; return Array.Empty<Message>();
} }
return messagesObj.Value.EnumerateObject().Select(item => { return messagesObj.Value.EnumerateObject().Select(item => {

View File

@ -1,10 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Linq; using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations; using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters; using DHT.Server.Data.Filters;
using DHT.Server.Download; using DHT.Server.Download;
@ -14,7 +14,7 @@ namespace DHT.Server.Database.Repositories;
public interface IDownloadRepository { public interface IDownloadRepository {
IObservable<long> TotalCount { get; } IObservable<long> TotalCount { get; }
Task AddDownload(Data.Download item, Stream? stream); Task AddDownload(DownloadWithData item);
Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken = default); Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken = default);
@ -22,9 +22,9 @@ public interface IDownloadRepository {
IAsyncEnumerable<Data.Download> Get(); IAsyncEnumerable<Data.Download> Get();
Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor); Task<DownloadWithData> HydrateWithData(Data.Download download);
Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor); Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl);
IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default); IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default);
@ -35,7 +35,7 @@ public interface IDownloadRepository {
internal sealed class Dummy : IDownloadRepository { internal sealed class Dummy : IDownloadRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L); public IObservable<long> TotalCount { get; } = Observable.Return(0L);
public Task AddDownload(Data.Download item, Stream? stream) { public Task AddDownload(DownloadWithData item) {
return Task.CompletedTask; return Task.CompletedTask;
} }
@ -51,12 +51,12 @@ public interface IDownloadRepository {
return AsyncEnumerable.Empty<Data.Download>(); return AsyncEnumerable.Empty<Data.Download>();
} }
public Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) { public Task<DownloadWithData> HydrateWithData(Data.Download download) {
return Task.FromResult(false); return Task.FromResult(new DownloadWithData(download, Data: null));
} }
public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) { public Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) {
return Task.FromResult(false); return Task.FromResult<DownloadWithData?>(null);
} }
public IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken) { public IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken) {

View File

@ -19,9 +19,9 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
} }
public async Task Add(IReadOnlyList<Channel> channels) { public async Task Add(IReadOnlyList<Channel> channels) {
await using (var conn = await pool.Take()) { await using var conn = await pool.Take();
await conn.BeginTransactionAsync();
await using (var tx = await conn.BeginTransactionAsync()) {
await using var cmd = conn.Upsert("channels", [ await using var cmd = conn.Upsert("channels", [
("id", SqliteType.Integer), ("id", SqliteType.Integer),
("server", SqliteType.Integer), ("server", SqliteType.Integer),
@ -43,7 +43,7 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
} }
await conn.CommitTransactionAsync(); await tx.CommitAsync();
} }
UpdateTotalCount(); UpdateTotalCount();

View File

@ -1,6 +1,5 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -15,9 +14,15 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories; namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository { sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepository {
private static readonly Log Log = Log.ForType<SqliteDownloadRepository>(); private static readonly Log Log = Log.ForType<SqliteDownloadRepository>();
private readonly SqliteConnectionPool pool;
public SqliteDownloadRepository(SqliteConnectionPool pool) : base(Log) {
this.pool = pool;
}
internal sealed class NewDownloadCollector : IAsyncDisposable { internal sealed class NewDownloadCollector : IAsyncDisposable {
private readonly SqliteDownloadRepository repository; private readonly SqliteDownloadRepository repository;
private bool hasAdded = false; private bool hasAdded = false;
@ -61,9 +66,11 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
} }
} }
public async Task AddDownload(Data.Download item, Stream? stream) { public async Task AddDownload(DownloadWithData item) {
var (download, data) = item;
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); var tx = await conn.BeginTransactionAsync();
await using var metadataCmd = conn.Upsert("download_metadata", [ await using var metadataCmd = conn.Upsert("download_metadata", [
("normalized_url", SqliteType.Text), ("normalized_url", SqliteType.Text),
@ -73,37 +80,30 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
("size", SqliteType.Integer), ("size", SqliteType.Integer),
]); ]);
metadataCmd.Set(":normalized_url", item.NormalizedUrl); metadataCmd.Set(":normalized_url", download.NormalizedUrl);
metadataCmd.Set(":download_url", item.DownloadUrl); metadataCmd.Set(":download_url", download.DownloadUrl);
metadataCmd.Set(":status", (int) item.Status); metadataCmd.Set(":status", (int) download.Status);
metadataCmd.Set(":type", item.Type); metadataCmd.Set(":type", download.Type);
metadataCmd.Set(":size", item.Size); metadataCmd.Set(":size", download.Size);
await metadataCmd.ExecuteNonQueryAsync(); await metadataCmd.ExecuteNonQueryAsync();
if (stream == null) { if (data == null) {
await using var deleteBlobCmd = conn.Command("DELETE FROM download_blobs WHERE normalized_url = :normalized_url"); await using var deleteBlobCmd = conn.Command("DELETE FROM download_blobs WHERE normalized_url = :normalized_url");
deleteBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl); deleteBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, download.NormalizedUrl);
await deleteBlobCmd.ExecuteNonQueryAsync(); await deleteBlobCmd.ExecuteNonQueryAsync();
} }
else { else {
await using var upsertBlobCmd = conn.Command( await using var upsertBlobCmd = conn.Upsert("download_blobs", [
""" ("normalized_url", SqliteType.Text),
INSERT INTO download_blobs (normalized_url, blob) ("blob", SqliteType.Blob)
VALUES (:normalized_url, ZEROBLOB(:blob_length)) ]);
ON CONFLICT (normalized_url) DO UPDATE SET blob = excluded.blob
RETURNING rowid
"""
);
upsertBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl); upsertBlobCmd.Set(":normalized_url", download.NormalizedUrl);
upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, item.Size); upsertBlobCmd.Set(":blob", data);
long rowid = await upsertBlobCmd.ExecuteLongScalarAsync(); await upsertBlobCmd.ExecuteNonQueryAsync();
await using var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid);
await stream.CopyToAsync(blob);
} }
await conn.CommitTransactionAsync(); await tx.CommitAsync();
} }
UpdateTotalCount(); UpdateTotalCount();
@ -187,35 +187,24 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
} }
} }
public async Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) { public async Task<DownloadWithData> HydrateWithData(Data.Download download) {
await using var conn = await pool.Take(); await using var conn = await pool.Take();
await using var cmd = conn.Command("SELECT rowid FROM download_blobs WHERE normalized_url = :normalized_url"); await using var cmd = conn.Command("SELECT blob FROM download_blobs WHERE normalized_url = :url");
cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl); cmd.AddAndSet(":url", SqliteType.Text, download.NormalizedUrl);
long rowid; await using var reader = await cmd.ExecuteReaderAsync();
var data = await reader.ReadAsync() && !reader.IsDBNull(0) ? (byte[]) reader["blob"] : null;
await using (var reader = await cmd.ExecuteReaderAsync()) { return new DownloadWithData(download, data);
if (!await reader.ReadAsync()) {
return false;
}
rowid = reader.GetInt64(0);
}
await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
await dataProcessor(blob);
}
return true;
} }
public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) { public async Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) {
await using var conn = await pool.Take(); await using var conn = await pool.Take();
await using var cmd = conn.Command( await using var cmd = conn.Command(
""" """
SELECT dm.download_url, dm.type, db.rowid FROM download_metadata dm SELECT dm.download_url, dm.type, db.blob FROM download_metadata dm
JOIN download_blobs db ON dm.normalized_url = db.normalized_url JOIN download_blobs db ON dm.normalized_url = db.normalized_url
WHERE dm.normalized_url = :normalized_url AND dm.status = :success IS NOT NULL WHERE dm.normalized_url = :normalized_url AND dm.status = :success IS NOT NULL
""" """
@ -224,25 +213,19 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl); cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl);
cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success); cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success);
string downloadUrl; await using var reader = await cmd.ExecuteReaderAsync();
string? type;
long rowid;
await using (var reader = await cmd.ExecuteReaderAsync()) { if (!await reader.ReadAsync()) {
if (!await reader.ReadAsync()) { return null;
return false;
}
downloadUrl = reader.GetString(0);
type = reader.IsDBNull(1) ? null : reader.GetString(1);
rowid = reader.GetInt64(2);
} }
await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) { var downloadUrl = reader.GetString(0);
await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob); var type = reader.IsDBNull(1) ? null : reader.GetString(1);
} var data = (byte[]) reader[2];
var size = (ulong) data.LongLength;
var download = new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, size);
return true; return new DownloadWithData(download, data);
} }
public async IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, [EnumeratorCancellation] CancellationToken cancellationToken) { public async IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, [EnumeratorCancellation] CancellationToken cancellationToken) {

View File

@ -39,7 +39,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
} }
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); await using var tx = await conn.BeginTransactionAsync();
await using var messageCmd = conn.Upsert("messages", [ await using var messageCmd = conn.Upsert("messages", [
("message_id", SqliteType.Integer), ("message_id", SqliteType.Integer),
@ -167,7 +167,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
} }
} }
await conn.CommitTransactionAsync(); await tx.CommitAsync();
downloadCollector.OnCommitted(); downloadCollector.OnCommitted();
} }
@ -183,11 +183,11 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken); return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
} }
private sealed class MessageToManyCommand<T> : IAsyncDisposable { private sealed class MesageToManyCommand<T> : IAsyncDisposable {
private readonly SqliteCommand cmd; private readonly SqliteCommand cmd;
private readonly Func<SqliteDataReader, T> readItem; private readonly Func<SqliteDataReader, T> readItem;
public MessageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) { public MesageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) {
this.cmd = conn.Command(sql); this.cmd = conn.Command(sql);
this.cmd.Add(":message_id", SqliteType.Integer); this.cmd.Add(":message_id", SqliteType.Integer);
@ -223,7 +223,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
WHERE message_id = :message_id WHERE message_id = :message_id
"""; """;
await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment { await using var attachmentCmd = new MesageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment {
Id = reader.GetUint64(0), Id = reader.GetUint64(0),
Name = reader.GetString(1), Name = reader.GetString(1),
Type = reader.IsDBNull(2) ? null : reader.GetString(2), Type = reader.IsDBNull(2) ? null : reader.GetString(2),
@ -241,7 +241,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
WHERE message_id = :message_id WHERE message_id = :message_id
"""; """;
await using var embedCmd = new MessageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed { await using var embedCmd = new MesageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed {
Json = reader.GetString(0) Json = reader.GetString(0)
}); });
@ -252,7 +252,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
WHERE message_id = :message_id WHERE message_id = :message_id
"""; """;
await using var reactionsCmd = new MessageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction { await using var reactionsCmd = new MesageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction {
EmojiId = reader.IsDBNull(0) ? null : reader.GetUint64(0), EmojiId = reader.IsDBNull(0) ? null : reader.GetUint64(0),
EmojiName = reader.IsDBNull(1) ? null : reader.GetString(1), EmojiName = reader.IsDBNull(1) ? null : reader.GetString(1),
EmojiFlags = (EmojiFlags) reader.GetInt16(2), EmojiFlags = (EmojiFlags) reader.GetInt16(2),

View File

@ -19,9 +19,9 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
} }
public async Task Add(IReadOnlyList<Data.Server> servers) { public async Task Add(IReadOnlyList<Data.Server> servers) {
await using (var conn = await pool.Take()) { await using var conn = await pool.Take();
await conn.BeginTransactionAsync();
await using (var tx = await conn.BeginTransactionAsync()) {
await using var cmd = conn.Upsert("servers", [ await using var cmd = conn.Upsert("servers", [
("id", SqliteType.Integer), ("id", SqliteType.Integer),
("name", SqliteType.Text), ("name", SqliteType.Text),
@ -35,7 +35,7 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
} }
await conn.CommitTransactionAsync(); await tx.CommitAsync();
} }
UpdateTotalCount(); UpdateTotalCount();

View File

@ -23,7 +23,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
public async Task Add(IReadOnlyList<User> users) { public async Task Add(IReadOnlyList<User> users) {
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); await using var tx = await conn.BeginTransactionAsync();
await using var cmd = conn.Upsert("users", [ await using var cmd = conn.Upsert("users", [
("id", SqliteType.Integer), ("id", SqliteType.Integer),
@ -46,7 +46,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
} }
} }
await conn.CommitTransactionAsync(); await tx.CommitAsync();
downloadCollector.OnCommitted(); downloadCollector.OnCommitted();
} }

View File

@ -1,4 +1,5 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Data.Common;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Server.Download; using DHT.Server.Download;
@ -38,7 +39,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
} }
} }
await conn.BeginTransactionAsync(); await using var tx = await conn.BeginTransactionAsync();
int totalUrls = normalizedUrls.Count; int totalUrls = normalizedUrls.Count;
int processedUrls = -1; int processedUrls = -1;
@ -60,7 +61,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
await conn.CommitTransactionAsync(); await tx.CommitAsync();
} }
private async Task NormalizeDownloadUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) { private async Task NormalizeDownloadUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
@ -83,23 +84,26 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
} }
await conn.ExecuteAsync("PRAGMA cache_size = -20000"); await conn.ExecuteAsync("PRAGMA cache_size = -20000");
await conn.BeginTransactionAsync();
await reporter.SubWork("Deleting duplicates...", 0, 0); DbTransaction tx;
await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) { await using (tx = await conn.BeginTransactionAsync()) {
foreach (var duplicateUrl in duplicateUrlsToDelete) { await reporter.SubWork("Deleting duplicates...", 0, 0);
deleteCmd.Set(":url", duplicateUrl);
await deleteCmd.ExecuteNonQueryAsync(); await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) {
foreach (var duplicateUrl in duplicateUrlsToDelete) {
deleteCmd.Set(":url", duplicateUrl);
await deleteCmd.ExecuteNonQueryAsync();
}
} }
}
await conn.CommitTransactionAsync(); await tx.CommitAsync();
}
int totalUrls = normalizedUrlsToOriginalUrls.Count; int totalUrls = normalizedUrlsToOriginalUrls.Count;
int processedUrls = -1; int processedUrls = -1;
await conn.BeginTransactionAsync(); tx = await conn.BeginTransactionAsync();
await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) { await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) {
updateCmd.Add(":normalized_url", SqliteType.Text); updateCmd.Add(":normalized_url", SqliteType.Text);
@ -111,10 +115,11 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
// Not proper way of dealing with transactions, but it avoids a long commit at the end. // Not proper way of dealing with transactions, but it avoids a long commit at the end.
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse. // Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
await conn.CommitTransactionAsync(); await tx.CommitAsync();
await tx.DisposeAsync();
await conn.BeginTransactionAsync(); tx = await conn.BeginTransactionAsync();
conn.AssignActiveTransaction(updateCmd); updateCmd.Transaction = (SqliteTransaction) tx;
} }
updateCmd.Set(":normalized_url", normalizedUrl); updateCmd.Set(":normalized_url", normalizedUrl);
@ -125,7 +130,8 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade {
await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); await reporter.SubWork("Updating URLs...", totalUrls, totalUrls);
await conn.CommitTransactionAsync(); await tx.CommitAsync();
await tx.DisposeAsync();
await conn.ExecuteAsync("PRAGMA cache_size = -2000"); await conn.ExecuteAsync("PRAGMA cache_size = -2000");
} }

View File

@ -37,7 +37,7 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await reporter.SubWork("Processing downloaded files...", 0, totalFiles); await reporter.SubWork("Processing downloaded files...", 0, totalFiles);
await conn.BeginTransactionAsync(); var tx = await conn.BeginTransactionAsync();
await using (var insertCmd = conn.Command("INSERT INTO download_blobs (normalized_url, blob) SELECT normalized_url, blob FROM downloads WHERE normalized_url = :normalized_url")) await using (var insertCmd = conn.Command("INSERT INTO download_blobs (normalized_url, blob) SELECT normalized_url, blob FROM downloads WHERE normalized_url = :normalized_url"))
await using (var deleteCmd = conn.Command("DELETE FROM downloads WHERE normalized_url = :normalized_url")) { await using (var deleteCmd = conn.Command("DELETE FROM downloads WHERE normalized_url = :normalized_url")) {
@ -50,11 +50,12 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
// Not proper way of dealing with transactions, but it avoids a long commit at the end. // Not proper way of dealing with transactions, but it avoids a long commit at the end.
// Schema upgrades are already non-atomic anyways, so this doesn't make it worse. // Schema upgrades are already non-atomic anyways, so this doesn't make it worse.
await conn.CommitTransactionAsync(); await tx.CommitAsync();
await tx.DisposeAsync();
await conn.BeginTransactionAsync(); tx = await conn.BeginTransactionAsync();
conn.AssignActiveTransaction(insertCmd); insertCmd.Transaction = (SqliteTransaction) tx;
conn.AssignActiveTransaction(deleteCmd); deleteCmd.Transaction = (SqliteTransaction) tx;
} }
insertCmd.Set(":normalized_url", url); insertCmd.Set(":normalized_url", url);
@ -67,7 +68,8 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await reporter.SubWork("Processing downloaded files...", totalFiles, totalFiles); await reporter.SubWork("Processing downloaded files...", totalFiles, totalFiles);
await conn.CommitTransactionAsync(); await tx.CommitAsync();
await tx.DisposeAsync();
} }
private async Task<List<string>> GetDownloadedFileUrls(ISqliteConnection conn) { private async Task<List<string>> GetDownloadedFileUrls(ISqliteConnection conn) {
@ -109,45 +111,45 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await insertCmd.ExecuteNonQueryAsync(); await insertCmd.ExecuteNonQueryAsync();
} }
await conn.BeginTransactionAsync(); await using (var tx = await conn.BeginTransactionAsync()) {
await using var insertCmd = conn.Command("INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size) VALUES (:normalized_url, :download_url, :status, :type, :size)");
insertCmd.Add(":normalized_url", SqliteType.Text);
insertCmd.Add(":download_url", SqliteType.Text);
insertCmd.Add(":status", SqliteType.Integer);
insertCmd.Add(":type", SqliteType.Text);
insertCmd.Add(":size", SqliteType.Integer);
await using var insertCmd = conn.Command("INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size) VALUES (:normalized_url, :download_url, :status, :type, :size)"); await reporter.SubWork("Processing embeds...", 1, 4);
insertCmd.Add(":normalized_url", SqliteType.Text);
insertCmd.Add(":download_url", SqliteType.Text);
insertCmd.Add(":status", SqliteType.Integer);
insertCmd.Add(":type", SqliteType.Text);
insertCmd.Add(":size", SqliteType.Integer);
await reporter.SubWork("Processing embeds...", 1, 4); await using (var embedCmd = conn.Command("SELECT json FROM embeds")) {
await using var reader = await embedCmd.ExecuteReaderAsync();
await using (var embedCmd = conn.Command("SELECT json FROM embeds")) { while (await reader.ReadAsync()) {
await using var reader = await embedCmd.ExecuteReaderAsync(); await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)));
}
while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)));
} }
}
await reporter.SubWork("Processing users...", 2, 4); await reporter.SubWork("Processing users...", 2, 4);
await using (var avatarCmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) { await using (var avatarCmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) {
await using var reader = await avatarCmd.ExecuteReaderAsync(); await using var reader = await avatarCmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) { while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1))); await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1)));
}
} }
}
await reporter.SubWork("Processing reactions...", 3, 4); await reporter.SubWork("Processing reactions...", 3, 4);
await using (var avatarCmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM reactions WHERE emoji_id IS NOT NULL")) { await using (var avatarCmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM reactions WHERE emoji_id IS NOT NULL")) {
await using var reader = await avatarCmd.ExecuteReaderAsync(); await using var reader = await avatarCmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) { while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1))); await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1)));
}
} }
}
await conn.CommitTransactionAsync(); await tx.CommitAsync();
}
} }
} }

View File

@ -1,15 +1,8 @@
using System; using System;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils; namespace DHT.Server.Database.Sqlite.Utils;
interface ISqliteConnection : IAsyncDisposable { interface ISqliteConnection : IAsyncDisposable {
SqliteConnection InnerConnection { get; } SqliteConnection InnerConnection { get; }
Task BeginTransactionAsync();
Task CommitTransactionAsync();
Task RollbackTransactionAsync();
void AssignActiveTransaction(SqliteCommand command);
} }

View File

@ -1,6 +1,5 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Data.Common;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Utils.Collections; using DHT.Utils.Collections;
@ -74,48 +73,17 @@ sealed class SqliteConnectionPool : IAsyncDisposable {
disposalTokenSource.Dispose(); disposalTokenSource.Dispose();
} }
private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) : ISqliteConnection { private sealed class PooledConnection : ISqliteConnection {
public SqliteConnection InnerConnection { get; } = conn; public SqliteConnection InnerConnection { get; }
private DbTransaction? activeTransaction; private readonly SqliteConnectionPool pool;
public async Task BeginTransactionAsync() { public PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) {
if (activeTransaction != null) { this.pool = pool;
throw new InvalidOperationException("A transaction is already active."); this.InnerConnection = conn;
}
activeTransaction = await InnerConnection.BeginTransactionAsync();
}
public async Task CommitTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to commit.");
}
await activeTransaction.CommitAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public async Task RollbackTransactionAsync() {
if (activeTransaction == null) {
throw new InvalidOperationException("No active transaction to rollback.");
}
await activeTransaction.RollbackAsync();
await activeTransaction.DisposeAsync();
activeTransaction = null;
}
public void AssignActiveTransaction(SqliteCommand command) {
command.Transaction = (SqliteTransaction?) activeTransaction;
} }
public async ValueTask DisposeAsync() { public async ValueTask DisposeAsync() {
if (activeTransaction != null) {
await RollbackTransactionAsync();
}
await pool.Return(this); await pool.Return(this);
} }
} }

View File

@ -1,4 +1,5 @@
using System; using System;
using System.Data.Common;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -8,6 +9,10 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Utils; namespace DHT.Server.Database.Sqlite.Utils;
static class SqliteExtensions { static class SqliteExtensions {
public static ValueTask<DbTransaction> BeginTransactionAsync(this ISqliteConnection conn) {
return conn.InnerConnection.BeginTransactionAsync();
}
public static SqliteCommand Command(this ISqliteConnection conn, [LanguageInjection("sql")] string sql) { public static SqliteCommand Command(this ISqliteConnection conn, [LanguageInjection("sql")] string sql) {
var cmd = conn.InnerConnection.CreateCommand(); var cmd = conn.InnerConnection.CreateCommand();
cmd.CommandText = sql; cmd.CommandText = sql;
@ -26,10 +31,6 @@ static class SqliteExtensions {
return await reader.ReadAsync(cancellationToken) ? readFunction(reader) : readFunction(null); return await reader.ReadAsync(cancellationToken) ? readFunction(reader) : readFunction(null);
} }
public static async Task<long> ExecuteLongScalarAsync(this SqliteCommand command) {
return (long) (await command.ExecuteScalarAsync())!;
}
public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) { public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
string columnNames = string.Join(',', columns.Select(static c => c.Name)); string columnNames = string.Join(',', columns.Select(static c => c.Name));
string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name)); string columnParams = string.Join(',', columns.Select(static c => ':' + c.Name));

View File

@ -10,12 +10,13 @@ public readonly struct DownloadItem {
public string? Type { get; init; } public string? Type { get; init; }
public ulong? Size { get; init; } public ulong? Size { get; init; }
internal Data.Download ToSuccess(long size) { internal DownloadWithData ToSuccess(byte[] data) {
return new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, (ulong) Math.Max(size, 0)); var size = (ulong) Math.Max(data.LongLength, 0);
return new DownloadWithData(new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, size), data);
} }
internal Data.Download ToFailure(HttpStatusCode? statusCode = null) { internal DownloadWithData ToFailure(HttpStatusCode? statusCode = null) {
var status = statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError; var status = statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError;
return new Data.Download(NormalizedUrl, DownloadUrl, status, Type, Size); return new DownloadWithData(new Data.Download(NormalizedUrl, DownloadUrl, status, Type, Size), Data: null);
} }
} }

View File

@ -67,39 +67,28 @@ sealed class DownloaderTask : IAsyncDisposable {
private async Task RunDownloadTask(int taskIndex) { private async Task RunDownloadTask(int taskIndex) {
var log = Log.ForType<DownloaderTask>("Task " + taskIndex); var log = Log.ForType<DownloaderTask>("Task " + taskIndex);
var client = new HttpClient(new SocketsHttpHandler { var client = new HttpClient();
ConnectTimeout = TimeSpan.FromSeconds(30)
});
client.Timeout = Timeout.InfiniteTimeSpan;
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
client.Timeout = TimeSpan.FromSeconds(30);
while (!cancellationToken.IsCancellationRequested) { while (!cancellationToken.IsCancellationRequested) {
var item = await downloadQueue.Reader.ReadAsync(cancellationToken); var item = await downloadQueue.Reader.ReadAsync(cancellationToken);
log.Debug("Downloading " + item.DownloadUrl + "..."); log.Debug("Downloading " + item.DownloadUrl + "...");
try { try {
var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, item.DownloadUrl), HttpCompletionOption.ResponseHeadersRead, cancellationToken); var downloadedBytes = await client.GetByteArrayAsync(item.DownloadUrl, cancellationToken);
response.EnsureSuccessStatusCode(); await db.Downloads.AddDownload(item.ToSuccess(downloadedBytes));
if (response.Content.Headers.ContentLength is {} contentLength) {
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
await db.Downloads.AddDownload(item.ToSuccess(contentLength), stream);
}
else {
await db.Downloads.AddDownload(item.ToFailure(), stream: null);
log.Error("Download response has no content length: " + item.DownloadUrl);
}
} catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) { } catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) {
// Ignore. // Ignore.
} catch (TaskCanceledException e) when (e.InnerException is TimeoutException) { } catch (TaskCanceledException e) {
await db.Downloads.AddDownload(item.ToFailure(), stream: null); // HttpClient request timed out.
log.Error("Download timed out: " + item.DownloadUrl); await db.Downloads.AddDownload(item.ToFailure());
log.Error(e.Message);
} catch (HttpRequestException e) { } catch (HttpRequestException e) {
await db.Downloads.AddDownload(item.ToFailure(e.StatusCode), stream: null); await db.Downloads.AddDownload(item.ToFailure(e.StatusCode));
log.Error(e); log.Error(e);
} catch (Exception e) { } catch (Exception e) {
await db.Downloads.AddDownload(item.ToFailure(), stream: null); await db.Downloads.AddDownload(item.ToFailure());
log.Error(e); log.Error(e);
} finally { } finally {
try { try {

View File

@ -9,37 +9,37 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
abstract class BaseEndpoint(IDatabaseFile db) { abstract class BaseEndpoint {
private static readonly Log Log = Log.ForType<BaseEndpoint>(); private static readonly Log Log = Log.ForType<BaseEndpoint>();
protected IDatabaseFile Db { get; } = db; protected IDatabaseFile Db { get; }
protected BaseEndpoint(IDatabaseFile db) {
this.Db = db;
}
public async Task Handle(HttpContext ctx) { public async Task Handle(HttpContext ctx) {
var response = ctx.Response; var response = ctx.Response;
try { try {
response.StatusCode = (int) HttpStatusCode.OK; response.StatusCode = (int) HttpStatusCode.OK;
await Respond(ctx.Request, response); var output = await Respond(ctx);
await output.WriteTo(response);
} catch (HttpException e) { } catch (HttpException e) {
Log.Error(e); Log.Error(e);
response.StatusCode = (int) e.StatusCode; response.StatusCode = (int) e.StatusCode;
if (response.HasStarted) { await response.WriteAsync(e.Message);
Log.Warn("Response has already started, cannot write status message: " + e.Message);
}
else {
await response.WriteAsync(e.Message);
}
} catch (Exception e) { } catch (Exception e) {
Log.Error(e); Log.Error(e);
response.StatusCode = (int) HttpStatusCode.InternalServerError; response.StatusCode = (int) HttpStatusCode.InternalServerError;
} }
} }
protected abstract Task Respond(HttpRequest request, HttpResponse response); protected abstract Task<IHttpOutput> Respond(HttpContext ctx);
protected static async Task<JsonElement> ReadJson(HttpRequest request) { protected static async Task<JsonElement> ReadJson(HttpContext ctx) {
try { try {
return await request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement); return await ctx.Request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement);
} catch (JsonException) { } catch (JsonException) {
throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON."); throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON.");
} }

View File

@ -7,13 +7,18 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
sealed class GetDownloadedFileEndpoint(IDatabaseFile db) : BaseEndpoint(db) { sealed class GetDownloadedFileEndpoint : BaseEndpoint {
protected override async Task Respond(HttpRequest request, HttpResponse response) { public GetDownloadedFileEndpoint(IDatabaseFile db) : base(db) {}
string url = WebUtility.UrlDecode((string) request.RouteValues["url"]!);
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
string url = WebUtility.UrlDecode((string) ctx.Request.RouteValues["url"]!);
string normalizedUrl = DiscordCdn.NormalizeUrl(url); string normalizedUrl = DiscordCdn.NormalizeUrl(url);
if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, (download, stream) => response.WriteStreamAsync(download.Type, download.Size, stream))) { if (await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl) is { Download: {} download, Data: {} data }) {
response.Redirect(url, permanent: false); return new HttpOutput.File(download.Type, data);
}
else {
return new HttpOutput.Redirect(url, permanent: false);
} }
} }
} }

View File

@ -1,5 +1,5 @@
using System.Net.Mime;
using System.Reflection; using System.Reflection;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Web; using System.Web;
using DHT.Server.Database; using DHT.Server.Database;
@ -10,19 +10,25 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : BaseEndpoint(db) { sealed class GetTrackingScriptEndpoint : BaseEndpoint {
private static ResourceLoader Resources { get; } = new (Assembly.GetExecutingAssembly()); private static ResourceLoader Resources { get; } = new (Assembly.GetExecutingAssembly());
protected override async Task Respond(HttpRequest request, HttpResponse response) { private readonly ServerParameters serverParameters;
public GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : base(db) {
serverParameters = parameters;
}
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
string bootstrap = await Resources.ReadTextAsync("Tracker/bootstrap.js"); string bootstrap = await Resources.ReadTextAsync("Tracker/bootstrap.js");
string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + parameters.Port + ";") string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + serverParameters.Port + ";")
.Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(parameters.Token)) .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(serverParameters.Token))
.Replace("/*[IMPORTS]*/", await Resources.ReadJoinedAsync("Tracker/scripts/", '\n')) .Replace("/*[IMPORTS]*/", await Resources.ReadJoinedAsync("Tracker/scripts/", '\n'))
.Replace("/*[CSS-CONTROLLER]*/", await Resources.ReadTextAsync("Tracker/styles/controller.css")) .Replace("/*[CSS-CONTROLLER]*/", await Resources.ReadTextAsync("Tracker/styles/controller.css"))
.Replace("/*[CSS-SETTINGS]*/", await Resources.ReadTextAsync("Tracker/styles/settings.css")) .Replace("/*[CSS-SETTINGS]*/", await Resources.ReadTextAsync("Tracker/styles/settings.css"))
.Replace("/*[DEBUGGER]*/", request.Query.ContainsKey("debug") ? "debugger;" : ""); .Replace("/*[DEBUGGER]*/", ctx.Request.Query.ContainsKey("debug") ? "debugger;" : "");
response.Headers.Append("X-DHT", "1"); ctx.Response.Headers.Append("X-DHT", "1");
await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script); return new HttpOutput.File("text/javascript", Encoding.UTF8.GetBytes(script));
} }
} }

View File

@ -8,14 +8,18 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint(db) { sealed class TrackChannelEndpoint : BaseEndpoint {
protected override async Task Respond(HttpRequest request, HttpResponse response) { public TrackChannelEndpoint(IDatabaseFile db) : base(db) {}
var root = await ReadJson(request);
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
var server = ReadServer(root.RequireObject("server"), "server"); var server = ReadServer(root.RequireObject("server"), "server");
var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id); var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id);
await Db.Servers.Add([server]); await Db.Servers.Add([server]);
await Db.Channels.Add([channel]); await Db.Channels.Add([channel]);
return HttpOutput.None;
} }
private static Data.Server ReadServer(JsonElement json, string path) => new () { private static Data.Server ReadServer(JsonElement json, string path) => new () {

View File

@ -15,12 +15,14 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) { sealed class TrackMessagesEndpoint : BaseEndpoint {
private const string HasNewMessages = "1"; private const string HasNewMessages = "1";
private const string NoNewMessages = "0"; private const string NoNewMessages = "0";
protected override async Task Respond(HttpRequest request, HttpResponse response) { public TrackMessagesEndpoint(IDatabaseFile db) : base(db) {}
var root = await ReadJson(request);
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
if (root.ValueKind != JsonValueKind.Array) { if (root.ValueKind != JsonValueKind.Array) {
throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array."); throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array.");
@ -41,7 +43,7 @@ sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
await Db.Messages.Add(messages); await Db.Messages.Add(messages);
await response.WriteTextAsync(anyNewMessages ? HasNewMessages : NoNewMessages); return new HttpOutput.Text(anyNewMessages ? HasNewMessages : NoNewMessages);
} }
private static Message ReadMessage(JsonElement json, string path) => new () { private static Message ReadMessage(JsonElement json, string path) => new () {

View File

@ -8,9 +8,11 @@ using Microsoft.AspNetCore.Http;
namespace DHT.Server.Endpoints; namespace DHT.Server.Endpoints;
sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) { sealed class TrackUsersEndpoint : BaseEndpoint {
protected override async Task Respond(HttpRequest request, HttpResponse response) { public TrackUsersEndpoint(IDatabaseFile db) : base(db) {}
var root = await ReadJson(request);
protected override async Task<IHttpOutput> Respond(HttpContext ctx) {
var root = await ReadJson(ctx);
if (root.ValueKind != JsonValueKind.Array) { if (root.ValueKind != JsonValueKind.Array) {
throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array."); throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array.");
@ -24,6 +26,8 @@ sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
} }
await Db.Users.Add(users); await Db.Users.Add(users);
return HttpOutput.None;
} }
private static User ReadUser(JsonElement json, string path) => new () { private static User ReadUser(JsonElement json, string path) => new () {

View File

@ -8,7 +8,7 @@ public static class LinqExtensions {
HashSet<TKey>? seenKeys = null; HashSet<TKey>? seenKeys = null;
foreach (var item in collection) { foreach (var item in collection) {
seenKeys ??= []; seenKeys ??= new HashSet<TKey>();
if (seenKeys.Add(getKeyFromItem(item))) { if (seenKeys.Add(getKeyFromItem(item))) {
yield return item; yield return item;

View File

@ -1,33 +0,0 @@
using System.IO;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public static class HttpExtensions {
public static Task WriteTextAsync(this HttpResponse response, string text) {
return WriteTextAsync(response, MediaTypeNames.Text.Plain, text);
}
public static async Task WriteTextAsync(this HttpResponse response, string contentType, string text) {
response.ContentType = contentType;
await response.StartAsync();
await response.WriteAsync(text, Encoding.UTF8);
}
public static async Task WriteFileAsync(this HttpResponse response, string? contentType, byte[] bytes) {
response.ContentType = contentType ?? string.Empty;
response.ContentLength = bytes.Length;
await response.StartAsync();
await response.Body.WriteAsync(bytes);
}
public static async Task WriteStreamAsync(this HttpResponse response, string? contentType, ulong? contentLength, Stream source) {
response.ContentType = contentType ?? string.Empty;
response.ContentLength = (long?) contentLength;
await response.StartAsync();
await source.CopyToAsync(response.Body);
}
}

View File

@ -0,0 +1,35 @@
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public static class HttpOutput {
public static IHttpOutput None { get; } = new NoneImpl();
private sealed class NoneImpl : IHttpOutput {
public Task WriteTo(HttpResponse response) {
return Task.CompletedTask;
}
}
public sealed class Text(string text) : IHttpOutput {
public Task WriteTo(HttpResponse response) {
return response.WriteAsync(text, Encoding.UTF8);
}
}
public sealed class File(string? contentType, byte[] bytes) : IHttpOutput {
public async Task WriteTo(HttpResponse response) {
response.ContentType = contentType ?? string.Empty;
await response.Body.WriteAsync(bytes);
}
}
public sealed class Redirect(string url, bool permanent) : IHttpOutput {
public Task WriteTo(HttpResponse response) {
response.Redirect(url, permanent);
return Task.CompletedTask;
}
}
}

View File

@ -0,0 +1,8 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace DHT.Utils.Http;
public interface IHttpOutput {
Task WriteTo(HttpResponse response);
}

View File

@ -8,5 +8,5 @@ using DHT.Utils;
namespace DHT.Utils; namespace DHT.Utils;
static class Version { static class Version {
public const string Tag = "41.2.0.0"; public const string Tag = "41.1.0.0";
} }