1
0
mirror of https://github.com/chylex/Discord-History-Tracker.git synced 2025-07-04 10:38:52 +02:00

Compare commits

..

No commits in common. "312be6609d3e7d7f7055c723a309a52825f20cd3" and "0d3600492ed5cccad479ee36beeff2c540f217dc" have entirely different histories.

24 changed files with 99 additions and 159 deletions

View File

@ -74,7 +74,7 @@ sealed class DebugPageModel {
Id = RandomId(rand), Id = RandomId(rand),
Name = RandomName("u"), Name = RandomName("u"),
DisplayName = RandomName("u"), DisplayName = RandomName("u"),
AvatarHash = null, AvatarUrl = null,
Discriminator = rand.Next(minValue: 0, maxValue: 9999).ToString(), Discriminator = rand.Next(minValue: 0, maxValue: 9999).ToString(),
}).ToArray(); }).ToArray();

View File

@ -180,8 +180,8 @@ sealed partial class DownloadsPageModel : IAsyncDisposable {
HashSet<string> reachableNormalizedUrls = []; HashSet<string> reachableNormalizedUrls = [];
HashSet<string> orphanedNormalizedUrls = []; HashSet<string> orphanedNormalizedUrls = [];
await foreach (FileUrl fileUrl in state.Db.Downloads.FindReachableFiles()) { await foreach (Download download in state.Db.Downloads.FindAllDownloadableUrls()) {
reachableNormalizedUrls.Add(fileUrl.NormalizedUrl); reachableNormalizedUrls.Add(download.NormalizedUrl);
} }
await foreach (Download download in state.Db.Downloads.Get()) { await foreach (Download download in state.Db.Downloads.Get()) {

View File

@ -142,19 +142,11 @@ const STATE = (function() {
if (DISCORD.CHANNEL_TYPE.isPrivate(channelInfo.type)) { if (DISCORD.CHANNEL_TYPE.isPrivate(channelInfo.type)) {
server.id = channelInfo.id; server.id = channelInfo.id;
server.name = channel.name = getPrivateChannelName(channelInfo); server.name = channel.name = getPrivateChannelName(channelInfo);
if (channelInfo.icon) {
server.icon = channelInfo.icon;
}
} }
else if (serverInfo) { else if (serverInfo) {
server.id = serverInfo.id; server.id = serverInfo.id;
server.name = serverInfo.name; server.name = serverInfo.name;
channel.name = channelInfo.name; channel.name = channelInfo.name;
if (serverInfo.icon) {
server.icon = serverInfo.icon;
}
} }
else { else {
return; return;

View File

@ -2,7 +2,6 @@
* @name DiscordGuild * @name DiscordGuild
* @property {String} id * @property {String} id
* @property {String} name * @property {String} name
* @property {String|null|undefined} [icon]
*/ */
/** /**
@ -15,7 +14,6 @@
* @property {Number} [position] * @property {Number} [position]
* @property {String} [topic] * @property {String} [topic]
* @property {Boolean} [nsfw] * @property {Boolean} [nsfw]
* @property {String|null|undefined} [icon]
* @property {DiscordUser[]} [rawRecipients] * @property {DiscordUser[]} [rawRecipients]
*/ */

View File

@ -1,9 +0,0 @@
namespace DHT.Server.Data;
public readonly record struct FileUrl(string NormalizedUrl, string DownloadUrl, string? Type) {
public FileUrl(string url, string? type) : this(url, url, type) {}
public Download ToPendingDownload() {
return new Download(NormalizedUrl, DownloadUrl, DownloadStatus.Pending, Type, size: null);
}
}

View File

@ -1,12 +1,7 @@
using DHT.Server.Download;
namespace DHT.Server.Data; namespace DHT.Server.Data;
public readonly struct Server { public readonly struct Server {
public ulong Id { get; init; } public ulong Id { get; init; }
public string Name { get; init; } public string Name { get; init; }
public ServerType? Type { get; init; } public ServerType? Type { get; init; }
public string? IconHash { get; init; }
internal FileUrl? IconUrl => Type == null || IconHash == null ? null : DownloadLinkExtractor.ServerIcon(Type.Value, Id, IconHash);
} }

View File

@ -1,13 +1,9 @@
using DHT.Server.Download;
namespace DHT.Server.Data; namespace DHT.Server.Data;
public readonly struct User { public readonly struct User {
public ulong Id { get; init; } public ulong Id { get; init; }
public string Name { get; init; } public string Name { get; init; }
public string? DisplayName { get; init; } public string? DisplayName { get; init; }
public string? AvatarHash { get; init; } public string? AvatarUrl { get; init; }
public string? Discriminator { get; init; } public string? Discriminator { get; init; }
internal FileUrl? AvatarUrl => AvatarHash == null ? null : DownloadLinkExtractor.UserAvatar(Id, AvatarHash);
} }

View File

@ -23,9 +23,6 @@ static class ViewerJson {
public sealed class JsonServer { public sealed class JsonServer {
public required string Name { get; init; } public required string Name { get; init; }
public required string Type { get; init; } public required string Type { get; init; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? IconUrl { get; init; }
} }
public sealed class JsonChannel { public sealed class JsonChannel {

View File

@ -90,7 +90,7 @@ static class ViewerJsonExport {
users[user.Id] = new ViewerJson.JsonUser { users[user.Id] = new ViewerJson.JsonUser {
Name = user.Name, Name = user.Name,
DisplayName = user.DisplayName, DisplayName = user.DisplayName,
Avatar = user.AvatarHash, Avatar = user.AvatarUrl,
}; };
} }
@ -108,7 +108,6 @@ static class ViewerJsonExport {
servers[server.Id] = new ViewerJson.JsonServer { servers[server.Id] = new ViewerJson.JsonServer {
Name = server.Name, Name = server.Name,
Type = ServerTypes.ToJsonViewerString(server.Type), Type = ServerTypes.ToJsonViewerString(server.Type),
IconUrl = server.IconUrl?.DownloadUrl,
}; };
} }

View File

@ -106,7 +106,7 @@ public static class LegacyArchiveImport {
users[userindex[userId]] = new User { users[userindex[userId]] = new User {
Id = userId, Id = userId,
Name = userObj.RequireString("name", path), Name = userObj.RequireString("name", path),
AvatarHash = userObj.HasKey("avatar") ? userObj.RequireString("avatar", path) : null, AvatarUrl = userObj.HasKey("avatar") ? userObj.RequireString("avatar", path) : null,
Discriminator = userObj.HasKey("tag") ? userObj.RequireString("tag", path) : null, Discriminator = userObj.HasKey("tag") ? userObj.RequireString("tag", path) : null,
}; };
} }

View File

@ -5,7 +5,6 @@ using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DHT.Server.Data;
using DHT.Server.Data.Aggregations; using DHT.Server.Data.Aggregations;
using DHT.Server.Data.Filters; using DHT.Server.Data.Filters;
using DHT.Server.Download; using DHT.Server.Download;
@ -35,7 +34,7 @@ public interface IDownloadRepository {
Task Remove(ICollection<string> normalizedUrls); Task Remove(ICollection<string> normalizedUrls);
IAsyncEnumerable<FileUrl> FindReachableFiles(CancellationToken cancellationToken = default); IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken = default);
internal sealed class Dummy : IDownloadRepository { internal sealed class Dummy : IDownloadRepository {
public IObservable<long> TotalCount { get; } = Observable.Return(0L); public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@ -80,8 +79,8 @@ public interface IDownloadRepository {
return Task.CompletedTask; return Task.CompletedTask;
} }
public IAsyncEnumerable<FileUrl> FindReachableFiles(CancellationToken cancellationToken) { public IAsyncEnumerable<Data.Download> FindAllDownloadableUrls(CancellationToken cancellationToken) {
return AsyncEnumerable.Empty<FileUrl>(); return AsyncEnumerable.Empty<Data.Download>();
} }
} }
} }

View File

@ -62,15 +62,6 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
hasChanged |= await metadataCmd.ExecuteNonQueryAsync() > 0; hasChanged |= await metadataCmd.ExecuteNonQueryAsync() > 0;
} }
public Task AddIfNotNull(Data.Download? download) {
if (download != null) {
return Add(download);
}
else {
return Task.CompletedTask;
}
}
public void OnCommitted() { public void OnCommitted() {
if (hasChanged) { if (hasChanged) {
repository.UpdateTotalCount(); repository.UpdateTotalCount();
@ -370,17 +361,14 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
UpdateTotalCount(); UpdateTotalCount();
} }
public async IAsyncEnumerable<FileUrl> FindReachableFiles([EnumeratorCancellation] CancellationToken cancellationToken) { public async IAsyncEnumerable<Data.Download> FindAllDownloadableUrls([EnumeratorCancellation] CancellationToken cancellationToken = default) {
await using var conn = await pool.Take(); await using var conn = await pool.Take();
await using (var cmd = conn.Command("SELECT type, normalized_url, download_url FROM attachments")) { await using (var cmd = conn.Command("SELECT normalized_url, download_url, type, size FROM attachments")) {
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) {
string? type = reader.IsDBNull(0) ? null : reader.GetString(0); yield return DownloadLinkExtractor.FromAttachment(reader.GetString(0), reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetUint64(3));
string normalizedUrl = reader.GetString(1);
string downloadUrl = reader.GetString(2);
yield return new FileUrl(normalizedUrl, downloadUrl, type);
} }
} }
@ -388,31 +376,8 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) {
if (await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)) is {} result) { var result = await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0));
yield return result; if (result is not null) {
}
}
}
await using (var cmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM message_reactions WHERE emoji_id IS NOT NULL")) {
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) {
ulong emojiId = reader.GetUint64(0);
EmojiFlags emojiFlags = (EmojiFlags) reader.GetInt16(1);
yield return DownloadLinkExtractor.Emoji(emojiId, emojiFlags);
}
}
await using (var cmd = conn.Command("SELECT id, type, icon_hash FROM servers WHERE icon_hash IS NOT NULL")) {
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) {
ulong id = reader.GetUint64(0);
ServerType? type = ServerTypes.FromString(reader.GetString(1));
string iconHash = reader.GetString(2);
if (DownloadLinkExtractor.ServerIcon(type, id, iconHash) is {} result) {
yield return result; yield return result;
} }
} }
@ -422,9 +387,15 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) {
ulong id = reader.GetUint64(0); yield return DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1));
string avatarHash = reader.GetString(1); }
yield return DownloadLinkExtractor.UserAvatar(id, avatarHash); }
await using (var cmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM message_reactions WHERE emoji_id IS NOT NULL")) {
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) {
yield return DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1));
} }
} }
} }

View File

@ -10,7 +10,6 @@ using DHT.Server.Database.Repositories;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Server.Download; using DHT.Server.Download;
using DHT.Utils.Logging; using DHT.Utils.Logging;
using JetBrains.Annotations;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories; namespace DHT.Server.Database.Sqlite.Repositories;
@ -18,16 +17,15 @@ namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IMessageRepository { sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IMessageRepository {
private static readonly Log Log = Log.ForType<SqliteMessageRepository>(); private static readonly Log Log = Log.ForType<SqliteMessageRepository>();
// Moved outside the Add method due to language injections not working in local methods.
private static SqliteCommand DeleteByMessageId(ISqliteConnection conn, [LanguageInjection("sql", Prefix = "SELECT * FROM ")] string tableName) {
return conn.Delete(tableName, ("message_id", SqliteType.Integer));
}
public async Task Add(IReadOnlyList<Message> messages) { public async Task Add(IReadOnlyList<Message> messages) {
if (messages.Count == 0) { if (messages.Count == 0) {
return; return;
} }
static SqliteCommand DeleteByMessageId(ISqliteConnection conn, string tableName) {
return conn.Delete(tableName, ("message_id", SqliteType.Integer));
}
static async Task ExecuteDeleteByMessageId(SqliteCommand cmd, object id) { static async Task ExecuteDeleteByMessageId(SqliteCommand cmd, object id) {
cmd.Set(":message_id", id); cmd.Set(":message_id", id);
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
@ -139,7 +137,7 @@ sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRe
messageAttachmentCmd.Set(":attachment_id", attachmentId); messageAttachmentCmd.Set(":attachment_id", attachmentId);
await messageAttachmentCmd.ExecuteNonQueryAsync(); await messageAttachmentCmd.ExecuteNonQueryAsync();
await downloadCollector.Add(new Data.Download(attachment.NormalizedUrl, attachment.DownloadUrl, DownloadStatus.Pending, attachment.Type, attachment.Size)); await downloadCollector.Add(DownloadLinkExtractor.FromAttachment(attachment));
} }
} }
@ -150,7 +148,7 @@ sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRe
await messageEmbedCmd.ExecuteNonQueryAsync(); await messageEmbedCmd.ExecuteNonQueryAsync();
if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) { if (DownloadLinkExtractor.TryFromEmbedJson(embed.Json) is {} download) {
await downloadCollector.Add(download.ToPendingDownload()); await downloadCollector.Add(download);
} }
} }
} }
@ -165,7 +163,7 @@ sealed class SqliteMessageRepository(SqliteConnectionPool pool, SqliteDownloadRe
await messageReactionCmd.ExecuteNonQueryAsync(); await messageReactionCmd.ExecuteNonQueryAsync();
if (reaction.EmojiId is {} emojiId) { if (reaction.EmojiId is {} emojiId) {
await downloadCollector.Add(DownloadLinkExtractor.Emoji(emojiId, reaction.EmojiFlags).ToPendingDownload()); await downloadCollector.Add(DownloadLinkExtractor.FromEmoji(emojiId, reaction.EmojiFlags));
} }
} }
} }

View File

@ -10,9 +10,15 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories; namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteServerRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IServerRepository { sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
private static readonly Log Log = Log.ForType<SqliteServerRepository>(); private static readonly Log Log = Log.ForType<SqliteServerRepository>();
private readonly SqliteConnectionPool pool;
public SqliteServerRepository(SqliteConnectionPool pool) : base(Log) {
this.pool = pool;
}
public async Task Add(IReadOnlyList<Data.Server> servers) { public async Task Add(IReadOnlyList<Data.Server> servers) {
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); await conn.BeginTransactionAsync();
@ -21,18 +27,13 @@ sealed class SqliteServerRepository(SqliteConnectionPool pool, SqliteDownloadRep
("id", SqliteType.Integer), ("id", SqliteType.Integer),
("name", SqliteType.Text), ("name", SqliteType.Text),
("type", SqliteType.Text), ("type", SqliteType.Text),
("icon_hash", SqliteType.Text),
]); ]);
await using var downloadCollector = new SqliteDownloadRepository.NewDownloadCollector(downloads, conn);
foreach (Data.Server server in servers) { foreach (Data.Server server in servers) {
cmd.Set(":id", server.Id); cmd.Set(":id", server.Id);
cmd.Set(":name", server.Name); cmd.Set(":name", server.Name);
cmd.Set(":type", ServerTypes.ToString(server.Type)); cmd.Set(":type", ServerTypes.ToString(server.Type));
cmd.Set(":icon_hash", server.IconHash);
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
await downloadCollector.AddIfNotNull(server.IconUrl?.ToPendingDownload());
} }
await conn.CommitTransactionAsync(); await conn.CommitTransactionAsync();
@ -49,7 +50,7 @@ sealed class SqliteServerRepository(SqliteConnectionPool pool, SqliteDownloadRep
public async IAsyncEnumerable<Data.Server> Get([EnumeratorCancellation] CancellationToken cancellationToken) { public async IAsyncEnumerable<Data.Server> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
await using var conn = await pool.Take(); await using var conn = await pool.Take();
await using var cmd = conn.Command("SELECT id, name, type, icon_hash FROM servers"); await using var cmd = conn.Command("SELECT id, name, type FROM servers");
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) {
@ -57,7 +58,6 @@ sealed class SqliteServerRepository(SqliteConnectionPool pool, SqliteDownloadRep
Id = reader.GetUint64(0), Id = reader.GetUint64(0),
Name = reader.GetString(1), Name = reader.GetString(1),
Type = ServerTypes.FromString(reader.GetString(2)), Type = ServerTypes.FromString(reader.GetString(2)),
IconHash = reader.IsDBNull(3) ? null : reader.GetString(3),
}; };
} }
} }

View File

@ -5,14 +5,23 @@ using System.Threading.Tasks;
using DHT.Server.Data; using DHT.Server.Data;
using DHT.Server.Database.Repositories; using DHT.Server.Database.Repositories;
using DHT.Server.Database.Sqlite.Utils; using DHT.Server.Database.Sqlite.Utils;
using DHT.Server.Download;
using DHT.Utils.Logging; using DHT.Utils.Logging;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite.Repositories; namespace DHT.Server.Database.Sqlite.Repositories;
sealed class SqliteUserRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : BaseSqliteRepository(Log), IUserRepository { sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
private static readonly Log Log = Log.ForType<SqliteUserRepository>(); private static readonly Log Log = Log.ForType<SqliteUserRepository>();
private readonly SqliteConnectionPool pool;
private readonly SqliteDownloadRepository downloads;
public SqliteUserRepository(SqliteConnectionPool pool, SqliteDownloadRepository downloads) : base(Log) {
this.pool = pool;
this.downloads = downloads;
}
public async Task Add(IReadOnlyList<User> users) { public async Task Add(IReadOnlyList<User> users) {
await using (var conn = await pool.Take()) { await using (var conn = await pool.Take()) {
await conn.BeginTransactionAsync(); await conn.BeginTransactionAsync();
@ -31,10 +40,13 @@ sealed class SqliteUserRepository(SqliteConnectionPool pool, SqliteDownloadRepos
cmd.Set(":id", user.Id); cmd.Set(":id", user.Id);
cmd.Set(":name", user.Name); cmd.Set(":name", user.Name);
cmd.Set(":display_name", user.DisplayName); cmd.Set(":display_name", user.DisplayName);
cmd.Set(":avatar_url", user.AvatarHash); cmd.Set(":avatar_url", user.AvatarUrl);
cmd.Set(":discriminator", user.Discriminator); cmd.Set(":discriminator", user.Discriminator);
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
await downloadCollector.AddIfNotNull(user.AvatarUrl?.ToPendingDownload());
if (user.AvatarUrl is {} avatarUrl) {
await downloadCollector.Add(DownloadLinkExtractor.FromUserAvatar(user.Id, avatarUrl));
}
} }
await conn.CommitTransactionAsync(); await conn.CommitTransactionAsync();
@ -60,7 +72,7 @@ sealed class SqliteUserRepository(SqliteConnectionPool pool, SqliteDownloadRepos
Id = reader.GetUint64(0), Id = reader.GetUint64(0),
Name = reader.GetString(1), Name = reader.GetString(1),
DisplayName = reader.IsDBNull(2) ? null : reader.GetString(2), DisplayName = reader.IsDBNull(2) ? null : reader.GetString(2),
AvatarHash = reader.IsDBNull(3) ? null : reader.GetString(3), AvatarUrl = reader.IsDBNull(3) ? null : reader.GetString(3),
Discriminator = reader.IsDBNull(4) ? null : reader.GetString(4), Discriminator = reader.IsDBNull(4) ? null : reader.GetString(4),
}; };
} }

View File

@ -1,11 +0,0 @@
using System.Threading.Tasks;
using DHT.Server.Database.Sqlite.Utils;
namespace DHT.Server.Database.Sqlite.Schema;
sealed class SqliteSchemaUpgradeTo11 : ISchemaUpgrade {
async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) {
await reporter.MainWork("Applying schema changes...", finishedItems: 0, totalItems: 1);
await conn.ExecuteAsync("ALTER TABLE servers ADD icon_hash TEXT");
}
}

View File

@ -124,7 +124,7 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await using var reader = await embedCmd.ExecuteReaderAsync(); await using var reader = await embedCmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) { while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, (await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)))?.ToPendingDownload()); await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0)));
} }
} }
@ -134,7 +134,7 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await using var reader = await avatarCmd.ExecuteReaderAsync(); await using var reader = await avatarCmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) { while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, DownloadLinkExtractor.UserAvatar(reader.GetUint64(0), reader.GetString(1)).ToPendingDownload()); await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1)));
} }
} }
@ -144,7 +144,7 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade {
await using var reader = await avatarCmd.ExecuteReaderAsync(); await using var reader = await avatarCmd.ExecuteReaderAsync();
while (await reader.ReadAsync()) { while (await reader.ReadAsync()) {
await InsertDownload(insertCmd, DownloadLinkExtractor.Emoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1)).ToPendingDownload()); await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1)));
} }
} }

View File

@ -66,16 +66,16 @@ public sealed class SqliteDatabaseFile : IDatabaseFile {
downloads = new SqliteDownloadRepository(pool); downloads = new SqliteDownloadRepository(pool);
settings = new SqliteSettingsRepository(pool); settings = new SqliteSettingsRepository(pool);
users = new SqliteUserRepository(pool, downloads); users = new SqliteUserRepository(pool, downloads);
servers = new SqliteServerRepository(pool, downloads); servers = new SqliteServerRepository(pool);
channels = new SqliteChannelRepository(pool); channels = new SqliteChannelRepository(pool);
messages = new SqliteMessageRepository(pool, downloads); messages = new SqliteMessageRepository(pool, downloads);
} }
public async ValueTask DisposeAsync() { public async ValueTask DisposeAsync() {
messages.Dispose();
channels.Dispose();
servers.Dispose();
users.Dispose(); users.Dispose();
servers.Dispose();
channels.Dispose();
messages.Dispose();
downloads.Dispose(); downloads.Dispose();
await pool.DisposeAsync(); await pool.DisposeAsync();
} }

View File

@ -13,7 +13,7 @@ using Microsoft.Data.Sqlite;
namespace DHT.Server.Database.Sqlite; namespace DHT.Server.Database.Sqlite;
sealed class SqliteSchema(CustomSqliteConnection conn) { sealed class SqliteSchema(CustomSqliteConnection conn) {
internal const int Version = 11; internal const int Version = 10;
private static readonly Log Log = Log.ForType<SqliteSchema>(); private static readonly Log Log = Log.ForType<SqliteSchema>();
@ -92,10 +92,9 @@ sealed class SqliteSchema(CustomSqliteConnection conn) {
await conn.ExecuteAsync(""" await conn.ExecuteAsync("""
CREATE TABLE servers ( CREATE TABLE servers (
id INTEGER PRIMARY KEY NOT NULL, id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
type TEXT NOT NULL, type TEXT NOT NULL
icon_hash TEXT
) )
"""); """);
@ -223,7 +222,6 @@ sealed class SqliteSchema(CustomSqliteConnection conn) {
{ 7, new SqliteSchemaUpgradeTo8() }, { 7, new SqliteSchemaUpgradeTo8() },
{ 8, new SqliteSchemaUpgradeTo9() }, { 8, new SqliteSchemaUpgradeTo9() },
{ 9, new SqliteSchemaUpgradeTo10() }, { 9, new SqliteSchemaUpgradeTo10() },
{ 10, new SqliteSchemaUpgradeTo11() },
}; };
Perf perf = Log.Start("from version " + dbVersion); Perf perf = Log.Start("from version " + dbVersion);

View File

@ -30,7 +30,7 @@ static class SqliteExtensions {
return (long) (await command.ExecuteScalarAsync())!; return (long) (await command.ExecuteScalarAsync())!;
} }
public static SqliteCommand Insert(this ISqliteConnection conn, [LanguageInjection("sql", Prefix = "SELECT * FROM ")] string tableName, (string Name, SqliteType Type)[] columns) { public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
string columnNames = string.Join(separator: ',', columns.Select(static c => c.Name)); string columnNames = string.Join(separator: ',', columns.Select(static c => c.Name));
string columnParams = string.Join(separator: ',', columns.Select(static c => ':' + c.Name)); string columnParams = string.Join(separator: ',', columns.Select(static c => ':' + c.Name));
@ -41,7 +41,7 @@ static class SqliteExtensions {
return cmd; return cmd;
} }
public static SqliteCommand Upsert(this ISqliteConnection conn, [LanguageInjection("sql", Prefix = "SELECT * FROM ")] string tableName, (string Name, SqliteType Type)[] columns) { public static SqliteCommand Upsert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) {
string columnNames = string.Join(separator: ',', columns.Select(static c => c.Name)); string columnNames = string.Join(separator: ',', columns.Select(static c => c.Name));
string columnParams = string.Join(separator: ',', columns.Select(static c => ':' + c.Name)); string columnParams = string.Join(separator: ',', columns.Select(static c => ':' + c.Name));
string columnUpdates = string.Join(separator: ',', columns.Skip(1).Select(static c => c.Name + " = excluded." + c.Name)); string columnUpdates = string.Join(separator: ',', columns.Skip(1).Select(static c => c.Name + " = excluded." + c.Name));
@ -55,7 +55,7 @@ static class SqliteExtensions {
return cmd; return cmd;
} }
public static SqliteCommand Delete(this ISqliteConnection conn, [LanguageInjection("sql", Prefix = "SELECT * FROM ")] string tableName, (string Name, SqliteType Type) column) { public static SqliteCommand Delete(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type) column) {
var cmd = conn.Command("DELETE FROM " + tableName + " WHERE " + column.Name + " = :" + column.Name); var cmd = conn.Command("DELETE FROM " + tableName + " WHERE " + column.Name + " = :" + column.Name);
CreateParameters(cmd, [column]); CreateParameters(cmd, [column]);
return cmd; return cmd;

View File

@ -12,28 +12,30 @@ namespace DHT.Server.Download;
static class DownloadLinkExtractor { static class DownloadLinkExtractor {
private static readonly Log Log = Log.ForType(typeof(DownloadLinkExtractor)); private static readonly Log Log = Log.ForType(typeof(DownloadLinkExtractor));
public static FileUrl? ServerIcon(ServerType? type, ulong id, string iconHash) { public static Data.Download FromUserAvatar(ulong userId, string avatarPath) {
return type switch { string url = $"https://cdn.discordapp.com/avatars/{userId}/{avatarPath}.webp";
ServerType.Server => new FileUrl($"https://cdn.discordapp.com/icons/{id}/{iconHash}.webp", MediaTypeNames.Image.Webp), return new Data.Download(url, url, DownloadStatus.Pending, MediaTypeNames.Image.Webp, size: null);
ServerType.Group => new FileUrl($"https://cdn.discordapp.com/channel-icons/{id}/{iconHash}.webp", MediaTypeNames.Image.Webp),
_ => null,
};
} }
public static FileUrl UserAvatar(ulong id, string avatarHash) { public static Data.Download FromEmoji(ulong emojiId, EmojiFlags flags) {
return new FileUrl($"https://cdn.discordapp.com/avatars/{id}/{avatarHash}.webp", MediaTypeNames.Image.Webp);
}
public static FileUrl Emoji(ulong emojiId, EmojiFlags flags) {
bool isAnimated = flags.HasFlag(EmojiFlags.Animated); bool isAnimated = flags.HasFlag(EmojiFlags.Animated);
string ext = isAnimated ? "gif" : "webp"; string ext = isAnimated ? "gif" : "webp";
string type = isAnimated ? MediaTypeNames.Image.Gif : MediaTypeNames.Image.Webp; string type = isAnimated ? MediaTypeNames.Image.Gif : MediaTypeNames.Image.Webp;
return new FileUrl($"https://cdn.discordapp.com/emojis/{emojiId}.{ext}", type); string url = $"https://cdn.discordapp.com/emojis/{emojiId}.{ext}";
return new Data.Download(url, url, DownloadStatus.Pending, type, size: null);
} }
public static async Task<FileUrl?> TryFromEmbedJson(Stream jsonStream) { public static Data.Download FromAttachment(Attachment attachment) {
return FromAttachment(attachment.NormalizedUrl, attachment.DownloadUrl, attachment.Type, attachment.Size);
}
public static Data.Download FromAttachment(string normalizedUrl, string downloadUrl, string? type, ulong size) {
return new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Pending, type, size);
}
public static async Task<Data.Download?> TryFromEmbedJson(Stream jsonStream) {
try { try {
return FromEmbed(await JsonSerializer.DeserializeAsync(jsonStream, DiscordEmbedJsonContext.Default.DiscordEmbedJson)); return FromEmbed(await JsonSerializer.DeserializeAsync(jsonStream, DiscordEmbedJsonContext.Default.DiscordEmbedJson));
} catch (Exception e) { } catch (Exception e) {
@ -42,7 +44,7 @@ static class DownloadLinkExtractor {
} }
} }
public static FileUrl? TryFromEmbedJson(string json) { public static Data.Download? TryFromEmbedJson(string json) {
try { try {
return FromEmbed(JsonSerializer.Deserialize(json, DiscordEmbedJsonContext.Default.DiscordEmbedJson)); return FromEmbed(JsonSerializer.Deserialize(json, DiscordEmbedJsonContext.Default.DiscordEmbedJson));
} catch (Exception e) { } catch (Exception e) {
@ -51,17 +53,21 @@ static class DownloadLinkExtractor {
} }
} }
private static FileUrl? FromEmbed(DiscordEmbedJson? embed) { private static Data.Download? FromEmbed(DiscordEmbedJson? embed) {
return embed switch { if (embed is { Type: "image", Image.Url: {} imageUrl }) {
{ Type: "image", Image.Url: {} imageUrl } => FromEmbedImage(imageUrl), return FromEmbedImage(imageUrl);
{ Type: "video", Video.Url: {} videoUrl } => FromEmbedVideo(videoUrl), }
_ => null, else if (embed is { Type: "video", Video.Url: {} videoUrl }) {
}; return FromEmbedVideo(videoUrl);
}
else {
return null;
}
} }
private static FileUrl? FromEmbedImage(string url) { private static Data.Download? FromEmbedImage(string url) {
if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out string normalizedUrl)) { if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out string normalizedUrl)) {
return new FileUrl(normalizedUrl, url, GuessImageType(normalizedUrl)); return new Data.Download(normalizedUrl, url, DownloadStatus.Pending, GuessImageType(normalizedUrl), size: null);
} }
else { else {
Log.Debug("Skipping non-CDN image url: " + url); Log.Debug("Skipping non-CDN image url: " + url);
@ -69,9 +75,9 @@ static class DownloadLinkExtractor {
} }
} }
private static FileUrl? FromEmbedVideo(string url) { private static Data.Download? FromEmbedVideo(string url) {
if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out string normalizedUrl)) { if (DiscordCdn.NormalizeUrlAndReturnIfCdn(url, out string normalizedUrl)) {
return new FileUrl(normalizedUrl, url, GuessVideoType(normalizedUrl)); return new Data.Download(normalizedUrl, url, DownloadStatus.Pending, GuessVideoType(normalizedUrl), size: null);
} }
else { else {
Log.Debug("Skipping non-CDN video url: " + url); Log.Debug("Skipping non-CDN video url: " + url);

View File

@ -24,7 +24,6 @@ sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint {
Id = json.RequireSnowflake("id", path), Id = json.RequireSnowflake("id", path),
Name = json.RequireString("name", path), Name = json.RequireString("name", path),
Type = ServerTypes.FromString(json.RequireString("type", path)) ?? throw new HttpException(HttpStatusCode.BadRequest, "Server type must be either 'SERVER', 'GROUP', or 'DM'."), Type = ServerTypes.FromString(json.RequireString("type", path)) ?? throw new HttpException(HttpStatusCode.BadRequest, "Server type must be either 'SERVER', 'GROUP', or 'DM'."),
IconHash = json.HasKey("icon") ? json.RequireString("icon", path) : null,
}; };
} }

View File

@ -32,7 +32,7 @@ sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint {
Id = json.RequireSnowflake("id", path), Id = json.RequireSnowflake("id", path),
Name = json.RequireString("name", path), Name = json.RequireString("name", path),
DisplayName = json.HasKey("displayName") ? json.RequireString("displayName", path) : null, DisplayName = json.HasKey("displayName") ? json.RequireString("displayName", path) : null,
AvatarHash = json.HasKey("avatar") ? json.RequireString("avatar", path) : null, AvatarUrl = json.HasKey("avatar") ? json.RequireString("avatar", path) : null,
Discriminator = json.HasKey("discriminator") ? json.RequireString("discriminator", path) : null, Discriminator = json.HasKey("discriminator") ? json.RequireString("discriminator", path) : null,
}; };
} }

Binary file not shown.