mirror of
				https://github.com/chylex/Discord-History-Tracker.git
				synced 2025-10-31 20:17:15 +01:00 
			
		
		
		
	Compare commits
	
		
			14 Commits
		
	
	
		
			v41.0
			...
			ebdabb3dd2
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| ebdabb3dd2 | |||
| 052d24d1e7 | |||
| b058f1fb77 | |||
| d4da64a5ed | |||
| 8de309a6c4 | |||
| de8d6a1e11 | |||
| d79e6f53b4 | |||
| 70c04fc986 | |||
| c8d8d95daa | |||
| daafdbbfaf | |||
| 07615de87a | |||
| 7fdc19880e | |||
| 67b9c12843 | |||
| 9030a2f010 | 
| @@ -13,10 +13,10 @@ | ||||
|     <CssCodeStyleSettings> | ||||
|       <option name="HEX_COLOR_LOWER_CASE" value="true" /> | ||||
|     </CssCodeStyleSettings> | ||||
|     <DB2CodeStyleSettings version="6"> | ||||
|     <DB2CodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </DB2CodeStyleSettings> | ||||
|     <DerbyCodeStyleSettings version="6"> | ||||
|     <DerbyCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </DerbyCodeStyleSettings> | ||||
|     <GoCodeStyleSettings> | ||||
| @@ -26,10 +26,10 @@ | ||||
|       <option name="WRAP_FUNC_PARAMS" value="5" /> | ||||
|       <option name="WRAP_FUNC_RESULT" value="5" /> | ||||
|     </GoCodeStyleSettings> | ||||
|     <H2CodeStyleSettings version="6"> | ||||
|     <H2CodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </H2CodeStyleSettings> | ||||
|     <HSQLCodeStyleSettings version="6"> | ||||
|     <HSQLCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </HSQLCodeStyleSettings> | ||||
|     <HTMLCodeStyleSettings> | ||||
| @@ -85,10 +85,10 @@ | ||||
|     <LessCodeStyleSettings> | ||||
|       <option name="HEX_COLOR_LOWER_CASE" value="true" /> | ||||
|     </LessCodeStyleSettings> | ||||
|     <MSSQLCodeStyleSettings version="6"> | ||||
|     <MSSQLCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </MSSQLCodeStyleSettings> | ||||
|     <MySQLCodeStyleSettings version="6"> | ||||
|     <MySQLCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </MySQLCodeStyleSettings> | ||||
|     <Objective-C> | ||||
| @@ -98,7 +98,7 @@ | ||||
|       <option name="SPACE_BEFORE_INIT_LIST" value="true" /> | ||||
|       <option name="SPACE_AFTER_DICTIONARY_LITERAL_COLON" value="false" /> | ||||
|     </Objective-C> | ||||
|     <OracleCodeStyleSettings version="6"> | ||||
|     <OracleCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </OracleCodeStyleSettings> | ||||
|     <PHPCodeStyleSettings> | ||||
| @@ -116,7 +116,7 @@ | ||||
|       <option name="NEW_LINE_AFTER_PHP_OPENING_TAG" value="true" /> | ||||
|       <option name="SPACE_AROUND_ASSIGNMENT_IN_DECLARE" value="true" /> | ||||
|     </PHPCodeStyleSettings> | ||||
|     <PostgresCodeStyleSettings version="6"> | ||||
|     <PostgresCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </PostgresCodeStyleSettings> | ||||
|     <Properties> | ||||
| @@ -143,13 +143,16 @@ | ||||
|       <option name="INDENT_WHEN_CASES" value="true" /> | ||||
|       <option name="CHAIN_CALLS_ALIGNMENT" value="2" /> | ||||
|     </Ruby> | ||||
|     <SQLiteCodeStyleSettings version="6"> | ||||
|     <SQLiteCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </SQLiteCodeStyleSettings> | ||||
|     <ScssCodeStyleSettings> | ||||
|       <option name="HEX_COLOR_LOWER_CASE" value="true" /> | ||||
|     </ScssCodeStyleSettings> | ||||
|     <SqlCodeStyleSettings version="6"> | ||||
|     <Shell_Script> | ||||
|       <option name="KEEP_COLUMN_ALIGNMENT_PADDING" value="true" /> | ||||
|     </Shell_Script> | ||||
|     <SqlCodeStyleSettings version="7"> | ||||
|       <option name="KEYWORD_CASE" value="2" /> | ||||
|       <option name="TYPE_CASE" value="2" /> | ||||
|       <option name="CUSTOM_TYPE_CASE" value="2" /> | ||||
| @@ -173,9 +176,10 @@ | ||||
|       <option name="CONSTRAINT_WRAP_3" value="true" /> | ||||
|       <option name="CONSTRAINT_WRAP_4" value="true" /> | ||||
|       <option name="VIEW_INDENT_QUERY" value="true" /> | ||||
|       <option name="IMP_LOOP_LOOP_INDENT" value="false" /> | ||||
|       <option name="EXPR_CASE_WHEN_WRAP" value="false" /> | ||||
|     </SqlCodeStyleSettings> | ||||
|     <SybaseCodeStyleSettings version="6"> | ||||
|     <SybaseCodeStyleSettings version="7"> | ||||
|       <option name="USE_GENERIC_STYLE" value="true" /> | ||||
|     </SybaseCodeStyleSettings> | ||||
|     <TypeScriptCodeStyleSettings version="0"> | ||||
| @@ -203,6 +207,11 @@ | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="Gherkin"> | ||||
|       <indentOptions> | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="Groovy"> | ||||
|       <option name="ELSE_ON_NEW_LINE" value="true" /> | ||||
|       <option name="SPACE_WITHIN_ARRAY_INITIALIZER_BRACES" value="true" /> | ||||
| @@ -282,6 +291,13 @@ | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="Markdown"> | ||||
|       <indentOptions> | ||||
|         <option name="INDENT_SIZE" value="2" /> | ||||
|         <option name="CONTINUATION_INDENT_SIZE" value="2" /> | ||||
|         <option name="TAB_SIZE" value="2" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="ObjectiveC"> | ||||
|       <option name="LINE_COMMENT_AT_FIRST_COLUMN" value="false" /> | ||||
|       <option name="BLOCK_COMMENT_AT_FIRST_COLUMN" value="false" /> | ||||
| @@ -342,7 +358,6 @@ | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="Rust"> | ||||
|       <option name="RIGHT_MARGIN" value="140" /> | ||||
|       <option name="ALIGN_MULTILINE_CHAINED_METHODS" value="true" /> | ||||
|       <indentOptions> | ||||
|         <option name="USE_TAB_CHARACTER" value="true" /> | ||||
| @@ -357,6 +372,7 @@ | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="SCSS"> | ||||
|       <indentOptions> | ||||
|         <option name="CONTINUATION_INDENT_SIZE" value="2" /> | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
| @@ -367,13 +383,6 @@ | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="Shell Script"> | ||||
|       <indentOptions> | ||||
|         <option name="INDENT_SIZE" value="4" /> | ||||
|         <option name="TAB_SIZE" value="4" /> | ||||
|         <option name="USE_TAB_CHARACTER" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="TOML"> | ||||
|       <indentOptions> | ||||
|         <option name="CONTINUATION_INDENT_SIZE" value="4" /> | ||||
| @@ -427,6 +436,9 @@ | ||||
|     </codeStyleSettings> | ||||
|     <codeStyleSettings language="liquid"> | ||||
|       <indentOptions> | ||||
|         <option name="INDENT_SIZE" value="2" /> | ||||
|         <option name="CONTINUATION_INDENT_SIZE" value="2" /> | ||||
|         <option name="TAB_SIZE" value="2" /> | ||||
|         <option name="KEEP_INDENTS_ON_EMPTY_LINES" value="true" /> | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
| @@ -442,4 +454,4 @@ | ||||
|       </indentOptions> | ||||
|     </codeStyleSettings> | ||||
|   </code_scheme> | ||||
| </component> | ||||
| </component> | ||||
| @@ -91,7 +91,6 @@ | ||||
|     <inspection_tool class="ComparatorNotSerializable" enabled="true" level="WARNING" enabled_by_default="true" /> | ||||
|     <inspection_tool class="CompareToUsesNonFinalVariable" enabled="true" level="WARNING" enabled_by_default="true" /> | ||||
|     <inspection_tool class="ComparisonOfShortAndChar" enabled="true" level="WARNING" enabled_by_default="true" /> | ||||
|     <inspection_tool class="ComposeMissingKeys" enabled="true" level="ERROR" enabled_by_default="true" /> | ||||
|     <inspection_tool class="ConditionSignal" enabled="true" level="WARNING" enabled_by_default="true" /> | ||||
|     <inspection_tool class="ConditionalExpression" enabled="true" level="INFORMATION" enabled_by_default="true"> | ||||
|       <option name="ignoreSimpleAssignmentsAndReturns" value="true" /> | ||||
| @@ -362,7 +361,7 @@ | ||||
|     </inspection_tool> | ||||
|     <inspection_tool class="LocalVariableDeclarationSideOnly" enabled="true" level="ERROR" enabled_by_default="true" /> | ||||
|     <inspection_tool class="LocalVariableNamingConventionJS" enabled="true" level="WARNING" enabled_by_default="true"> | ||||
|       <option name="m_regex" value="[a-z][A-Za-z]*" /> | ||||
|       <option name="m_regex" value="[a-z][A-Za-z0-9]*" /> | ||||
|       <option name="m_minLength" value="0" /> | ||||
|       <option name="m_maxLength" value="99" /> | ||||
|     </inspection_tool> | ||||
|   | ||||
| @@ -1,22 +0,0 @@ | ||||
| <component name="libraryTable"> | ||||
|   <library name="Generated files" type="javaScript"> | ||||
|     <properties> | ||||
|       <sourceFilesUrls> | ||||
|         <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/discord.js" /> | ||||
|         <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/dom.js" /> | ||||
|         <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/gui.js" /> | ||||
|         <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/settings.js" /> | ||||
|         <item url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/state.js" /> | ||||
|       </sourceFilesUrls> | ||||
|     </properties> | ||||
|     <CLASSES> | ||||
|       <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/discord.js" /> | ||||
|       <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/dom.js" /> | ||||
|       <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/gui.js" /> | ||||
|       <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/settings.js" /> | ||||
|       <root url="file://$PROJECT_DIR$/Resources/Tracker/scripts.min/state.js" /> | ||||
|     </CLASSES> | ||||
|     <JAVADOC /> | ||||
|     <SOURCES /> | ||||
|   </library> | ||||
| </component> | ||||
| @@ -15,6 +15,7 @@ sealed class Arguments { | ||||
| 	public string? DatabaseFile { get; } | ||||
| 	public ushort? ServerPort { get; } | ||||
| 	public string? ServerToken { get; } | ||||
| 	public byte? ConcurrentDownloads { get; } | ||||
|  | ||||
| 	public Arguments(IReadOnlyList<string> args) { | ||||
| 		for (int i = FirstArgument; i < args.Count; i++) { | ||||
| @@ -50,11 +51,11 @@ sealed class Arguments { | ||||
| 					continue; | ||||
|  | ||||
| 				case "-port": { | ||||
| 					if (ushort.TryParse(value, out var port)) { | ||||
| 						ServerPort = port; | ||||
| 					if (!ushort.TryParse(value, out var port)) { | ||||
| 						Log.Warn("Invalid port number: " + value); | ||||
| 					} | ||||
| 					else { | ||||
| 						Log.Warn("Invalid port number: " + value); | ||||
| 						ServerPort = port; | ||||
| 					} | ||||
|  | ||||
| 					continue; | ||||
| @@ -63,6 +64,20 @@ sealed class Arguments { | ||||
| 				case "-token": | ||||
| 					ServerToken = value; | ||||
| 					continue; | ||||
| 				 | ||||
| 				case "-concurrentdownloads": | ||||
| 					if (!ulong.TryParse(value, out var concurrentDownloads) || concurrentDownloads == 0) { | ||||
| 						Log.Warn("Invalid concurrent downloads count: " + value); | ||||
| 					} | ||||
| 					else if (concurrentDownloads > 10) { | ||||
| 						Log.Warn("Limiting concurrent downloads to 10"); | ||||
| 						ConcurrentDownloads = 10; | ||||
| 					} | ||||
| 					else { | ||||
| 						ConcurrentDownloads = (byte) concurrentDownloads; | ||||
| 					} | ||||
| 					 | ||||
| 					continue; | ||||
|  | ||||
| 				default: | ||||
| 					Log.Warn("Unknown command line argument: " + key); | ||||
|   | ||||
| @@ -39,11 +39,6 @@ | ||||
|     <EmbeddedResource Include="Resources/tracker-loader.js"> | ||||
|       <LogicalName>tracker-loader.js</LogicalName> | ||||
|     </EmbeddedResource> | ||||
|     <EmbeddedResource Include="../Resources/Viewer/**"> | ||||
|       <LogicalName>Viewer\%(RecursiveDir)%(Filename)%(Extension)</LogicalName> | ||||
|       <Link>Resources/Viewer/%(RecursiveDir)%(Filename)%(Extension)</Link> | ||||
|       <Visible>false</Visible> | ||||
|     </EmbeddedResource> | ||||
|   </ItemGroup> | ||||
|    | ||||
| </Project> | ||||
|   | ||||
| @@ -30,6 +30,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable { | ||||
| 	private MainContentScreenModel? mainContentScreenModel; | ||||
|  | ||||
| 	private readonly Window window; | ||||
| 	private readonly int? concurrentDownloads; | ||||
|  | ||||
| 	private State? state; | ||||
|  | ||||
| @@ -73,6 +74,8 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable { | ||||
| 		if (args.ServerToken != null) { | ||||
| 			ServerConfiguration.Token = args.ServerToken; | ||||
| 		} | ||||
| 		 | ||||
| 		concurrentDownloads = args.ConcurrentDownloads; | ||||
| 	} | ||||
|  | ||||
| 	private async void OnDatabaseSelected(object? sender, IDatabaseFile db) { | ||||
| @@ -80,7 +83,7 @@ sealed partial class MainWindowModel : ObservableObject, IAsyncDisposable { | ||||
| 		 | ||||
| 		await DisposeState(); | ||||
| 		 | ||||
| 		state = new State(db); | ||||
| 		state = new State(db, concurrentDownloads); | ||||
|  | ||||
| 		try { | ||||
| 			await state.Server.Start(ServerConfiguration.Port, ServerConfiguration.Token); | ||||
|   | ||||
| @@ -10,164 +10,164 @@ using DHT.Server; | ||||
| using DHT.Server.Data; | ||||
| using DHT.Server.Service; | ||||
|  | ||||
| namespace DHT.Desktop.Main.Pages { | ||||
| 	sealed class DebugPageModel { | ||||
| 		public string GenerateChannels { get; set; } = "0"; | ||||
| 		public string GenerateUsers { get; set; } = "0"; | ||||
| 		public string GenerateMessages { get; set; } = "0"; | ||||
| namespace DHT.Desktop.Main.Pages; | ||||
|  | ||||
| 		private readonly Window window; | ||||
| 		private readonly State state; | ||||
| sealed class DebugPageModel { | ||||
| 	public string GenerateChannels { get; set; } = "0"; | ||||
| 	public string GenerateUsers { get; set; } = "0"; | ||||
| 	public string GenerateMessages { get; set; } = "0"; | ||||
|  | ||||
| 		[Obsolete("Designer")] | ||||
| 		public DebugPageModel() : this(null!, State.Dummy) {} | ||||
| 	private readonly Window window; | ||||
| 	private readonly State state; | ||||
|  | ||||
| 		public DebugPageModel(Window window, State state) { | ||||
| 			this.window = window; | ||||
| 			this.state = state; | ||||
| 	[Obsolete("Designer")] | ||||
| 	public DebugPageModel() : this(null!, State.Dummy) {} | ||||
|  | ||||
| 	public DebugPageModel(Window window, State state) { | ||||
| 		this.window = window; | ||||
| 		this.state = state; | ||||
| 	} | ||||
|  | ||||
| 	public async void OnClickAddRandomDataToDatabase() { | ||||
| 		if (!int.TryParse(GenerateChannels, out int channels) || channels < 1) { | ||||
| 			await Dialog.ShowOk(window, "Generate Random Data", "Amount of channels must be at least 1!"); | ||||
| 			return; | ||||
| 		} | ||||
|  | ||||
| 		public async void OnClickAddRandomDataToDatabase() { | ||||
| 			if (!int.TryParse(GenerateChannels, out int channels) || channels < 1) { | ||||
| 				await Dialog.ShowOk(window, "Generate Random Data", "Amount of channels must be at least 1!"); | ||||
| 				return; | ||||
| 			} | ||||
|  | ||||
| 			if (!int.TryParse(GenerateUsers, out int users) || users < 1) { | ||||
| 				await Dialog.ShowOk(window, "Generate Random Data", "Amount of users must be at least 1!"); | ||||
| 				return; | ||||
| 			} | ||||
|  | ||||
| 			if (!int.TryParse(GenerateMessages, out int messages) || messages < 1) { | ||||
| 				await Dialog.ShowOk(window, "Generate Random Data", "Amount of messages must be at least 1!"); | ||||
| 				return; | ||||
| 			} | ||||
|  | ||||
| 			await ProgressDialog.Show(window, "Generating Random Data", async (_, callback) => await GenerateRandomData(channels, users, messages, callback)); | ||||
| 		if (!int.TryParse(GenerateUsers, out int users) || users < 1) { | ||||
| 			await Dialog.ShowOk(window, "Generate Random Data", "Amount of users must be at least 1!"); | ||||
| 			return; | ||||
| 		} | ||||
|  | ||||
| 		private const int BatchSize = 500; | ||||
| 		if (!int.TryParse(GenerateMessages, out int messages) || messages < 1) { | ||||
| 			await Dialog.ShowOk(window, "Generate Random Data", "Amount of messages must be at least 1!"); | ||||
| 			return; | ||||
| 		} | ||||
|  | ||||
| 		private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) { | ||||
| 			int batchCount = (messageCount + BatchSize - 1) / BatchSize; | ||||
| 			await callback.Update("Adding messages in batches of " + BatchSize, 0, batchCount); | ||||
| 		await ProgressDialog.Show(window, "Generating Random Data", async (_, callback) => await GenerateRandomData(channels, users, messages, callback)); | ||||
| 	} | ||||
|  | ||||
| 			var rand = new Random(); | ||||
| 			var server = new DHT.Server.Data.Server { | ||||
| 				Id = RandomId(rand), | ||||
| 				Name = RandomName("s"), | ||||
| 				Type = ServerType.Server, | ||||
| 			}; | ||||
| 	private const int BatchSize = 500; | ||||
|  | ||||
| 			var channels = Enumerable.Range(0, channelCount).Select(i => new Channel { | ||||
| 				Id = RandomId(rand), | ||||
| 				Server = server.Id, | ||||
| 				Name = RandomName("c"), | ||||
| 				ParentId = null, | ||||
| 				Position = i, | ||||
| 				Topic = RandomText(rand, 10), | ||||
| 				Nsfw = rand.Next(4) == 0, | ||||
| 	private async Task GenerateRandomData(int channelCount, int userCount, int messageCount, IProgressCallback callback) { | ||||
| 		int batchCount = (messageCount + BatchSize - 1) / BatchSize; | ||||
| 		await callback.Update("Adding messages in batches of " + BatchSize, 0, batchCount); | ||||
|  | ||||
| 		var rand = new Random(); | ||||
| 		var server = new DHT.Server.Data.Server { | ||||
| 			Id = RandomId(rand), | ||||
| 			Name = RandomName("s"), | ||||
| 			Type = ServerType.Server, | ||||
| 		}; | ||||
|  | ||||
| 		var channels = Enumerable.Range(0, channelCount).Select(i => new Channel { | ||||
| 			Id = RandomId(rand), | ||||
| 			Server = server.Id, | ||||
| 			Name = RandomName("c"), | ||||
| 			ParentId = null, | ||||
| 			Position = i, | ||||
| 			Topic = RandomText(rand, 10), | ||||
| 			Nsfw = rand.Next(4) == 0, | ||||
| 		}).ToArray(); | ||||
|  | ||||
| 		var users = Enumerable.Range(0, userCount).Select(_ => new User { | ||||
| 			Id = RandomId(rand), | ||||
| 			Name = RandomName("u"), | ||||
| 			AvatarUrl = null, | ||||
| 			Discriminator = rand.Next(0, 9999).ToString(), | ||||
| 		}).ToArray(); | ||||
|  | ||||
| 		await state.Db.Users.Add(users); | ||||
| 		await state.Db.Servers.Add([server]); | ||||
| 		await state.Db.Channels.Add(channels); | ||||
|  | ||||
| 		var now = DateTimeOffset.Now; | ||||
| 		int batchIndex = 0; | ||||
|  | ||||
| 		while (messageCount > 0) { | ||||
| 			int hourOffset = batchIndex; | ||||
|  | ||||
| 			var messages = Enumerable.Range(0, Math.Min(messageCount, BatchSize)).Select(i => { | ||||
| 				DateTimeOffset time = now.AddHours(hourOffset).AddMinutes(i * 60.0 / BatchSize); | ||||
| 				DateTimeOffset? edit = rand.Next(100) == 0 ? time.AddSeconds(rand.Next(1, 60)) : null; | ||||
|  | ||||
| 				var timeMillis = time.ToUnixTimeMilliseconds(); | ||||
| 				var editMillis = edit?.ToUnixTimeMilliseconds(); | ||||
|  | ||||
| 				return new Message { | ||||
| 					Id = (ulong) timeMillis, | ||||
| 					Sender = RandomBiasedIndex(rand, users).Id, | ||||
| 					Channel = RandomBiasedIndex(rand, channels).Id, | ||||
| 					Text = RandomText(rand, 100), | ||||
| 					Timestamp = timeMillis, | ||||
| 					EditTimestamp = editMillis, | ||||
| 					RepliedToId = null, | ||||
| 					Attachments = ImmutableList<Attachment>.Empty, | ||||
| 					Embeds = ImmutableList<Embed>.Empty, | ||||
| 					Reactions = ImmutableList<Reaction>.Empty, | ||||
| 				}; | ||||
| 			}).ToArray(); | ||||
|  | ||||
| 			var users = Enumerable.Range(0, userCount).Select(_ => new User { | ||||
| 				Id = RandomId(rand), | ||||
| 				Name = RandomName("u"), | ||||
| 				AvatarUrl = null, | ||||
| 				Discriminator = rand.Next(0, 9999).ToString(), | ||||
| 			}).ToArray(); | ||||
| 			await state.Db.Messages.Add(messages); | ||||
|  | ||||
| 			await state.Db.Users.Add(users); | ||||
| 			await state.Db.Servers.Add([server]); | ||||
| 			await state.Db.Channels.Add(channels); | ||||
|  | ||||
| 			var now = DateTimeOffset.Now; | ||||
| 			int batchIndex = 0; | ||||
|  | ||||
| 			while (messageCount > 0) { | ||||
| 				int hourOffset = batchIndex; | ||||
|  | ||||
| 				var messages = Enumerable.Range(0, Math.Min(messageCount, BatchSize)).Select(i => { | ||||
| 					DateTimeOffset time = now.AddHours(hourOffset).AddMinutes(i * 60.0 / BatchSize); | ||||
| 					DateTimeOffset? edit = rand.Next(100) == 0 ? time.AddSeconds(rand.Next(1, 60)) : null; | ||||
|  | ||||
| 					var timeMillis = time.ToUnixTimeMilliseconds(); | ||||
| 					var editMillis = edit?.ToUnixTimeMilliseconds(); | ||||
|  | ||||
| 					return new Message { | ||||
| 						Id = (ulong) timeMillis, | ||||
| 						Sender = RandomBiasedIndex(rand, users).Id, | ||||
| 						Channel = RandomBiasedIndex(rand, channels).Id, | ||||
| 						Text = RandomText(rand, 100), | ||||
| 						Timestamp = timeMillis, | ||||
| 						EditTimestamp = editMillis, | ||||
| 						RepliedToId = null, | ||||
| 						Attachments = ImmutableList<Attachment>.Empty, | ||||
| 						Embeds = ImmutableList<Embed>.Empty, | ||||
| 						Reactions = ImmutableList<Reaction>.Empty, | ||||
| 					}; | ||||
| 				}).ToArray(); | ||||
|  | ||||
| 				await state.Db.Messages.Add(messages); | ||||
|  | ||||
| 				messageCount -= BatchSize; | ||||
| 				await callback.Update("Adding messages in batches of " + BatchSize, ++batchIndex, batchCount); | ||||
| 			} | ||||
| 			messageCount -= BatchSize; | ||||
| 			await callback.Update("Adding messages in batches of " + BatchSize, ++batchIndex, batchCount); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 		private static ulong RandomId(Random rand) { | ||||
| 			ulong h = unchecked((ulong) rand.Next()); | ||||
| 			ulong l = unchecked((ulong) rand.Next()); | ||||
| 			return (h << 32) | l; | ||||
| 		} | ||||
| 	private static ulong RandomId(Random rand) { | ||||
| 		ulong h = unchecked((ulong) rand.Next()); | ||||
| 		ulong l = unchecked((ulong) rand.Next()); | ||||
| 		return (h << 32) | l; | ||||
| 	} | ||||
|  | ||||
| 		private static string RandomName(string prefix) { | ||||
| 			return prefix + "-" + ServerUtils.GenerateRandomToken(5); | ||||
| 		} | ||||
| 	private static string RandomName(string prefix) { | ||||
| 		return prefix + "-" + ServerUtils.GenerateRandomToken(5); | ||||
| 	} | ||||
|  | ||||
| 		private static T RandomBiasedIndex<T>(Random rand, T[] options) { | ||||
| 			return options[(int) Math.Floor(options.Length * rand.NextDouble() * rand.NextDouble())]; | ||||
| 		} | ||||
| 	private static T RandomBiasedIndex<T>(Random rand, T[] options) { | ||||
| 		return options[(int) Math.Floor(options.Length * rand.NextDouble() * rand.NextDouble())]; | ||||
| 	} | ||||
|  | ||||
| 		private static readonly string[] RandomWords = [ | ||||
| 			"apple", "apricot", "artichoke", "arugula", "asparagus", "avocado", | ||||
| 			"banana", "bean", "beechnut", "beet", "blackberry", "blackcurrant", "blueberry", "boysenberry", "bramble", "broccoli", | ||||
| 			"cabbage", "cacao", "cantaloupe", "caper", "carambola", "carrot", "cauliflower", "celery", "chard", "cherry", "chokeberry", "citron", "clementine", "coconut", "corn", "crabapple", "cranberry", "cucumber", "currant", | ||||
| 			"daikon", "date", "dewberry", "durian", | ||||
| 			"edamame", "eggplant", "elderberry", "endive", | ||||
| 			"fig", | ||||
| 			"garlic", "ginger", "gooseberry", "grape", "grapefruit", "guava", | ||||
| 			"honeysuckle", "horseradish", "huckleberry", | ||||
| 			"jackfruit", "jicama", | ||||
| 			"kale", "kiwi", "kohlrabi", "kumquat", | ||||
| 			"leek", "lemon", "lentil", "lettuce", "lime", | ||||
| 			"mandarin", "mango", "mushroom", "myrtle", | ||||
| 			"nectarine", "nut", | ||||
| 			"olive", "okra", "onion", "orange", | ||||
| 			"papaya", "parsnip", "pawpaw", "peach", "pear", "pea", "pepper", "persimmon", "pineapple", "plum", "plantain", "pomegranate", "pomelo", "potato", "prune", "pumpkin", | ||||
| 			"quandong", "quinoa", | ||||
| 			"radicchio", "radish", "raisin", "raspberry", "redcurrant", "rhubarb", "rutabaga", | ||||
| 			"spinach", "strawberry", "squash", | ||||
| 			"tamarind", "tangerine", "tomatillo", "tomato", "turnip", | ||||
| 			"vanilla", | ||||
| 			"watercress", "watermelon", | ||||
| 			"yam", | ||||
| 			"zucchini" | ||||
| 		]; | ||||
| 	private static readonly string[] RandomWords = [ | ||||
| 		"apple", "apricot", "artichoke", "arugula", "asparagus", "avocado", | ||||
| 		"banana", "bean", "beechnut", "beet", "blackberry", "blackcurrant", "blueberry", "boysenberry", "bramble", "broccoli", | ||||
| 		"cabbage", "cacao", "cantaloupe", "caper", "carambola", "carrot", "cauliflower", "celery", "chard", "cherry", "chokeberry", "citron", "clementine", "coconut", "corn", "crabapple", "cranberry", "cucumber", "currant", | ||||
| 		"daikon", "date", "dewberry", "durian", | ||||
| 		"edamame", "eggplant", "elderberry", "endive", | ||||
| 		"fig", | ||||
| 		"garlic", "ginger", "gooseberry", "grape", "grapefruit", "guava", | ||||
| 		"honeysuckle", "horseradish", "huckleberry", | ||||
| 		"jackfruit", "jicama", | ||||
| 		"kale", "kiwi", "kohlrabi", "kumquat", | ||||
| 		"leek", "lemon", "lentil", "lettuce", "lime", | ||||
| 		"mandarin", "mango", "mushroom", "myrtle", | ||||
| 		"nectarine", "nut", | ||||
| 		"olive", "okra", "onion", "orange", | ||||
| 		"papaya", "parsnip", "pawpaw", "peach", "pear", "pea", "pepper", "persimmon", "pineapple", "plum", "plantain", "pomegranate", "pomelo", "potato", "prune", "pumpkin", | ||||
| 		"quandong", "quinoa", | ||||
| 		"radicchio", "radish", "raisin", "raspberry", "redcurrant", "rhubarb", "rutabaga", | ||||
| 		"spinach", "strawberry", "squash", | ||||
| 		"tamarind", "tangerine", "tomatillo", "tomato", "turnip", | ||||
| 		"vanilla", | ||||
| 		"watercress", "watermelon", | ||||
| 		"yam", | ||||
| 		"zucchini" | ||||
| 	]; | ||||
|  | ||||
| 		private static string RandomText(Random rand, int maxWords) { | ||||
| 			int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3)); | ||||
| 			return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)])); | ||||
| 		} | ||||
| 	private static string RandomText(Random rand, int maxWords) { | ||||
| 		int wordCount = 1 + (int) Math.Floor(maxWords * Math.Pow(rand.NextDouble(), 3)); | ||||
| 		return string.Join(' ', Enumerable.Range(0, wordCount).Select(_ => RandomWords[rand.Next(RandomWords.Length)])); | ||||
| 	} | ||||
| } | ||||
| #else | ||||
| namespace DHT.Desktop.Main.Pages { | ||||
| 	sealed class DebugPageModel { | ||||
| 		public string GenerateChannels { get; set; } = "0"; | ||||
| 		public string GenerateUsers { get; set; } = "0"; | ||||
| 		public string GenerateMessages { get; set; } = "0"; | ||||
| namespace DHT.Desktop.Main.Pages; | ||||
|  | ||||
| 		public void OnClickAddRandomDataToDatabase() {} | ||||
| 	} | ||||
| sealed class DebugPageModel { | ||||
| 	public string GenerateChannels { get; set; } = "0"; | ||||
| 	public string GenerateUsers { get; set; } = "0"; | ||||
| 	public string GenerateMessages { get; set; } = "0"; | ||||
|  | ||||
| 	public void OnClickAddRandomDataToDatabase() {} | ||||
| } | ||||
| #endif | ||||
|   | ||||
| @@ -21,7 +21,6 @@ | ||||
|     <StackPanel Orientation="Vertical" Spacing="20"> | ||||
|         <StackPanel Orientation="Horizontal" VerticalAlignment="Top"> | ||||
|             <Button Command="{Binding OnClickOpenViewer}" Margin="0 0 5 0">Open Viewer</Button> | ||||
|             <Button Command="{Binding OnClickSaveViewer}" Margin="5 0 0 0">Save Viewer</Button> | ||||
|         </StackPanel> | ||||
|         <controls:MessageFilterPanel DataContext="{Binding FilterModel}" /> | ||||
|         <Expander Header="Database Tools"> | ||||
|   | ||||
| @@ -2,8 +2,6 @@ using System; | ||||
| using System.Collections.Concurrent; | ||||
| using System.ComponentModel; | ||||
| using System.Diagnostics; | ||||
| using System.IO; | ||||
| using System.Text; | ||||
| using System.Threading.Tasks; | ||||
| using System.Web; | ||||
| using Avalonia.Controls; | ||||
| @@ -17,8 +15,6 @@ using DHT.Desktop.Main.Controls; | ||||
| using DHT.Desktop.Server; | ||||
| using DHT.Server; | ||||
| using DHT.Server.Data.Filters; | ||||
| using DHT.Server.Database.Export; | ||||
| using static DHT.Desktop.Program; | ||||
|  | ||||
| namespace DHT.Desktop.Main.Pages; | ||||
|  | ||||
| @@ -61,101 +57,17 @@ sealed partial class ViewerPageModel : ObservableObject, IDisposable { | ||||
|  | ||||
| 	public async void OnClickOpenViewer() { | ||||
| 		try { | ||||
| 			var fullPath = await PrepareTemporaryViewerFile(); | ||||
| 			string serverUrl = "http://127.0.0.1:" + ServerConfiguration.Port; | ||||
| 			string serverToken = ServerConfiguration.Token; | ||||
| 			 | ||||
| 			string jsConstants = $""" | ||||
| 			                      window.DHT_SERVER_URL = "{HttpUtility.JavaScriptStringEncode("http://127.0.0.1:" + ServerConfiguration.Port)}"; | ||||
| 			                      window.DHT_SERVER_TOKEN = "{HttpUtility.JavaScriptStringEncode(ServerConfiguration.Token)}"; | ||||
| 			                      """; | ||||
|  | ||||
| 			await ProgressDialog.ShowIndeterminate(window, "Open Viewer", "Creating viewer...", _ => Task.Run(() => WriteViewerFile(fullPath, jsConstants))); | ||||
| 			 | ||||
| 			Process.Start(new ProcessStartInfo(fullPath) { | ||||
| 			Process.Start(new ProcessStartInfo(serverUrl + "/viewer/?token=" + HttpUtility.UrlEncode(serverToken)) { | ||||
| 				UseShellExecute = true | ||||
| 			}); | ||||
| 		} catch (Exception e) { | ||||
| 			await Dialog.ShowOk(window, "Open Viewer", "Could not create or save viewer: " + e.Message); | ||||
| 			await Dialog.ShowOk(window, "Open Viewer", "Could not open viewer: " + e.Message); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	private async Task<string> PrepareTemporaryViewerFile() { | ||||
| 		return await Task.Run(() => { | ||||
| 			string rootPath = Path.Combine(Path.GetTempPath(), "DiscordHistoryTracker"); | ||||
| 			string filenameBase = Path.GetFileNameWithoutExtension(state.Db.Path) + "-" + DateTime.Now.ToString("yyyy-MM-dd"); | ||||
| 			string fullPath = Path.Combine(rootPath, filenameBase + ".html"); | ||||
| 			 | ||||
| 			int counter = 0; | ||||
|  | ||||
| 			while (File.Exists(fullPath)) { | ||||
| 				++counter; | ||||
| 				fullPath = Path.Combine(rootPath, filenameBase + "-" + counter + ".html"); | ||||
| 			} | ||||
|  | ||||
| 			TemporaryFiles.Add(fullPath); | ||||
|  | ||||
| 			Directory.CreateDirectory(rootPath); | ||||
|  | ||||
| 			return fullPath; | ||||
| 		}); | ||||
| 	} | ||||
|  | ||||
| 	public async void OnClickSaveViewer() { | ||||
| 		string? path = await window.StorageProvider.SaveFile(new FilePickerSaveOptions { | ||||
| 			Title = "Save Viewer", | ||||
| 			FileTypeChoices = ViewerFileTypes, | ||||
| 			SuggestedFileName = Path.GetFileNameWithoutExtension(state.Db.Path) + ".html", | ||||
| 			SuggestedStartLocation = await FileDialogs.GetSuggestedStartLocation(window, Path.GetDirectoryName(state.Db.Path)), | ||||
| 		}); | ||||
|  | ||||
| 		if (path == null) { | ||||
| 			return; | ||||
| 		} | ||||
|  | ||||
| 		try { | ||||
| 			await ProgressDialog.ShowIndeterminate(window, "Save Viewer", "Creating viewer...", _ => Task.Run(() => WriteViewerFile(path, string.Empty))); | ||||
| 		} catch (Exception e) { | ||||
| 			await Dialog.ShowOk(window, "Save Viewer", "Could not create or save viewer: " + e.Message); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	private async Task WriteViewerFile(string path, string jsConstants) { | ||||
| 		const string ArchiveTag = "/*[ARCHIVE]*/"; | ||||
|  | ||||
| 		string indexFile = await Resources.ReadTextAsync("Viewer/index.html"); | ||||
| 		string viewerTemplate = indexFile.Replace("/*[CONSTANTS]*/", jsConstants) | ||||
| 		                                 .Replace("/*[JS]*/", await Resources.ReadJoinedAsync("Viewer/scripts/", '\n')) | ||||
| 		                                 .Replace("/*[CSS]*/", await Resources.ReadJoinedAsync("Viewer/styles/", '\n')); | ||||
|  | ||||
| 		int viewerArchiveTagStart = viewerTemplate.IndexOf(ArchiveTag); | ||||
| 		int viewerArchiveTagEnd = viewerArchiveTagStart + ArchiveTag.Length; | ||||
|  | ||||
| 		string jsonTempFile = path + ".tmp"; | ||||
|  | ||||
| 		await using (var jsonStream = new FileStream(jsonTempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) { | ||||
| 			await ViewerJsonExport.Generate(jsonStream, state.Db, FilterModel.CreateFilter()); | ||||
|  | ||||
| 			char[] jsonBuffer = new char[Math.Min(32768, jsonStream.Position)]; | ||||
| 			jsonStream.Position = 0; | ||||
|  | ||||
| 			await using (var outputStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read)) | ||||
| 			await using (var outputWriter = new StreamWriter(outputStream, Encoding.UTF8)) { | ||||
| 				await outputWriter.WriteAsync(viewerTemplate[..viewerArchiveTagStart]); | ||||
|  | ||||
| 				using (var jsonReader = new StreamReader(jsonStream, Encoding.UTF8)) { | ||||
| 					int readBytes; | ||||
| 					while ((readBytes = await jsonReader.ReadAsync(jsonBuffer, 0, jsonBuffer.Length)) > 0) { | ||||
| 						string jsonChunk = new string(jsonBuffer, 0, readBytes); | ||||
| 						await outputWriter.WriteAsync(HttpUtility.JavaScriptStringEncode(jsonChunk)); | ||||
| 					} | ||||
| 				} | ||||
|  | ||||
| 				await outputWriter.WriteAsync(viewerTemplate[viewerArchiveTagEnd..]); | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		File.Delete(jsonTempFile); | ||||
| 	} | ||||
|  | ||||
| 	public async Task OnClickApplyFiltersToDatabase() { | ||||
| 		var filter = FilterModel.CreateFilter(); | ||||
| 		var messageCount = await ProgressDialog.ShowIndeterminate(window, "Apply Filters", "Counting matching messages...", _ => state.Db.Messages.Count(filter)); | ||||
|   | ||||
| @@ -48,6 +48,16 @@ const STATE = (function() { | ||||
| 		}); | ||||
| 	}; | ||||
| 	 | ||||
| 	const getDate = function(date) { | ||||
| 		if (date instanceof Date) { | ||||
| 			return date; | ||||
| 		} | ||||
| 		else { | ||||
| 			// noinspection JSUnresolvedReference | ||||
| 			return date.toDate(); | ||||
| 		} | ||||
| 	}; | ||||
| 	 | ||||
| 	const trackingStateChangedListeners = []; | ||||
| 	let isTracking = false; | ||||
| 	 | ||||
| @@ -69,8 +79,8 @@ const STATE = (function() { | ||||
| 	 * @property {String} channel_id | ||||
| 	 * @property {DiscordUser} author | ||||
| 	 * @property {String} content | ||||
| 	 * @property {Timestamp} timestamp | ||||
| 	 * @property {Timestamp|null} editedTimestamp | ||||
| 	 * @property {Date} timestamp | ||||
| 	 * @property {Date|null} editedTimestamp | ||||
| 	 * @property {DiscordAttachment[]} attachments | ||||
| 	 * @property {Object[]} embeds | ||||
| 	 * @property {DiscordMessageReaction[]} [reactions] | ||||
| @@ -106,11 +116,6 @@ const STATE = (function() { | ||||
| 	 * @property {Boolean} animated | ||||
| 	 */ | ||||
| 	 | ||||
| 	/** | ||||
| 	 * @name Timestamp | ||||
| 	 * @property {Function} toDate | ||||
| 	 */ | ||||
| 	 | ||||
| 	return { | ||||
| 		setup(port, token) { | ||||
| 			serverPort = port; | ||||
| @@ -223,12 +228,12 @@ const STATE = (function() { | ||||
| 					sender: msg.author.id, | ||||
| 					channel: msg.channel_id, | ||||
| 					text: msg.content, | ||||
| 					timestamp: msg.timestamp.toDate().getTime() | ||||
| 					timestamp: getDate(msg.timestamp).getTime() | ||||
| 				}; | ||||
| 				 | ||||
| 				if (msg.editedTimestamp !== null) { | ||||
| 					// noinspection JSUnusedGlobalSymbols | ||||
| 					obj.editTimestamp = msg.editedTimestamp.toDate().getTime(); | ||||
| 					obj.editTimestamp = getDate(msg.editedTimestamp).getTime(); | ||||
| 				} | ||||
| 				 | ||||
| 				if (msg.messageReference !== null) { | ||||
|   | ||||
| @@ -4,14 +4,18 @@ | ||||
|     <meta charset="utf-8"> | ||||
|     <title>Discord Offline History</title> | ||||
|      | ||||
|     <link rel="icon" href="favicon.ico"> | ||||
|      | ||||
|     <link rel="stylesheet" href="styles/main.css"> | ||||
|     <link rel="stylesheet" href="styles/menu.css"> | ||||
|     <link rel="stylesheet" href="styles/channels.css"> | ||||
|     <link rel="stylesheet" href="styles/messages.css"> | ||||
|     <link rel="stylesheet" href="styles/modal.css"> | ||||
|      | ||||
|     <script type="text/javascript"> | ||||
| 		window.DHT_EMBEDDED = "/*[ARCHIVE]*/"; | ||||
| 		/*[CONSTANTS]*/ | ||||
| 		/*[JS]*/ | ||||
| 		window.DHT_SERVER_TOKEN = new URLSearchParams(location.search).get("token"); | ||||
|     </script> | ||||
|     <style> | ||||
|       /*[CSS]*/ | ||||
|     </style> | ||||
|     <script type="module" src="scripts/bootstrap.mjs"></script> | ||||
|   </head> | ||||
|   <body> | ||||
|     <div id="menu"> | ||||
| @@ -65,7 +69,9 @@ | ||||
|     </div> | ||||
|      | ||||
|     <div id="app"> | ||||
|       <div id="channels"></div> | ||||
|       <div id="channels"> | ||||
|         <div class="loading"></div> | ||||
|       </div> | ||||
|       <div id="messages"></div> | ||||
|     </div> | ||||
|      | ||||
|   | ||||
							
								
								
									
										39
									
								
								app/Resources/Viewer/scripts/bootstrap.js
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										39
									
								
								app/Resources/Viewer/scripts/bootstrap.js
									
									
									
									
										vendored
									
									
								
							| @@ -1,39 +0,0 @@ | ||||
| document.addEventListener("DOMContentLoaded", () => { | ||||
| 	DISCORD.setup(); | ||||
| 	GUI.setup(); | ||||
| 	 | ||||
| 	GUI.onOptionMessagesPerPageChanged(() => { | ||||
| 		STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage()); | ||||
| 	}); | ||||
| 	 | ||||
| 	STATE.setMessagesPerPage(GUI.getOptionMessagesPerPage()); | ||||
| 	 | ||||
| 	GUI.onOptMessageFilterChanged(filter => { | ||||
| 		STATE.setActiveFilter(filter); | ||||
| 	}); | ||||
| 	 | ||||
| 	GUI.onNavigationButtonClicked(action => { | ||||
| 		STATE.updateCurrentPage(action); | ||||
| 	}); | ||||
| 	 | ||||
| 	STATE.onUsersRefreshed(users => { | ||||
| 		GUI.updateUserList(users); | ||||
| 	}); | ||||
| 	 | ||||
| 	STATE.onChannelsRefreshed((channels, selected) => { | ||||
| 		GUI.updateChannelList(channels, selected, STATE.selectChannel); | ||||
| 	}); | ||||
| 	 | ||||
| 	STATE.onMessagesRefreshed(messages => { | ||||
| 		GUI.updateNavigation(STATE.getCurrentPage(), STATE.getPageCount()); | ||||
| 		GUI.updateMessageList(messages); | ||||
| 		GUI.scrollMessagesToTop(); | ||||
| 	}); | ||||
| 	 | ||||
| 	try { | ||||
| 		STATE.uploadFile(JSON.parse(window.DHT_EMBEDDED)); | ||||
| 	} catch (e) { | ||||
| 		console.error(e); | ||||
| 		alert("Could not parse embedded file, see console for details."); | ||||
| 	} | ||||
| }); | ||||
							
								
								
									
										60
									
								
								app/Resources/Viewer/scripts/bootstrap.mjs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										60
									
								
								app/Resources/Viewer/scripts/bootstrap.mjs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,60 @@ | ||||
| import discord from "./discord.mjs"; | ||||
| import gui from "./gui.mjs"; | ||||
| import state from "./state.mjs"; | ||||
|  | ||||
| window.DISCORD = discord; | ||||
|  | ||||
| document.addEventListener("DOMContentLoaded", () => { | ||||
| 	discord.setup(); | ||||
| 	gui.setup(); | ||||
| 	 | ||||
| 	gui.onOptionMessagesPerPageChanged(() => { | ||||
| 		state.setMessagesPerPage(gui.getOptionMessagesPerPage()); | ||||
| 	}); | ||||
| 	 | ||||
| 	state.setMessagesPerPage(gui.getOptionMessagesPerPage()); | ||||
| 	 | ||||
| 	gui.onOptMessageFilterChanged(filter => { | ||||
| 		state.setActiveFilter(filter); | ||||
| 	}); | ||||
| 	 | ||||
| 	gui.onNavigationButtonClicked(action => { | ||||
| 		state.updateCurrentPage(action); | ||||
| 	}); | ||||
| 	 | ||||
| 	state.onUsersRefreshed(users => { | ||||
| 		gui.updateUserList(users); | ||||
| 	}); | ||||
| 	 | ||||
| 	state.onChannelsRefreshed((channels, selected) => { | ||||
| 		gui.updateChannelList(channels, selected, state.selectChannel); | ||||
| 	}); | ||||
| 	 | ||||
| 	state.onMessagesRefreshed(messages => { | ||||
| 		gui.updateNavigation(state.getCurrentPage(), state.getPageCount()); | ||||
| 		gui.updateMessageList(messages); | ||||
| 		gui.scrollMessagesToTop(); | ||||
| 	}); | ||||
| 	 | ||||
| 	async function loadData() { | ||||
| 		try { | ||||
| 			const response = await fetch("/get-viewer-data?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN), { | ||||
| 				method: "GET", | ||||
| 				headers: { | ||||
| 					"Content-Type": "application/json", | ||||
| 				}, | ||||
| 				credentials: "omit", | ||||
| 				redirect: "error", | ||||
| 			}); | ||||
| 			 | ||||
| 			const json = await response.json(); | ||||
| 			state.uploadFile(json); | ||||
| 		} catch (e) { | ||||
| 			console.error(e); | ||||
| 			alert("Could not load data, see console for details."); | ||||
| 			document.querySelector("#channels > div.loading").remove(); | ||||
| 		} | ||||
| 	} | ||||
| 	 | ||||
| 	loadData(); | ||||
| }); | ||||
| @@ -1,4 +1,10 @@ | ||||
| const DISCORD = (function() { | ||||
| import discord from "./discord.mjs"; | ||||
| import dom from "./dom.mjs"; | ||||
| import template from "./template.mjs"; | ||||
| import settings from "./settings.mjs"; | ||||
| import state from "./state.mjs"; | ||||
| 
 | ||||
| export default (function() { | ||||
| 	const regex = { | ||||
| 		formatBold: /\*\*([\s\S]+?)\*\*(?!\*)/g, | ||||
| 		formatItalic1: /\*([\s\S]+?)\*(?!\*)/g, | ||||
| @@ -35,17 +41,14 @@ const DISCORD = (function() { | ||||
| 	let templateReaction; | ||||
| 	let templateReactionCustom; | ||||
| 	 | ||||
| 	const fileUrlProcessor = function(serverUrl, serverToken) { | ||||
| 		if (typeof serverUrl === "string" && typeof serverToken === "string") { | ||||
| 			return url => serverUrl + "/get-downloaded-file/" + encodeURIComponent(url) + "?token=" + encodeURIComponent(serverToken); | ||||
| 	const fileUrlProcessor = function(serverToken) { | ||||
| 		if (typeof serverToken === "string") { | ||||
| 			return url => "/get-downloaded-file/" + encodeURIComponent(url) + "?token=" + encodeURIComponent(serverToken); | ||||
| 		} | ||||
| 		else { | ||||
| 			return url => url; | ||||
| 		} | ||||
| 	}( | ||||
| 		window["DHT_SERVER_URL"], | ||||
| 		window["DHT_SERVER_TOKEN"] | ||||
| 	); | ||||
| 	}(window.DHT_SERVER_TOKEN); | ||||
| 	 | ||||
| 	const getEmoji = function(name, id, extension) { | ||||
| 		const tag = ":" + name + ":"; | ||||
| @@ -53,9 +56,9 @@ const DISCORD = (function() { | ||||
| 	}; | ||||
| 	 | ||||
| 	const processMessageContents = function(contents) { | ||||
| 		let processed = DOM.escapeHTML(contents.replace(regex.formatUrlNoEmbed, "$1")); | ||||
| 		let processed = dom.escapeHTML(contents.replace(regex.formatUrlNoEmbed, "$1")); | ||||
| 		 | ||||
| 		if (SETTINGS.enableFormatting) { | ||||
| 		if (settings.enableFormatting) { | ||||
| 			const escapeHtmlMatch = (full, match) => "&#" + match.charCodeAt(0) + ";"; | ||||
| 			 | ||||
| 			processed = processed | ||||
| @@ -71,13 +74,13 @@ const DISCORD = (function() { | ||||
| 				.replace(regex.formatStrike, "<s>$1</s>"); | ||||
| 		} | ||||
| 		 | ||||
| 		const animatedEmojiExtension = SETTINGS.enableAnimatedEmoji ? "gif" : "webp"; | ||||
| 		const animatedEmojiExtension = settings.enableAnimatedEmoji ? "gif" : "webp"; | ||||
| 		 | ||||
| 		// noinspection HtmlUnknownTarget
 | ||||
| 		processed = processed | ||||
| 			.replace(regex.formatUrl, "<a href='$1' target='_blank' rel='noreferrer'>$1</a>") | ||||
| 			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + STATE.getChannelName(match) + "</span>") | ||||
| 			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (STATE.getUserTag(match) || "????") + "'>@" + STATE.getUserName(match) + "</span>") | ||||
| 			.replace(regex.mentionChannel, (full, match) => "<span class='link mention-chat'>#" + state.getChannelName(match) + "</span>") | ||||
| 			.replace(regex.mentionUser, (full, match) => "<span class='link mention-user' title='#" + (state.getUserTag(match) || "????") + "'>@" + state.getUserName(match) + "</span>") | ||||
| 			.replace(regex.customEmojiStatic, (full, m1, m2) => getEmoji(m1, m2, "webp")) | ||||
| 			.replace(regex.customEmojiAnimated, (full, m1, m2) => getEmoji(m1, m2, animatedEmojiExtension)); | ||||
| 		 | ||||
| @@ -89,7 +92,7 @@ const DISCORD = (function() { | ||||
| 	}; | ||||
| 	 | ||||
| 	const getImageEmbed = function(url, image) { | ||||
| 		if (!SETTINGS.enableImagePreviews) { | ||||
| 		if (!settings.enableImagePreviews) { | ||||
| 			return ""; | ||||
| 		} | ||||
| 		 | ||||
| @@ -109,21 +112,21 @@ const DISCORD = (function() { | ||||
| 	 | ||||
| 	return { | ||||
| 		setup() { | ||||
| 			templateChannelServer = new TEMPLATE([ | ||||
| 				"<div data-channel='{id}'>", | ||||
| 			templateChannelServer = new template([ | ||||
| 				"<div class='channel' data-channel='{id}'>", | ||||
| 				"<div class='info' title='{topic}'><strong class='name'>#{name}</strong>{nsfw}<span class='tag'>{msgcount}</span></div>", | ||||
| 				"<span class='server'>{server.name} ({server.type})</span>", | ||||
| 				"</div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			templateChannelPrivate = new TEMPLATE([ | ||||
| 				"<div data-channel='{id}'>", | ||||
| 			templateChannelPrivate = new template([ | ||||
| 				"<div class='channel' data-channel='{id}'>", | ||||
| 				"<div class='info'><strong class='name'>{name}</strong><span class='tag'>{msgcount}</span></div>", | ||||
| 				"<span class='server'>({server.type})</span>", | ||||
| 				"</div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			templateMessageNoAvatar = new TEMPLATE([ | ||||
| 			templateMessageNoAvatar = new template([ | ||||
| 				"<div>", | ||||
| 				"<div class='reply-message'>{reply}</div>", | ||||
| 				"<h2><strong class='username' title='#{user.tag}'>{user.name}</strong><span class='info time'>{timestamp}</span>{edit}{jump}</h2>", | ||||
| @@ -132,7 +135,7 @@ const DISCORD = (function() { | ||||
| 				"</div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			templateMessageWithAvatar = new TEMPLATE([ | ||||
| 			templateMessageWithAvatar = new template([ | ||||
| 				"<div>", | ||||
| 				"<div class='reply-message reply-message-with-avatar'>{reply}</div>", | ||||
| 				"<div class='avatar-wrapper'>", | ||||
| @@ -147,50 +150,50 @@ const DISCORD = (function() { | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateUserAvatar = new TEMPLATE([ | ||||
| 			templateUserAvatar = new template([ | ||||
| 				"<img src='{url}' alt=''>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateAttachmentDownload = new TEMPLATE([ | ||||
| 			templateAttachmentDownload = new template([ | ||||
| 				"<a href='{url}' class='embed download'>Download {name}</a>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateEmbedImage = new TEMPLATE([ | ||||
| 				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>" | ||||
| 			templateEmbedImage = new template([ | ||||
| 				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' alt='' onload='window.DISCORD.handleImageLoad(this)' onerror='window.DISCORD.handleImageLoadError(this)'></a><br>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateEmbedImageWithSize = new TEMPLATE([ | ||||
| 				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' width='{width}' height='{height}' alt='' onload='DISCORD.handleImageLoad(this)' onerror='DISCORD.handleImageLoadError(this)'></a><br>" | ||||
| 			templateEmbedImageWithSize = new template([ | ||||
| 				"<a href='{url}' class='embed thumbnail loading'><img src='{src}' width='{width}' height='{height}' alt='' onload='window.DISCORD.handleImageLoad(this)' onerror='window.DISCORD.handleImageLoadError(this)'></a><br>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateEmbedRich = new TEMPLATE([ | ||||
| 			templateEmbedRich = new template([ | ||||
| 				"<div class='embed download'><a href='{url}' class='title'>{title}</a><p class='desc'>{description}</p></div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateEmbedRichNoDescription = new TEMPLATE([ | ||||
| 			templateEmbedRichNoDescription = new template([ | ||||
| 				"<div class='embed download'><a href='{url}' class='title'>{title}</a></div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateEmbedUrl = new TEMPLATE([ | ||||
| 			templateEmbedUrl = new template([ | ||||
| 				"<a href='{url}' class='embed download'>{url}</a>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			templateEmbedUnsupported = new TEMPLATE([ | ||||
| 			templateEmbedUnsupported = new template([ | ||||
| 				"<div class='embed download'><p>(Unsupported embed)</p></div>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			templateReaction = new TEMPLATE([ | ||||
| 			templateReaction = new template([ | ||||
| 				"<span class='reaction-wrapper'><span class='reaction-emoji'>{n}</span><span class='count'>{c}</span></span>" | ||||
| 			].join("")); | ||||
| 			 | ||||
| 			// noinspection HtmlUnknownTarget
 | ||||
| 			templateReactionCustom = new TEMPLATE([ | ||||
| 			templateReactionCustom = new template([ | ||||
| 				"<span class='reaction-wrapper'><img src='{url}' alt=':{n}:' title=':{n}:' class='reaction-emoji-custom'><span class='count'>{c}</span></span>" | ||||
| 			].join("")); | ||||
| 		}, | ||||
| @@ -207,7 +210,7 @@ const DISCORD = (function() { | ||||
| 		}, | ||||
| 		 | ||||
| 		isImageAttachment(attachment) { | ||||
| 			const url = DOM.tryParseUrl(attachment.url); | ||||
| 			const url = dom.tryParseUrl(attachment.url); | ||||
| 			return url != null && isImageUrl(url); | ||||
| 		}, | ||||
| 		 | ||||
| @@ -220,7 +223,7 @@ const DISCORD = (function() { | ||||
| 		}, | ||||
| 		 | ||||
| 		getMessageHTML(message) { // noinspection FunctionWithInconsistentReturnsJS
 | ||||
| 			return (SETTINGS.enableUserAvatars ? templateMessageWithAvatar : templateMessageNoAvatar).apply(message, (property, value) => { | ||||
| 			return (settings.enableUserAvatars ? templateMessageWithAvatar : templateMessageNoAvatar).apply(message, (property, value) => { | ||||
| 				if (property === "avatar") { | ||||
| 					return value ? templateUserAvatar.apply(getAvatarUrlObject(value)) : ""; | ||||
| 				} | ||||
| @@ -228,7 +231,7 @@ const DISCORD = (function() { | ||||
| 					return value ? value : "????"; | ||||
| 				} | ||||
| 				else if (property === "timestamp") { | ||||
| 					return DOM.getHumanReadableTime(value); | ||||
| 					return dom.getHumanReadableTime(value); | ||||
| 				} | ||||
| 				else if (property === "contents") { | ||||
| 					return value && value.length > 0 ? processMessageContents(value) : ""; | ||||
| @@ -267,7 +270,7 @@ const DISCORD = (function() { | ||||
| 					return value.map(attachment => { | ||||
| 						const url = fileUrlProcessor(attachment.url); | ||||
| 						 | ||||
| 						if (!DISCORD.isImageAttachment(attachment) || !SETTINGS.enableImagePreviews) { | ||||
| 						if (!discord.isImageAttachment(attachment) || !settings.enableImagePreviews) { | ||||
| 							return templateAttachmentDownload.apply({ url, name: attachment.name }); | ||||
| 						} | ||||
| 						else if ("width" in attachment && "height" in attachment) { | ||||
| @@ -279,10 +282,10 @@ const DISCORD = (function() { | ||||
| 					}).join(""); | ||||
| 				} | ||||
| 				else if (property === "edit") { | ||||
| 					return value ? "<span class='info edited'>Edited " + DOM.getHumanReadableTime(value) + "</span>" : ""; | ||||
| 					return value ? "<span class='info edited'>Edited " + dom.getHumanReadableTime(value) + "</span>" : ""; | ||||
| 				} | ||||
| 				else if (property === "jump") { | ||||
| 					return STATE.hasActiveFilter ? "<span class='info jump' data-jump='" + value + "'>Jump to message</span>" : ""; | ||||
| 					return state.hasActiveFilter ? "<span class='info jump' data-jump='" + value + "'>Jump to message</span>" : ""; | ||||
| 				} | ||||
| 				else if (property === "reply") { | ||||
| 					if (!value) { | ||||
| @@ -290,7 +293,7 @@ const DISCORD = (function() { | ||||
| 					} | ||||
| 					 | ||||
| 					const user = "<span class='reply-username' title='#" + (value.user.tag ? value.user.tag : "????") + "'>" + value.user.name + "</span>"; | ||||
| 					const avatar = SETTINGS.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(getAvatarUrlObject(value.avatar)) + "</span>" : ""; | ||||
| 					const avatar = settings.enableUserAvatars && value.avatar ? "<span class='reply-avatar'>" + templateUserAvatar.apply(getAvatarUrlObject(value.avatar)) + "</span>" : ""; | ||||
| 					const contents = value.contents ? "<span class='reply-contents'>" + processMessageContents(value.contents) + "</span>" : ""; | ||||
| 					 | ||||
| 					return "<span class='jump' data-jump='" + value.id + "'>Jump to reply</span><span class='user'>" + avatar + user + "</span>" + contents; | ||||
| @@ -302,7 +305,7 @@ const DISCORD = (function() { | ||||
| 					 | ||||
| 					return "<div class='reactions'>" + value.map(reaction => { | ||||
| 						if ("id" in reaction){ | ||||
| 							const ext = reaction.a && SETTINGS.enableAnimatedEmoji ? "gif" : "webp"; | ||||
| 							const ext = reaction.a && settings.enableAnimatedEmoji ? "gif" : "webp"; | ||||
| 							const url = fileUrlProcessor("https://cdn.discordapp.com/emojis/" + reaction.id + "." + ext); | ||||
| 							// noinspection JSUnusedGlobalSymbols
 | ||||
| 							return templateReactionCustom.apply({ url, n: reaction.n, c: reaction.c }); | ||||
| @@ -8,9 +8,12 @@ const HTML_ENTITY_MAP = { | ||||
| 
 | ||||
| const HTML_ENTITY_REGEX = /[&<>"']/g; | ||||
| 
 | ||||
| class DOM { | ||||
| export default class { | ||||
| 	/** | ||||
| 	 * Returns a child element by its ID. Parent defaults to the entire document. | ||||
| 	 * @param {string} id | ||||
| 	 * @param {HTMLElement} [parent] | ||||
| 	 * @returns {HTMLElement|null} | ||||
| 	 */ | ||||
| 	static id(id, parent) { | ||||
| 		return (parent || document).getElementById(id); | ||||
| @@ -18,6 +21,9 @@ class DOM { | ||||
| 	 | ||||
| 	/** | ||||
| 	 * Returns an array of all child elements containing the specified class. Parent defaults to the entire document. | ||||
| 	 * @param {string} cls | ||||
| 	 * @param {HTMLElement} [parent] | ||||
| 	 * @returns {HTMLElement[]} | ||||
| 	 */ | ||||
| 	static cls(cls, parent) { | ||||
| 		return Array.prototype.slice.call((parent || document).getElementsByClassName(cls)); | ||||
| @@ -25,6 +31,9 @@ class DOM { | ||||
| 	 | ||||
| 	/** | ||||
| 	 * Returns an array of all child elements that have the specified tag. Parent defaults to the entire document. | ||||
| 	 * @param {string} tag | ||||
| 	 * @param {HTMLElement} [parent] | ||||
| 	 * @returns {HTMLElement[]} | ||||
| 	 */ | ||||
| 	static tag(tag, parent) { | ||||
| 		return Array.prototype.slice.call((parent || document).getElementsByTagName(tag)); | ||||
| @@ -32,6 +41,9 @@ class DOM { | ||||
| 	 | ||||
| 	/** | ||||
| 	 * Returns the first child element containing the specified class. Parent defaults to the entire document. | ||||
| 	 * @param {string} cls | ||||
| 	 * @param {HTMLElement} [parent] | ||||
| 	 * @returns {HTMLElement} | ||||
| 	 */ | ||||
| 	static fcls(cls, parent) { | ||||
| 		return (parent || document).getElementsByClassName(cls)[0]; | ||||
| @@ -1,10 +1,16 @@ | ||||
| const GUI = (function() { | ||||
| import dom from "./dom.mjs"; | ||||
| import discord from "./discord.mjs"; | ||||
| import settings from "./settings.mjs"; | ||||
| import state from "./state.mjs"; | ||||
| 
 | ||||
| export default (function() { | ||||
| 	let eventOnOptMessagesPerPageChanged; | ||||
| 	let eventOnOptMessageFilterChanged; | ||||
| 	let eventOnNavButtonClicked; | ||||
| 	 | ||||
| 	const getActiveFilter = function() { | ||||
| 		const active = DOM.fcls("active", DOM.id("opt-filter-list")); | ||||
| 		/** @type HTMLSelectElement */ | ||||
| 		const active = dom.fcls("active", dom.id("opt-filter-list")); | ||||
| 		 | ||||
| 		return active && active.value !== "" ? { | ||||
| 			"type": active.getAttribute("data-filter-type"), | ||||
| @@ -17,12 +23,12 @@ const GUI = (function() { | ||||
| 	}; | ||||
| 	 | ||||
| 	const showModal = function(width, html) { | ||||
| 		const dialog = DOM.id("dialog"); | ||||
| 		const dialog = dom.id("dialog"); | ||||
| 		dialog.innerHTML = html; | ||||
| 		dialog.style.width = width + "px"; | ||||
| 		dialog.style.marginLeft = (-width / 2) + "px"; | ||||
| 		 | ||||
| 		DOM.id("modal").classList.add("visible"); | ||||
| 		dom.id("modal").classList.add("visible"); | ||||
| 		return dialog; | ||||
| 	}; | ||||
| 	 | ||||
| @@ -38,9 +44,9 @@ const GUI = (function() { | ||||
| <label><input id='dht-cfg-animemoji' type='checkbox'> Animated Emoji</label><br>`); | ||||
| 		 | ||||
| 		const setupCheckBox = function(id, settingName) { | ||||
| 			const ele = DOM.id(id); | ||||
| 			ele.checked = SETTINGS[settingName]; | ||||
| 			ele.addEventListener("change", () => SETTINGS[settingName] = ele.checked); | ||||
| 			const ele = dom.id(id); | ||||
| 			ele.checked = settings[settingName]; | ||||
| 			ele.addEventListener("change", () => settings[settingName] = ele.checked); | ||||
| 		}; | ||||
| 		 | ||||
| 		setupCheckBox("dht-cfg-imgpreviews", "enableImagePreviews"); | ||||
| @@ -65,20 +71,20 @@ const GUI = (function() { | ||||
| 		// ---------
 | ||||
| 		 | ||||
| 		setup() { | ||||
| 			const inputMessageFilter = DOM.id("opt-messages-filter"); | ||||
| 			const containerFilterList = DOM.id("opt-filter-list"); | ||||
| 			const inputMessageFilter = dom.id("opt-messages-filter"); | ||||
| 			const containerFilterList = dom.id("opt-filter-list"); | ||||
| 			 | ||||
| 			const resetActiveFilter = function() { | ||||
| 				inputMessageFilter.value = ""; | ||||
| 				inputMessageFilter.dispatchEvent(new Event("change")); | ||||
| 				 | ||||
| 				DOM.id("opt-filter-contents").value = ""; | ||||
| 				dom.id("opt-filter-contents").value = ""; | ||||
| 			}; | ||||
| 			 | ||||
| 			inputMessageFilter.value = ""; // required to prevent browsers from remembering old value
 | ||||
| 			 | ||||
| 			inputMessageFilter.addEventListener("change", () => { | ||||
| 				DOM.cls("active", containerFilterList).forEach(ele => ele.classList.remove("active")); | ||||
| 				dom.cls("active", containerFilterList).forEach(ele => ele.classList.remove("active")); | ||||
| 				 | ||||
| 				if (inputMessageFilter.value) { | ||||
| 					containerFilterList.querySelector("[data-filter-type='" + inputMessageFilter.value + "']").classList.add("active"); | ||||
| @@ -91,11 +97,11 @@ const GUI = (function() { | ||||
| 				ele.addEventListener(ele.tagName === "SELECT" ? "change" : "input", () => triggerFilterChanged()); | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.id("opt-messages-per-page").addEventListener("change", () => { | ||||
| 			dom.id("opt-messages-per-page").addEventListener("change", () => { | ||||
| 				eventOnOptMessagesPerPageChanged && eventOnOptMessagesPerPageChanged(); | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.tag("button", DOM.fcls("nav")).forEach(button => { | ||||
| 			dom.tag("button", dom.fcls("nav")).forEach(button => { | ||||
| 				button.disabled = true; | ||||
| 				 | ||||
| 				button.addEventListener("click", () => { | ||||
| @@ -103,34 +109,34 @@ const GUI = (function() { | ||||
| 				}); | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.id("btn-settings").addEventListener("click", () => { | ||||
| 			dom.id("btn-settings").addEventListener("click", () => { | ||||
| 				showSettingsModal(); | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.id("btn-about").addEventListener("click", () => { | ||||
| 			dom.id("btn-about").addEventListener("click", () => { | ||||
| 				showInfoModal(); | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.id("messages").addEventListener("click", e => { | ||||
| 			dom.id("messages").addEventListener("click", e => { | ||||
| 				const jump = e.target.getAttribute("data-jump"); | ||||
| 				 | ||||
| 				if (jump) { | ||||
| 					resetActiveFilter(); | ||||
| 					 | ||||
| 					const index = STATE.navigateToMessage(jump); | ||||
| 					const index = state.navigateToMessage(jump); | ||||
| 					 | ||||
| 					if (index === -1) { | ||||
| 						alert("Message not found."); | ||||
| 					} | ||||
| 					else { | ||||
| 						DOM.id("messages").children[index].scrollIntoView(); | ||||
| 						dom.id("messages").children[index].scrollIntoView(); | ||||
| 					} | ||||
| 				} | ||||
| 			}); | ||||
| 			 | ||||
| 			DOM.id("overlay").addEventListener("click", () => { | ||||
| 				DOM.id("modal").classList.remove("visible"); | ||||
| 				DOM.id("dialog").innerHTML = ""; | ||||
| 			dom.id("overlay").addEventListener("click", () => { | ||||
| 				dom.id("modal").classList.remove("visible"); | ||||
| 				dom.id("dialog").innerHTML = ""; | ||||
| 			}); | ||||
| 		}, | ||||
| 		 | ||||
| @@ -168,19 +174,19 @@ const GUI = (function() { | ||||
| 		 */ | ||||
| 		getOptionMessagesPerPage() { | ||||
| 			/** @type HTMLInputElement */ | ||||
| 			const messagesPerPage = DOM.id("opt-messages-per-page"); | ||||
| 			const messagesPerPage = dom.id("opt-messages-per-page"); | ||||
| 			return parseInt(messagesPerPage.value, 10); | ||||
| 		}, | ||||
| 		 | ||||
| 		updateNavigation(currentPage, totalPages) { | ||||
| 			DOM.id("nav-page-current").innerHTML = currentPage; | ||||
| 			DOM.id("nav-page-total").innerHTML = totalPages || "?"; | ||||
| 			dom.id("nav-page-current").innerHTML = currentPage; | ||||
| 			dom.id("nav-page-total").innerHTML = totalPages || "?"; | ||||
| 			 | ||||
| 			DOM.id("nav-first").disabled = currentPage === 1; | ||||
| 			DOM.id("nav-prev").disabled = currentPage === 1; | ||||
| 			DOM.id("nav-pick").disabled = (totalPages || 0) <= 1; | ||||
| 			DOM.id("nav-next").disabled = currentPage === (totalPages || 1); | ||||
| 			DOM.id("nav-last").disabled = currentPage === (totalPages || 1); | ||||
| 			dom.id("nav-first").disabled = currentPage === 1; | ||||
| 			dom.id("nav-prev").disabled = currentPage === 1; | ||||
| 			dom.id("nav-pick").disabled = (totalPages || 0) <= 1; | ||||
| 			dom.id("nav-next").disabled = currentPage === (totalPages || 1); | ||||
| 			dom.id("nav-last").disabled = currentPage === (totalPages || 1); | ||||
| 		}, | ||||
| 		 | ||||
| 		// --------------
 | ||||
| @@ -191,7 +197,7 @@ const GUI = (function() { | ||||
| 		 * Updates the channel list and sets up their click events. The callback is triggered whenever a channel is selected, and takes the channel ID as its argument. | ||||
| 		 */ | ||||
| 		updateChannelList(channels, selected, callback) { | ||||
| 			const eleChannels = DOM.id("channels"); | ||||
| 			const eleChannels = dom.id("channels"); | ||||
| 			 | ||||
| 			if (!channels) { | ||||
| 				eleChannels.innerHTML = ""; | ||||
| @@ -201,11 +207,11 @@ const GUI = (function() { | ||||
| 					channels = channels.filter(channel => channel.msgcount > 0); | ||||
| 				} | ||||
| 				 | ||||
| 				eleChannels.innerHTML = channels.map(channel => DISCORD.getChannelHTML(channel)).join(""); | ||||
| 				eleChannels.innerHTML = channels.map(channel => discord.getChannelHTML(channel)).join(""); | ||||
| 				 | ||||
| 				Array.prototype.forEach.call(eleChannels.children, ele => { | ||||
| 					ele.addEventListener("click", () => { | ||||
| 						const currentChannel = DOM.fcls("active", eleChannels); | ||||
| 						const currentChannel = dom.fcls("active", eleChannels); | ||||
| 						 | ||||
| 						if (currentChannel) { | ||||
| 							currentChannel.classList.remove("active"); | ||||
| @@ -224,12 +230,12 @@ const GUI = (function() { | ||||
| 		}, | ||||
| 		 | ||||
| 		updateMessageList(messages) { | ||||
| 			DOM.id("messages").innerHTML = messages ? messages.map(message => DISCORD.getMessageHTML(message)).join("") : ""; | ||||
| 			dom.id("messages").innerHTML = messages ? messages.map(message => discord.getMessageHTML(message)).join("") : ""; | ||||
| 		}, | ||||
| 		 | ||||
| 		updateUserList(users) { | ||||
| 			/** @type HTMLSelectElement */ | ||||
| 			const eleSelect = DOM.id("opt-filter-user"); | ||||
| 			const eleSelect = dom.id("opt-filter-user"); | ||||
| 			 | ||||
| 			while (eleSelect.length > 1) { | ||||
| 				eleSelect.remove(1); | ||||
| @@ -249,7 +255,7 @@ const GUI = (function() { | ||||
| 		}, | ||||
| 		 | ||||
| 		scrollMessagesToTop() { | ||||
| 			DOM.id("messages").scrollTop = 0; | ||||
| 			dom.id("messages").scrollTop = 0; | ||||
| 		} | ||||
| 	}; | ||||
| })(); | ||||
| @@ -1,16 +1,16 @@ | ||||
| const PROCESSOR = {}; | ||||
| import discord from "./discord.mjs"; | ||||
| 
 | ||||
| // ------------------------
 | ||||
| // Global filter generators
 | ||||
| // ------------------------
 | ||||
| 
 | ||||
| PROCESSOR.FILTER = { | ||||
| const filter = { | ||||
| 	byUser: ((userindex) => message => message.u === userindex), | ||||
| 	byTime: ((timeStart, timeEnd) => message => message.t >= timeStart && message.t <= timeEnd), | ||||
| 	byContents: ((substr) => message => ("m" in message ? message.m : "").indexOf(substr) !== -1), | ||||
| 	byRegex: ((regex) => message => regex.test("m" in message ? message.m : "")), | ||||
| 	withImages: (() => message => (message.e && message.e.some(embed => embed.type === "image")) || (message.a && message.a.some(DISCORD.isImageAttachment))), | ||||
| 	withDownloads: (() => message => message.a && message.a.some(attachment => !DISCORD.isImageAttachment(attachment))), | ||||
| 	withImages: (() => message => (message.e && message.e.some(embed => embed.type === "image")) || (message.a && message.a.some(discord.isImageAttachment))), | ||||
| 	withDownloads: (() => message => message.a && message.a.some(attachment => !discord.isImageAttachment(attachment))), | ||||
| 	withEmbeds: (() => message => message.e && message.e.length > 0), | ||||
| 	withAttachments: (() => message => message.a && message.a.length > 0), | ||||
| 	isEdited: (() => message => ("te" in message) ? message.te : (message.f & 1) === 1) | ||||
| @@ -20,7 +20,7 @@ PROCESSOR.FILTER = { | ||||
| // Global sorters
 | ||||
| // --------------
 | ||||
| 
 | ||||
| PROCESSOR.SORTER = { | ||||
| const sorter = { | ||||
| 	oldestToNewest: (key1, key2) => { | ||||
| 		if (key1.length === key2.length) { | ||||
| 			return key1 > key2 ? 1 : key1 < key2 ? -1 : 0; | ||||
| @@ -39,3 +39,8 @@ PROCESSOR.SORTER = { | ||||
| 		} | ||||
| 	} | ||||
| }; | ||||
| 
 | ||||
| export default { | ||||
| 	FILTER: filter, | ||||
| 	SORTER: sorter | ||||
| }; | ||||
| @@ -1,4 +1,4 @@ | ||||
| const SETTINGS = (function() { | ||||
| export default (function() { | ||||
| 	/** | ||||
| 	 * @type {{}} | ||||
| 	 * @property {Function} onSettingsChanged | ||||
| @@ -1,5 +1,8 @@ | ||||
| import settings from "./settings.mjs"; | ||||
| import processor from "./processor.mjs"; | ||||
| 
 | ||||
| // noinspection FunctionWithInconsistentReturnsJS
 | ||||
| const STATE = (function() { | ||||
| export default (function() { | ||||
| 	/** | ||||
| 	 * @type {{}} | ||||
| 	 * @property {{}} users | ||||
| @@ -173,7 +176,7 @@ const STATE = (function() { | ||||
| 	}; | ||||
| 	 | ||||
| 	const getMessageChannel = function(id) { | ||||
| 		for (const [ channel, messages ] of Object.entries(loadedFileData)) { | ||||
| 		for (const [ channel, messages ] of Object.entries(loadedFileData)) {  | ||||
| 			if (id in messages) { | ||||
| 				return channel; | ||||
| 			} | ||||
| @@ -313,7 +316,7 @@ const STATE = (function() { | ||||
| 			triggerChannelsRefreshed(); | ||||
| 			triggerMessagesRefreshed(); | ||||
| 			 | ||||
| 			SETTINGS.onSettingsChanged(() => triggerMessagesRefreshed()); | ||||
| 			settings.onSettingsChanged(() => triggerMessagesRefreshed()); | ||||
| 		}, | ||||
| 		 | ||||
| 		getChannelName(channel) { | ||||
| @@ -335,7 +338,7 @@ const STATE = (function() { | ||||
| 			currentPage = 1; | ||||
| 			selectedChannel = channel; | ||||
| 			 | ||||
| 			loadedMessages = getFilteredMessageKeys(channel).sort(PROCESSOR.SORTER.oldestToNewest); | ||||
| 			loadedMessages = getFilteredMessageKeys(channel).sort(processor.SORTER.oldestToNewest); | ||||
| 			triggerMessagesRefreshed(); | ||||
| 		}, | ||||
| 		 | ||||
| @@ -416,23 +419,23 @@ const STATE = (function() { | ||||
| 		setActiveFilter(filter) { | ||||
| 			switch (filter ? filter.type : "") { | ||||
| 				case "user": | ||||
| 					filterFunction = PROCESSOR.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value)); | ||||
| 					filterFunction = processor.FILTER.byUser(loadedFileMeta.userindex.indexOf(filter.value)); | ||||
| 					break; | ||||
| 				 | ||||
| 				case "contents": | ||||
| 					filterFunction = PROCESSOR.FILTER.byContents(filter.value); | ||||
| 					filterFunction = processor.FILTER.byContents(filter.value); | ||||
| 					break; | ||||
| 				 | ||||
| 				case "withimages": | ||||
| 					filterFunction = PROCESSOR.FILTER.withImages(); | ||||
| 					filterFunction = processor.FILTER.withImages(); | ||||
| 					break; | ||||
| 				 | ||||
| 				case "withdownloads": | ||||
| 					filterFunction = PROCESSOR.FILTER.withDownloads(); | ||||
| 					filterFunction = processor.FILTER.withDownloads(); | ||||
| 					break; | ||||
| 				 | ||||
| 				case "edited": | ||||
| 					filterFunction = PROCESSOR.FILTER.isEdited(); | ||||
| 					filterFunction = processor.FILTER.isEdited(); | ||||
| 					break; | ||||
| 				 | ||||
| 				default: | ||||
| @@ -1,6 +1,8 @@ | ||||
| import dom from "./dom.mjs"; | ||||
| 
 | ||||
| const TEMPLATE_REGEX = /{([^{}]+?)}/g; | ||||
| 
 | ||||
| class TEMPLATE { | ||||
| export default class { | ||||
| 	constructor(contents) { | ||||
| 		this.contents = contents; | ||||
| 	}; | ||||
| @@ -11,10 +13,10 @@ class TEMPLATE { | ||||
| 			 | ||||
| 			if (processor) { | ||||
| 				const updated = processor(match, value); | ||||
| 				return typeof updated === "undefined" ? DOM.escapeHTML(value) : updated; | ||||
| 				return typeof updated === "undefined" ? dom.escapeHTML(value) : updated; | ||||
| 			} | ||||
| 			 | ||||
| 			return DOM.escapeHTML(value); | ||||
| 			return dom.escapeHTML(value); | ||||
| 		}); | ||||
| 	} | ||||
| } | ||||
| @@ -3,18 +3,24 @@ | ||||
|   min-width: 215px; | ||||
|   max-width: 300px; | ||||
|   overflow-y: auto; | ||||
|   color: #eee; | ||||
|   background-color: #1c1e22; | ||||
|   font-size: 15px; | ||||
| } | ||||
|  | ||||
| #channels > div { | ||||
| #channels > div.loading { | ||||
|   margin: 0 auto; | ||||
|   width: 150px; | ||||
|   height: 150px; | ||||
| } | ||||
|  | ||||
| #channels > div.channel { | ||||
|   cursor: pointer; | ||||
|   padding: 10px 12px; | ||||
|   color: #eee; | ||||
|   font-size: 15px; | ||||
|   border-bottom: 1px solid #333333; | ||||
| } | ||||
|  | ||||
| #channels > div:hover, #channels > div.active { | ||||
| #channels > div.channel:hover, #channels > div.channel.active { | ||||
|   background-color: #282b30; | ||||
| } | ||||
|  | ||||
|   | ||||
| @@ -11,3 +11,20 @@ body { | ||||
|   display: flex; | ||||
|   flex-direction: row; | ||||
| } | ||||
|  | ||||
| .loading { | ||||
|   position: relative; | ||||
|   --loading-backdrop: rgba(0, 0, 0, 0); | ||||
| } | ||||
|  | ||||
| .loading::after { | ||||
|   content: ""; | ||||
|   background: var(--loading-backdrop) | ||||
|   url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 300' preserveAspectRatio='xMidYMid'%3E %3Ccircle cx='150' cy='150' fill='none' stroke='%237983f5' stroke-width='8' r='42' stroke-dasharray='198 68'%3E %3CanimateTransform attributeName='transform' type='rotate' repeatCount='indefinite' dur='1.25s' values='0 150 150;360 150 150' keyTimes='0;1' /%3E %3C/circle%3E %3C/svg%3E") | ||||
|   no-repeat center center; | ||||
|   position: absolute; | ||||
|   top: 0; | ||||
|   left: 0; | ||||
|   width: 100%; | ||||
|   height: 100%; | ||||
| } | ||||
|   | ||||
| @@ -107,23 +107,11 @@ | ||||
| } | ||||
|  | ||||
| .message .thumbnail { | ||||
|   position: relative; | ||||
|   --loading-backdrop: rgba(0, 0, 0, 0.75); | ||||
|   max-width: calc(100% - 20px); | ||||
|   max-height: 320px; | ||||
| } | ||||
|  | ||||
| .message .thumbnail.loading::after { | ||||
|   content: ""; | ||||
|   background: rgba(0, 0, 0, 0.75) | ||||
|               url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 300' preserveAspectRatio='xMidYMid'%3E %3Ccircle cx='150' cy='150' fill='none' stroke='%237983f5' stroke-width='8' r='42' stroke-dasharray='198 68'%3E %3CanimateTransform attributeName='transform' type='rotate' repeatCount='indefinite' dur='1.25s' values='0 150 150;360 150 150' keyTimes='0;1' /%3E %3C/circle%3E %3C/svg%3E") | ||||
|               no-repeat center center; | ||||
|   position: absolute; | ||||
|   top: 0; | ||||
|   left: 0; | ||||
|   width: 100%; | ||||
|   height: 100%; | ||||
| } | ||||
|  | ||||
| .message .thumbnail img { | ||||
|   width: auto; | ||||
|   max-width: 100%; | ||||
|   | ||||
| @@ -1,3 +0,0 @@ | ||||
| namespace DHT.Server.Data; | ||||
|  | ||||
| public readonly record struct DownloadWithData(Download Download, byte[]? Data); | ||||
| @@ -26,7 +26,9 @@ public static class DatabaseExtensions { | ||||
| 		await target.Messages.Add(batchedMessages); | ||||
|  | ||||
| 		await foreach (var download in source.Downloads.Get()) { | ||||
| 			await target.Downloads.AddDownload(await source.Downloads.HydrateWithData(download)); | ||||
| 			if (download.Status != DownloadStatus.Success || !await source.Downloads.GetDownloadData(download.NormalizedUrl, stream => target.Downloads.AddDownload(download, stream))) { | ||||
| 				await target.Downloads.AddDownload(download, stream: null); | ||||
| 			} | ||||
| 		} | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -10,7 +10,7 @@ using DHT.Utils.Logging; | ||||
|  | ||||
| namespace DHT.Server.Database.Export; | ||||
|  | ||||
| public static class ViewerJsonExport { | ||||
| static class ViewerJsonExport { | ||||
| 	private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport)); | ||||
|  | ||||
| 	public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) { | ||||
|   | ||||
| @@ -161,7 +161,7 @@ public static class LegacyArchiveImport { | ||||
|  | ||||
| 		var messagesObj = data.HasKey(channelIdStr) ? data.RequireObject(channelIdStr, DataPath) : (JsonElement?) null; | ||||
| 		if (messagesObj == null) { | ||||
| 			return Array.Empty<Message>(); | ||||
| 			return []; | ||||
| 		} | ||||
|  | ||||
| 		return messagesObj.Value.EnumerateObject().Select(item => { | ||||
|   | ||||
| @@ -1,10 +1,10 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Linq; | ||||
| using System.Reactive.Linq; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using DHT.Server.Data; | ||||
| using DHT.Server.Data.Aggregations; | ||||
| using DHT.Server.Data.Filters; | ||||
| using DHT.Server.Download; | ||||
| @@ -14,7 +14,7 @@ namespace DHT.Server.Database.Repositories; | ||||
| public interface IDownloadRepository { | ||||
| 	IObservable<long> TotalCount { get; } | ||||
|  | ||||
| 	Task AddDownload(DownloadWithData item); | ||||
| 	Task AddDownload(Data.Download item, Stream? stream); | ||||
|  | ||||
| 	Task<long> Count(DownloadItemFilter filter, CancellationToken cancellationToken = default); | ||||
| 	 | ||||
| @@ -22,9 +22,9 @@ public interface IDownloadRepository { | ||||
| 	 | ||||
| 	IAsyncEnumerable<Data.Download> Get(); | ||||
|  | ||||
| 	Task<DownloadWithData> HydrateWithData(Data.Download download); | ||||
|  | ||||
| 	Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl); | ||||
| 	Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor); | ||||
| 	 | ||||
| 	Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor); | ||||
|  | ||||
| 	IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default); | ||||
| 	 | ||||
| @@ -35,7 +35,7 @@ public interface IDownloadRepository { | ||||
| 	internal sealed class Dummy : IDownloadRepository { | ||||
| 		public IObservable<long> TotalCount { get; } = Observable.Return(0L); | ||||
|  | ||||
| 		public Task AddDownload(DownloadWithData item) { | ||||
| 		public Task AddDownload(Data.Download item, Stream? stream) { | ||||
| 			return Task.CompletedTask; | ||||
| 		} | ||||
|  | ||||
| @@ -51,12 +51,12 @@ public interface IDownloadRepository { | ||||
| 			return AsyncEnumerable.Empty<Data.Download>(); | ||||
| 		} | ||||
|  | ||||
| 		public Task<DownloadWithData> HydrateWithData(Data.Download download) { | ||||
| 			return Task.FromResult(new DownloadWithData(download, Data: null)); | ||||
| 		public Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) { | ||||
| 			return Task.FromResult(false); | ||||
| 		} | ||||
|  | ||||
| 		public Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) { | ||||
| 			return Task.FromResult<DownloadWithData?>(null); | ||||
| 		public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) { | ||||
| 			return Task.FromResult(false); | ||||
| 		} | ||||
|  | ||||
| 		public IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken) { | ||||
|   | ||||
| @@ -19,9 +19,9 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository | ||||
| 	} | ||||
|  | ||||
| 	public async Task Add(IReadOnlyList<Channel> channels) { | ||||
| 		await using var conn = await pool.Take(); | ||||
|  | ||||
| 		await using (var tx = await conn.BeginTransactionAsync()) { | ||||
| 		await using (var conn = await pool.Take()) { | ||||
| 			await conn.BeginTransactionAsync(); | ||||
| 			 | ||||
| 			await using var cmd = conn.Upsert("channels", [ | ||||
| 				("id", SqliteType.Integer), | ||||
| 				("server", SqliteType.Integer), | ||||
| @@ -43,7 +43,7 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository | ||||
| 				await cmd.ExecuteNonQueryAsync(); | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 			await conn.CommitTransactionAsync(); | ||||
| 		} | ||||
|  | ||||
| 		UpdateTotalCount(); | ||||
|   | ||||
| @@ -1,5 +1,6 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Runtime.CompilerServices; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| @@ -14,14 +15,8 @@ using Microsoft.Data.Sqlite; | ||||
|  | ||||
| namespace DHT.Server.Database.Sqlite.Repositories; | ||||
|  | ||||
| sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepository { | ||||
| sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRepository(Log), IDownloadRepository { | ||||
| 	private static readonly Log Log = Log.ForType<SqliteDownloadRepository>(); | ||||
| 	 | ||||
| 	private readonly SqliteConnectionPool pool; | ||||
|  | ||||
| 	public SqliteDownloadRepository(SqliteConnectionPool pool) : base(Log) { | ||||
| 		this.pool = pool; | ||||
| 	} | ||||
|  | ||||
| 	internal sealed class NewDownloadCollector : IAsyncDisposable { | ||||
| 		private readonly SqliteDownloadRepository repository; | ||||
| @@ -66,12 +61,10 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	public async Task AddDownload(DownloadWithData item) { | ||||
| 		var (download, data) = item; | ||||
|  | ||||
| 	public async Task AddDownload(Data.Download item, Stream? stream) { | ||||
| 		await using (var conn = await pool.Take()) { | ||||
| 			var tx = await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 			await conn.BeginTransactionAsync(); | ||||
| 			 | ||||
| 			await using var metadataCmd = conn.Upsert("download_metadata", [ | ||||
| 				("normalized_url", SqliteType.Text), | ||||
| 				("download_url", SqliteType.Text), | ||||
| @@ -80,30 +73,37 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor | ||||
| 				("size", SqliteType.Integer), | ||||
| 			]); | ||||
|  | ||||
| 			metadataCmd.Set(":normalized_url", download.NormalizedUrl); | ||||
| 			metadataCmd.Set(":download_url", download.DownloadUrl); | ||||
| 			metadataCmd.Set(":status", (int) download.Status); | ||||
| 			metadataCmd.Set(":type", download.Type); | ||||
| 			metadataCmd.Set(":size", download.Size); | ||||
| 			metadataCmd.Set(":normalized_url", item.NormalizedUrl); | ||||
| 			metadataCmd.Set(":download_url", item.DownloadUrl); | ||||
| 			metadataCmd.Set(":status", (int) item.Status); | ||||
| 			metadataCmd.Set(":type", item.Type); | ||||
| 			metadataCmd.Set(":size", item.Size); | ||||
| 			await metadataCmd.ExecuteNonQueryAsync(); | ||||
|  | ||||
| 			if (data == null) { | ||||
| 			if (stream == null) { | ||||
| 				await using var deleteBlobCmd = conn.Command("DELETE FROM download_blobs WHERE normalized_url = :normalized_url"); | ||||
| 				deleteBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, download.NormalizedUrl); | ||||
| 				deleteBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl); | ||||
| 				await deleteBlobCmd.ExecuteNonQueryAsync(); | ||||
| 			} | ||||
| 			else { | ||||
| 				await using var upsertBlobCmd = conn.Upsert("download_blobs", [ | ||||
| 					("normalized_url", SqliteType.Text), | ||||
| 					("blob", SqliteType.Blob) | ||||
| 				]); | ||||
| 				await using var upsertBlobCmd = conn.Command( | ||||
| 					""" | ||||
| 					INSERT INTO download_blobs (normalized_url, blob) | ||||
| 					VALUES (:normalized_url, ZEROBLOB(:blob_length)) | ||||
| 					ON CONFLICT (normalized_url) DO UPDATE SET blob = excluded.blob | ||||
| 					RETURNING rowid | ||||
| 					""" | ||||
| 				); | ||||
| 				 | ||||
| 				upsertBlobCmd.AddAndSet(":normalized_url", SqliteType.Text, item.NormalizedUrl); | ||||
| 				upsertBlobCmd.AddAndSet(":blob_length", SqliteType.Integer, item.Size); | ||||
| 				long rowid = await upsertBlobCmd.ExecuteLongScalarAsync(); | ||||
|  | ||||
| 				upsertBlobCmd.Set(":normalized_url", download.NormalizedUrl); | ||||
| 				upsertBlobCmd.Set(":blob", data); | ||||
| 				await upsertBlobCmd.ExecuteNonQueryAsync(); | ||||
| 				await using var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid); | ||||
| 				await stream.CopyToAsync(blob); | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 			await conn.CommitTransactionAsync(); | ||||
| 		} | ||||
|  | ||||
| 		UpdateTotalCount(); | ||||
| @@ -187,24 +187,35 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	public async Task<DownloadWithData> HydrateWithData(Data.Download download) { | ||||
| 	public async Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor) { | ||||
| 		await using var conn = await pool.Take(); | ||||
|  | ||||
| 		await using var cmd = conn.Command("SELECT blob FROM download_blobs WHERE normalized_url = :url"); | ||||
| 		cmd.AddAndSet(":url", SqliteType.Text, download.NormalizedUrl); | ||||
|  | ||||
| 		await using var reader = await cmd.ExecuteReaderAsync(); | ||||
| 		var data = await reader.ReadAsync() && !reader.IsDBNull(0) ? (byte[]) reader["blob"] : null; | ||||
| 		await using var cmd = conn.Command("SELECT rowid FROM download_blobs WHERE normalized_url = :normalized_url"); | ||||
| 		cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl); | ||||
| 		 | ||||
| 		return new DownloadWithData(download, data); | ||||
| 		long rowid; | ||||
| 		 | ||||
| 		await using (var reader = await cmd.ExecuteReaderAsync()) { | ||||
| 			if (!await reader.ReadAsync()) { | ||||
| 				return false; | ||||
| 			} | ||||
|  | ||||
| 			rowid = reader.GetInt64(0); | ||||
| 		} | ||||
| 		 | ||||
| 		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) { | ||||
| 			await dataProcessor(blob); | ||||
| 		} | ||||
|  | ||||
| 		return true; | ||||
| 	} | ||||
|  | ||||
| 	public async Task<DownloadWithData?> GetSuccessfulDownloadWithData(string normalizedUrl) { | ||||
| 	public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) { | ||||
| 		await using var conn = await pool.Take(); | ||||
|  | ||||
| 		await using var cmd = conn.Command( | ||||
| 			""" | ||||
| 			SELECT dm.download_url, dm.type, db.blob FROM download_metadata dm | ||||
| 			SELECT dm.download_url, dm.type, db.rowid FROM download_metadata dm | ||||
| 			JOIN download_blobs db ON dm.normalized_url = db.normalized_url | ||||
| 			WHERE dm.normalized_url = :normalized_url AND dm.status = :success IS NOT NULL | ||||
| 			""" | ||||
| @@ -213,19 +224,25 @@ sealed class SqliteDownloadRepository : BaseSqliteRepository, IDownloadRepositor | ||||
| 		cmd.AddAndSet(":normalized_url", SqliteType.Text, normalizedUrl); | ||||
| 		cmd.AddAndSet(":success", SqliteType.Integer, (int) DownloadStatus.Success); | ||||
|  | ||||
| 		await using var reader = await cmd.ExecuteReaderAsync(); | ||||
| 		string downloadUrl; | ||||
| 		string? type; | ||||
| 		long rowid; | ||||
| 		 | ||||
| 		await using (var reader = await cmd.ExecuteReaderAsync()) { | ||||
| 			if (!await reader.ReadAsync()) { | ||||
| 				return false; | ||||
| 			} | ||||
|  | ||||
| 		if (!await reader.ReadAsync()) { | ||||
| 			return null; | ||||
| 			downloadUrl = reader.GetString(0); | ||||
| 			type = reader.IsDBNull(1) ? null : reader.GetString(1); | ||||
| 			rowid = reader.GetInt64(2); | ||||
| 		} | ||||
| 		 | ||||
| 		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) { | ||||
| 			await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob); | ||||
| 		} | ||||
|  | ||||
| 		var downloadUrl = reader.GetString(0); | ||||
| 		var type = reader.IsDBNull(1) ? null : reader.GetString(1); | ||||
| 		var data = (byte[]) reader[2]; | ||||
| 		var size = (ulong) data.LongLength; | ||||
| 		var download = new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, size); | ||||
| 		 | ||||
| 		return new DownloadWithData(download, data); | ||||
| 		return true; | ||||
| 	} | ||||
|  | ||||
| 	public async IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, [EnumeratorCancellation] CancellationToken cancellationToken) { | ||||
|   | ||||
| @@ -39,7 +39,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 		} | ||||
|  | ||||
| 		await using (var conn = await pool.Take()) { | ||||
| 			await using var tx = await conn.BeginTransactionAsync(); | ||||
| 			await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 			await using var messageCmd = conn.Upsert("messages", [ | ||||
| 				("message_id", SqliteType.Integer), | ||||
| @@ -167,7 +167,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 				} | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 			await conn.CommitTransactionAsync(); | ||||
| 			downloadCollector.OnCommitted(); | ||||
| 		} | ||||
|  | ||||
| @@ -183,11 +183,11 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM messages" + filter.GenerateConditions().BuildWhereClause(), static reader => reader?.GetInt64(0) ?? 0L, cancellationToken); | ||||
| 	} | ||||
|  | ||||
| 	private sealed class MesageToManyCommand<T> : IAsyncDisposable { | ||||
| 	private sealed class MessageToManyCommand<T> : IAsyncDisposable { | ||||
| 		private readonly SqliteCommand cmd; | ||||
| 		private readonly Func<SqliteDataReader, T> readItem; | ||||
|  | ||||
| 		public MesageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) { | ||||
| 		public MessageToManyCommand(ISqliteConnection conn, string sql, Func<SqliteDataReader, T> readItem) { | ||||
| 			this.cmd = conn.Command(sql); | ||||
| 			this.cmd.Add(":message_id", SqliteType.Integer); | ||||
|  | ||||
| @@ -223,7 +223,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 			WHERE message_id = :message_id | ||||
| 			"""; | ||||
|  | ||||
| 		await using var attachmentCmd = new MesageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment { | ||||
| 		await using var attachmentCmd = new MessageToManyCommand<Attachment>(conn, AttachmentSql, static reader => new Attachment { | ||||
| 			Id = reader.GetUint64(0), | ||||
| 			Name = reader.GetString(1), | ||||
| 			Type = reader.IsDBNull(2) ? null : reader.GetString(2), | ||||
| @@ -241,7 +241,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 			WHERE message_id = :message_id | ||||
| 			"""; | ||||
|  | ||||
| 		await using var embedCmd = new MesageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed { | ||||
| 		await using var embedCmd = new MessageToManyCommand<Embed>(conn, EmbedSql, static reader => new Embed { | ||||
| 			Json = reader.GetString(0) | ||||
| 		}); | ||||
|  | ||||
| @@ -252,7 +252,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository | ||||
| 			WHERE message_id = :message_id | ||||
| 			"""; | ||||
|  | ||||
| 		await using var reactionsCmd = new MesageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction { | ||||
| 		await using var reactionsCmd = new MessageToManyCommand<Reaction>(conn, ReactionSql, static reader => new Reaction { | ||||
| 			EmojiId = reader.IsDBNull(0) ? null : reader.GetUint64(0), | ||||
| 			EmojiName = reader.IsDBNull(1) ? null : reader.GetString(1), | ||||
| 			EmojiFlags = (EmojiFlags) reader.GetInt16(2), | ||||
|   | ||||
| @@ -19,9 +19,9 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository { | ||||
| 	} | ||||
|  | ||||
| 	public async Task Add(IReadOnlyList<Data.Server> servers) { | ||||
| 		await using var conn = await pool.Take(); | ||||
|  | ||||
| 		await using (var tx = await conn.BeginTransactionAsync()) { | ||||
| 		await using (var conn = await pool.Take()) { | ||||
| 			await conn.BeginTransactionAsync(); | ||||
| 			 | ||||
| 			await using var cmd = conn.Upsert("servers", [ | ||||
| 				("id", SqliteType.Integer), | ||||
| 				("name", SqliteType.Text), | ||||
| @@ -35,7 +35,7 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository { | ||||
| 				await cmd.ExecuteNonQueryAsync(); | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 			await conn.CommitTransactionAsync(); | ||||
| 		} | ||||
|  | ||||
| 		UpdateTotalCount(); | ||||
|   | ||||
| @@ -23,7 +23,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository { | ||||
|  | ||||
| 	public async Task Add(IReadOnlyList<User> users) { | ||||
| 		await using (var conn = await pool.Take()) { | ||||
| 			await using var tx = await conn.BeginTransactionAsync(); | ||||
| 			await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 			await using var cmd = conn.Upsert("users", [ | ||||
| 				("id", SqliteType.Integer), | ||||
| @@ -46,7 +46,7 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository { | ||||
| 				} | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 			await conn.CommitTransactionAsync(); | ||||
| 			downloadCollector.OnCommitted(); | ||||
| 		} | ||||
|  | ||||
|   | ||||
| @@ -1,5 +1,4 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Data.Common; | ||||
| using System.Threading.Tasks; | ||||
| using DHT.Server.Database.Sqlite.Utils; | ||||
| using DHT.Server.Download; | ||||
| @@ -23,7 +22,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
| 		await conn.ExecuteAsync("ALTER TABLE attachments RENAME COLUMN url TO normalized_url"); | ||||
| 		await conn.ExecuteAsync("ALTER TABLE downloads RENAME COLUMN url TO normalized_url"); | ||||
| 	} | ||||
| 	 | ||||
|  | ||||
| 	private async Task NormalizeAttachmentUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) { | ||||
| 		await reporter.SubWork("Preparing attachments...", 0, 0); | ||||
|  | ||||
| @@ -39,7 +38,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		await using var tx = await conn.BeginTransactionAsync(); | ||||
| 		await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 		int totalUrls = normalizedUrls.Count; | ||||
| 		int processedUrls = -1; | ||||
| @@ -61,7 +60,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
|  | ||||
| 		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); | ||||
|  | ||||
| 		await tx.CommitAsync(); | ||||
| 		await conn.CommitTransactionAsync(); | ||||
| 	} | ||||
|  | ||||
| 	private async Task NormalizeDownloadUrls(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) { | ||||
| @@ -84,26 +83,23 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
| 		} | ||||
|  | ||||
| 		await conn.ExecuteAsync("PRAGMA cache_size = -20000"); | ||||
| 		await conn.BeginTransactionAsync(); | ||||
| 		 | ||||
| 		await reporter.SubWork("Deleting duplicates...", 0, 0); | ||||
|  | ||||
| 		DbTransaction tx; | ||||
|  | ||||
| 		await using (tx = await conn.BeginTransactionAsync()) { | ||||
| 			await reporter.SubWork("Deleting duplicates...", 0, 0); | ||||
|  | ||||
| 			await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) { | ||||
| 				foreach (var duplicateUrl in duplicateUrlsToDelete) { | ||||
| 					deleteCmd.Set(":url", duplicateUrl); | ||||
| 					await deleteCmd.ExecuteNonQueryAsync(); | ||||
| 				} | ||||
| 		await using (var deleteCmd = conn.Delete("downloads", ("url", SqliteType.Text))) { | ||||
| 			foreach (var duplicateUrl in duplicateUrlsToDelete) { | ||||
| 				deleteCmd.Set(":url", duplicateUrl); | ||||
| 				await deleteCmd.ExecuteNonQueryAsync(); | ||||
| 			} | ||||
|  | ||||
| 			await tx.CommitAsync(); | ||||
| 		} | ||||
|  | ||||
| 		await conn.CommitTransactionAsync(); | ||||
|  | ||||
| 		int totalUrls = normalizedUrlsToOriginalUrls.Count; | ||||
| 		int processedUrls = -1; | ||||
|  | ||||
| 		tx = await conn.BeginTransactionAsync(); | ||||
| 		await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 		await using (var updateCmd = conn.Command("UPDATE downloads SET download_url = :download_url, url = :normalized_url WHERE url = :download_url")) { | ||||
| 			updateCmd.Add(":normalized_url", SqliteType.Text); | ||||
| @@ -115,11 +111,10 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
|  | ||||
| 					// Not proper way of dealing with transactions, but it avoids a long commit at the end. | ||||
| 					// Schema upgrades are already non-atomic anyways, so this doesn't make it worse. | ||||
| 					await tx.CommitAsync(); | ||||
| 					await tx.DisposeAsync(); | ||||
| 					await conn.CommitTransactionAsync(); | ||||
|  | ||||
| 					tx = await conn.BeginTransactionAsync(); | ||||
| 					updateCmd.Transaction = (SqliteTransaction) tx; | ||||
| 					await conn.BeginTransactionAsync(); | ||||
| 					conn.AssignActiveTransaction(updateCmd); | ||||
| 				} | ||||
|  | ||||
| 				updateCmd.Set(":normalized_url", normalizedUrl); | ||||
| @@ -130,8 +125,7 @@ sealed class SqliteSchemaUpgradeTo6 : ISchemaUpgrade { | ||||
|  | ||||
| 		await reporter.SubWork("Updating URLs...", totalUrls, totalUrls); | ||||
|  | ||||
| 		await tx.CommitAsync(); | ||||
| 		await tx.DisposeAsync(); | ||||
| 		await conn.CommitTransactionAsync(); | ||||
|  | ||||
| 		await conn.ExecuteAsync("PRAGMA cache_size = -2000"); | ||||
| 	} | ||||
|   | ||||
| @@ -11,56 +11,55 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade { | ||||
| 	async Task ISchemaUpgrade.Run(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) { | ||||
| 		await reporter.MainWork("Applying schema changes...", 0, 6); | ||||
| 		await SqliteSchema.CreateDownloadTables(conn); | ||||
| 		 | ||||
|  | ||||
| 		await reporter.MainWork("Migrating download metadata...", 1, 6); | ||||
| 		await conn.ExecuteAsync("INSERT INTO download_metadata (normalized_url, download_url, status, size) SELECT normalized_url, download_url, status, size FROM downloads"); | ||||
| 		 | ||||
|  | ||||
| 		await reporter.MainWork("Merging attachment metadata...", 2, 6); | ||||
| 		await conn.ExecuteAsync("UPDATE download_metadata SET type = (SELECT type FROM attachments WHERE download_metadata.normalized_url = attachments.normalized_url)"); | ||||
| 		 | ||||
|  | ||||
| 		await reporter.MainWork("Migrating downloaded files...", 3, 6); | ||||
| 		await MigrateDownloadBlobsToNewTable(conn, reporter); | ||||
| 		 | ||||
|  | ||||
| 		await reporter.MainWork("Applying schema changes...", 4, 6); | ||||
| 		await conn.ExecuteAsync("DROP TABLE downloads"); | ||||
| 		 | ||||
|  | ||||
| 		await reporter.MainWork("Discovering downloadable links...", 5, 6); | ||||
| 		await DiscoverDownloadableLinks(conn, reporter); | ||||
| 	} | ||||
|  | ||||
| 	private async Task MigrateDownloadBlobsToNewTable(ISqliteConnection conn, ISchemaUpgradeCallbacks.IProgressReporter reporter) { | ||||
| 		await reporter.SubWork("Listing downloaded files...", 0, 0); | ||||
| 		 | ||||
|  | ||||
| 		var urlsToMigrate = await GetDownloadedFileUrls(conn); | ||||
| 		int totalFiles = urlsToMigrate.Count; | ||||
| 		int processedFiles = -1; | ||||
|  | ||||
| 		await reporter.SubWork("Processing downloaded files...", 0, totalFiles); | ||||
| 		 | ||||
| 		var tx = await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 		await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 		await using (var insertCmd = conn.Command("INSERT INTO download_blobs (normalized_url, blob) SELECT normalized_url, blob FROM downloads WHERE normalized_url = :normalized_url")) | ||||
| 		await using (var deleteCmd = conn.Command("DELETE FROM downloads WHERE normalized_url = :normalized_url")) { | ||||
| 			insertCmd.Add(":normalized_url", SqliteType.Text); | ||||
| 			deleteCmd.Add(":normalized_url", SqliteType.Text); | ||||
| 			 | ||||
|  | ||||
| 			foreach (var url in urlsToMigrate) { | ||||
| 				if (++processedFiles % 10 == 0) { | ||||
| 					await reporter.SubWork("Processing downloaded files...", processedFiles, totalFiles); | ||||
| 			 | ||||
|  | ||||
| 					// Not proper way of dealing with transactions, but it avoids a long commit at the end. | ||||
| 					// Schema upgrades are already non-atomic anyways, so this doesn't make it worse. | ||||
| 					await tx.CommitAsync(); | ||||
| 					await tx.DisposeAsync(); | ||||
| 			 | ||||
| 					tx = await conn.BeginTransactionAsync(); | ||||
| 					insertCmd.Transaction = (SqliteTransaction) tx; | ||||
| 					deleteCmd.Transaction = (SqliteTransaction) tx; | ||||
| 					await conn.CommitTransactionAsync(); | ||||
| 					 | ||||
| 					await conn.BeginTransactionAsync(); | ||||
| 					conn.AssignActiveTransaction(insertCmd); | ||||
| 					conn.AssignActiveTransaction(deleteCmd); | ||||
| 				} | ||||
| 				 | ||||
|  | ||||
| 				insertCmd.Set(":normalized_url", url); | ||||
| 				await insertCmd.ExecuteNonQueryAsync(); | ||||
| 				 | ||||
|  | ||||
| 				deleteCmd.Set(":normalized_url", url); | ||||
| 				await deleteCmd.ExecuteNonQueryAsync(); | ||||
| 			} | ||||
| @@ -68,8 +67,7 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade { | ||||
|  | ||||
| 		await reporter.SubWork("Processing downloaded files...", totalFiles, totalFiles); | ||||
|  | ||||
| 		await tx.CommitAsync(); | ||||
| 		await tx.DisposeAsync(); | ||||
| 		await conn.CommitTransactionAsync(); | ||||
| 	} | ||||
|  | ||||
| 	private async Task<List<string>> GetDownloadedFileUrls(ISqliteConnection conn) { | ||||
| @@ -110,46 +108,46 @@ sealed class SqliteSchemaUpgradeTo7 : ISchemaUpgrade { | ||||
| 			insertCmd.Set(":size", download.Size); | ||||
| 			await insertCmd.ExecuteNonQueryAsync(); | ||||
| 		} | ||||
| 		 | ||||
| 		await using (var tx = await conn.BeginTransactionAsync()) { | ||||
| 			await using var insertCmd = conn.Command("INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size) VALUES (:normalized_url, :download_url, :status, :type, :size)"); | ||||
| 			insertCmd.Add(":normalized_url", SqliteType.Text); | ||||
| 			insertCmd.Add(":download_url", SqliteType.Text); | ||||
| 			insertCmd.Add(":status", SqliteType.Integer); | ||||
| 			insertCmd.Add(":type", SqliteType.Text); | ||||
| 			insertCmd.Add(":size", SqliteType.Integer); | ||||
|  | ||||
| 			await reporter.SubWork("Processing embeds...", 1, 4); | ||||
| 			 | ||||
| 			await using (var embedCmd = conn.Command("SELECT json FROM embeds")) { | ||||
| 				await using var reader = await embedCmd.ExecuteReaderAsync(); | ||||
| 		await conn.BeginTransactionAsync(); | ||||
|  | ||||
| 				while (await reader.ReadAsync()) { | ||||
| 					await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0))); | ||||
| 				} | ||||
| 		await using var insertCmd = conn.Command("INSERT OR IGNORE INTO download_metadata (normalized_url, download_url, status, type, size) VALUES (:normalized_url, :download_url, :status, :type, :size)"); | ||||
| 		insertCmd.Add(":normalized_url", SqliteType.Text); | ||||
| 		insertCmd.Add(":download_url", SqliteType.Text); | ||||
| 		insertCmd.Add(":status", SqliteType.Integer); | ||||
| 		insertCmd.Add(":type", SqliteType.Text); | ||||
| 		insertCmd.Add(":size", SqliteType.Integer); | ||||
|  | ||||
| 		await reporter.SubWork("Processing embeds...", 1, 4); | ||||
|  | ||||
| 		await using (var embedCmd = conn.Command("SELECT json FROM embeds")) { | ||||
| 			await using var reader = await embedCmd.ExecuteReaderAsync(); | ||||
|  | ||||
| 			while (await reader.ReadAsync()) { | ||||
| 				await InsertDownload(insertCmd, await DownloadLinkExtractor.TryFromEmbedJson(reader.GetStream(0))); | ||||
| 			} | ||||
|  | ||||
| 			await reporter.SubWork("Processing users...", 2, 4); | ||||
| 			 | ||||
| 			await using (var avatarCmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) { | ||||
| 				await using var reader = await avatarCmd.ExecuteReaderAsync(); | ||||
|  | ||||
| 				while (await reader.ReadAsync()) { | ||||
| 					await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1))); | ||||
| 				} | ||||
| 			} | ||||
| 			 | ||||
| 			await reporter.SubWork("Processing reactions...", 3, 4); | ||||
| 			 | ||||
| 			await using (var avatarCmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM reactions WHERE emoji_id IS NOT NULL")) { | ||||
| 				await using var reader = await avatarCmd.ExecuteReaderAsync(); | ||||
|  | ||||
| 				while (await reader.ReadAsync()) { | ||||
| 					await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1))); | ||||
| 				} | ||||
| 			} | ||||
| 			 | ||||
| 			await tx.CommitAsync(); | ||||
| 		} | ||||
|  | ||||
| 		await reporter.SubWork("Processing users...", 2, 4); | ||||
|  | ||||
| 		await using (var avatarCmd = conn.Command("SELECT id, avatar_url FROM users WHERE avatar_url IS NOT NULL")) { | ||||
| 			await using var reader = await avatarCmd.ExecuteReaderAsync(); | ||||
|  | ||||
| 			while (await reader.ReadAsync()) { | ||||
| 				await InsertDownload(insertCmd, DownloadLinkExtractor.FromUserAvatar(reader.GetUint64(0), reader.GetString(1))); | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		await reporter.SubWork("Processing reactions...", 3, 4); | ||||
|  | ||||
| 		await using (var avatarCmd = conn.Command("SELECT DISTINCT emoji_id, emoji_flags FROM reactions WHERE emoji_id IS NOT NULL")) { | ||||
| 			await using var reader = await avatarCmd.ExecuteReaderAsync(); | ||||
|  | ||||
| 			while (await reader.ReadAsync()) { | ||||
| 				await InsertDownload(insertCmd, DownloadLinkExtractor.FromEmoji(reader.GetUint64(0), (EmojiFlags) reader.GetInt16(1))); | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		await conn.CommitTransactionAsync(); | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -1,8 +1,15 @@ | ||||
| using System; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Data.Sqlite; | ||||
|  | ||||
| namespace DHT.Server.Database.Sqlite.Utils; | ||||
|  | ||||
| interface ISqliteConnection : IAsyncDisposable { | ||||
| 	SqliteConnection InnerConnection { get; } | ||||
| 	 | ||||
| 	Task BeginTransactionAsync(); | ||||
| 	Task CommitTransactionAsync(); | ||||
| 	Task RollbackTransactionAsync(); | ||||
| 	 | ||||
| 	void AssignActiveTransaction(SqliteCommand command); | ||||
| } | ||||
|   | ||||
| @@ -1,5 +1,6 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Data.Common; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using DHT.Utils.Collections; | ||||
| @@ -73,17 +74,48 @@ sealed class SqliteConnectionPool : IAsyncDisposable { | ||||
| 		disposalTokenSource.Dispose(); | ||||
| 	} | ||||
|  | ||||
| 	private sealed class PooledConnection : ISqliteConnection { | ||||
| 		public SqliteConnection InnerConnection { get; } | ||||
| 	private sealed class PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) : ISqliteConnection { | ||||
| 		public SqliteConnection InnerConnection { get; } = conn; | ||||
|  | ||||
| 		private readonly SqliteConnectionPool pool; | ||||
| 		private DbTransaction? activeTransaction; | ||||
|  | ||||
| 		public PooledConnection(SqliteConnectionPool pool, SqliteConnection conn) { | ||||
| 			this.pool = pool; | ||||
| 			this.InnerConnection = conn; | ||||
| 		public async Task BeginTransactionAsync() { | ||||
| 			if (activeTransaction != null) { | ||||
| 				throw new InvalidOperationException("A transaction is already active."); | ||||
| 			} | ||||
| 			 | ||||
| 			activeTransaction = await InnerConnection.BeginTransactionAsync(); | ||||
| 		} | ||||
|  | ||||
| 		public async Task CommitTransactionAsync() { | ||||
| 			if (activeTransaction == null) { | ||||
| 				throw new InvalidOperationException("No active transaction to commit."); | ||||
| 			} | ||||
|  | ||||
| 			await activeTransaction.CommitAsync(); | ||||
| 			await activeTransaction.DisposeAsync(); | ||||
| 			activeTransaction = null; | ||||
| 		} | ||||
|  | ||||
| 		public async Task RollbackTransactionAsync() { | ||||
| 			if (activeTransaction == null) { | ||||
| 				throw new InvalidOperationException("No active transaction to rollback."); | ||||
| 			} | ||||
| 			 | ||||
| 			await activeTransaction.RollbackAsync(); | ||||
| 			await activeTransaction.DisposeAsync(); | ||||
| 			activeTransaction = null; | ||||
| 		} | ||||
|  | ||||
| 		public void AssignActiveTransaction(SqliteCommand command) { | ||||
| 			command.Transaction = (SqliteTransaction?) activeTransaction; | ||||
| 		} | ||||
|  | ||||
| 		public async ValueTask DisposeAsync() { | ||||
| 			if (activeTransaction != null) { | ||||
| 				await RollbackTransactionAsync(); | ||||
| 			} | ||||
| 			 | ||||
| 			await pool.Return(this); | ||||
| 		} | ||||
| 	} | ||||
|   | ||||
| @@ -1,5 +1,4 @@ | ||||
| using System; | ||||
| using System.Data.Common; | ||||
| using System.Linq; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| @@ -9,10 +8,6 @@ using Microsoft.Data.Sqlite; | ||||
| namespace DHT.Server.Database.Sqlite.Utils; | ||||
|  | ||||
| static class SqliteExtensions { | ||||
| 	public static ValueTask<DbTransaction> BeginTransactionAsync(this ISqliteConnection conn) { | ||||
| 		return conn.InnerConnection.BeginTransactionAsync(); | ||||
| 	} | ||||
|  | ||||
| 	public static SqliteCommand Command(this ISqliteConnection conn, [LanguageInjection("sql")] string sql) { | ||||
| 		var cmd = conn.InnerConnection.CreateCommand(); | ||||
| 		cmd.CommandText = sql; | ||||
| @@ -30,6 +25,10 @@ static class SqliteExtensions { | ||||
|  | ||||
| 		return await reader.ReadAsync(cancellationToken) ? readFunction(reader) : readFunction(null); | ||||
| 	} | ||||
| 	 | ||||
| 	public static async Task<long> ExecuteLongScalarAsync(this SqliteCommand command) { | ||||
| 		return (long) (await command.ExecuteScalarAsync())!; | ||||
| 	} | ||||
|  | ||||
| 	public static SqliteCommand Insert(this ISqliteConnection conn, string tableName, (string Name, SqliteType Type)[] columns) { | ||||
| 		string columnNames = string.Join(',', columns.Select(static c => c.Name)); | ||||
|   | ||||
| @@ -10,13 +10,12 @@ public readonly struct DownloadItem { | ||||
| 	public string? Type { get; init; } | ||||
| 	public ulong? Size { get; init; } | ||||
| 	 | ||||
| 	internal DownloadWithData ToSuccess(byte[] data) { | ||||
| 		var size = (ulong) Math.Max(data.LongLength, 0); | ||||
| 		return new DownloadWithData(new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, size), data); | ||||
| 	internal Data.Download ToSuccess(long size) { | ||||
| 		return new Data.Download(NormalizedUrl, DownloadUrl, DownloadStatus.Success, Type, (ulong) Math.Max(size, 0)); | ||||
| 	} | ||||
| 	 | ||||
| 	internal DownloadWithData ToFailure(HttpStatusCode? statusCode = null) { | ||||
| 	internal Data.Download ToFailure(HttpStatusCode? statusCode = null) { | ||||
| 		var status = statusCode.HasValue ? (DownloadStatus) (int) statusCode : DownloadStatus.GenericError; | ||||
| 		return new DownloadWithData(new Data.Download(NormalizedUrl, DownloadUrl, status, Type, Size), Data: null); | ||||
| 		return new Data.Download(NormalizedUrl, DownloadUrl, status, Type, Size); | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -11,16 +11,18 @@ public sealed class Downloader { | ||||
| 	public bool IsDownloading => current != null; | ||||
| 	 | ||||
| 	private readonly IDatabaseFile db; | ||||
| 	private readonly int? concurrentDownloads; | ||||
| 	private readonly SemaphoreSlim semaphore = new (1, 1); | ||||
| 	 | ||||
| 	internal Downloader(IDatabaseFile db) { | ||||
| 	internal Downloader(IDatabaseFile db, int? concurrentDownloads) { | ||||
| 		this.db = db; | ||||
| 		this.concurrentDownloads = concurrentDownloads; | ||||
| 	} | ||||
|  | ||||
| 	public async Task<IObservable<DownloadItem>> Start(DownloadItemFilter filter) { | ||||
| 		await semaphore.WaitAsync(); | ||||
| 		try { | ||||
| 			current ??= new DownloaderTask(db, filter); | ||||
| 			current ??= new DownloaderTask(db, filter, concurrentDownloads); | ||||
| 			return current.FinishedItems; | ||||
| 		} finally { | ||||
| 			semaphore.Release(); | ||||
|   | ||||
| @@ -15,10 +15,14 @@ namespace DHT.Server.Download; | ||||
| sealed class DownloaderTask : IAsyncDisposable { | ||||
| 	private static readonly Log Log = Log.ForType<DownloaderTask>(); | ||||
|  | ||||
| 	private const int DownloadTasks = 4; | ||||
| 	private const int DefaultConcurrentDownloads = 4; | ||||
| 	private const int QueueSize = 25; | ||||
| 	private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; | ||||
|  | ||||
| 	private static int GetDownloadTaskCount(int? concurrentDownloads) { | ||||
| 		return Math.Max(1, concurrentDownloads ?? DefaultConcurrentDownloads); | ||||
| 	} | ||||
|  | ||||
| 	private readonly Channel<DownloadItem> downloadQueue = Channel.CreateBounded<DownloadItem>(new BoundedChannelOptions(QueueSize) { | ||||
| 		SingleReader = false, | ||||
| 		SingleWriter = true, | ||||
| @@ -38,12 +42,12 @@ sealed class DownloaderTask : IAsyncDisposable { | ||||
|  | ||||
| 	public IObservable<DownloadItem> FinishedItems => finishedItemPublisher; | ||||
|  | ||||
| 	internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter) { | ||||
| 	internal DownloaderTask(IDatabaseFile db, DownloadItemFilter filter, int? concurrentDownloads) { | ||||
| 		this.db = db; | ||||
| 		this.filter = filter; | ||||
| 		this.cancellationToken = cancellationTokenSource.Token; | ||||
| 		this.queueWriterTask = Task.Run(RunQueueWriterTask); | ||||
| 		this.downloadTasks = Enumerable.Range(1, DownloadTasks).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray(); | ||||
| 		this.downloadTasks = Enumerable.Range(1, GetDownloadTaskCount(concurrentDownloads)).Select(taskIndex => Task.Run(() => RunDownloadTask(taskIndex))).ToArray(); | ||||
| 	} | ||||
|  | ||||
| 	private async Task RunQueueWriterTask() { | ||||
| @@ -63,24 +67,39 @@ sealed class DownloaderTask : IAsyncDisposable { | ||||
| 	private async Task RunDownloadTask(int taskIndex) { | ||||
| 		var log = Log.ForType<DownloaderTask>("Task " + taskIndex); | ||||
|  | ||||
| 		var client = new HttpClient(); | ||||
| 		var client = new HttpClient(new SocketsHttpHandler { | ||||
| 			ConnectTimeout = TimeSpan.FromSeconds(30) | ||||
| 		}); | ||||
| 		 | ||||
| 		client.Timeout = Timeout.InfiniteTimeSpan; | ||||
| 		client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); | ||||
| 		client.Timeout = TimeSpan.FromSeconds(30); | ||||
|  | ||||
| 		while (!cancellationToken.IsCancellationRequested) { | ||||
| 			var item = await downloadQueue.Reader.ReadAsync(cancellationToken); | ||||
| 			log.Debug("Downloading " + item.DownloadUrl + "..."); | ||||
|  | ||||
| 			try { | ||||
| 				var downloadedBytes = await client.GetByteArrayAsync(item.DownloadUrl, cancellationToken); | ||||
| 				await db.Downloads.AddDownload(item.ToSuccess(downloadedBytes)); | ||||
| 			} catch (OperationCanceledException) { | ||||
| 				var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, item.DownloadUrl), HttpCompletionOption.ResponseHeadersRead, cancellationToken); | ||||
| 				response.EnsureSuccessStatusCode(); | ||||
|  | ||||
| 				if (response.Content.Headers.ContentLength is {} contentLength) { | ||||
| 					await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); | ||||
| 					await db.Downloads.AddDownload(item.ToSuccess(contentLength), stream); | ||||
| 				} | ||||
| 				else { | ||||
| 					await db.Downloads.AddDownload(item.ToFailure(), stream: null); | ||||
| 					log.Error("Download response has no content length: " + item.DownloadUrl); | ||||
| 				} | ||||
| 			} catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) { | ||||
| 				// Ignore. | ||||
| 			} catch (TaskCanceledException e) when (e.InnerException is TimeoutException) { | ||||
| 				await db.Downloads.AddDownload(item.ToFailure(), stream: null); | ||||
| 				log.Error("Download timed out: " + item.DownloadUrl); | ||||
| 			} catch (HttpRequestException e) { | ||||
| 				await db.Downloads.AddDownload(item.ToFailure(e.StatusCode)); | ||||
| 				await db.Downloads.AddDownload(item.ToFailure(e.StatusCode), stream: null); | ||||
| 				log.Error(e); | ||||
| 			} catch (Exception e) { | ||||
| 				await db.Downloads.AddDownload(item.ToFailure()); | ||||
| 				await db.Downloads.AddDownload(item.ToFailure(), stream: null); | ||||
| 				log.Error(e); | ||||
| 			} finally { | ||||
| 				try { | ||||
|   | ||||
| @@ -9,37 +9,37 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| abstract class BaseEndpoint { | ||||
| abstract class BaseEndpoint(IDatabaseFile db) { | ||||
| 	private static readonly Log Log = Log.ForType<BaseEndpoint>(); | ||||
|  | ||||
| 	protected IDatabaseFile Db { get; } | ||||
|  | ||||
| 	protected BaseEndpoint(IDatabaseFile db) { | ||||
| 		this.Db = db; | ||||
| 	} | ||||
| 	protected IDatabaseFile Db { get; } = db; | ||||
|  | ||||
| 	public async Task Handle(HttpContext ctx) { | ||||
| 		var response = ctx.Response; | ||||
|  | ||||
| 		try { | ||||
| 			response.StatusCode = (int) HttpStatusCode.OK; | ||||
| 			var output = await Respond(ctx); | ||||
| 			await output.WriteTo(response); | ||||
| 			await Respond(ctx.Request, response); | ||||
| 		} catch (HttpException e) { | ||||
| 			Log.Error(e); | ||||
| 			response.StatusCode = (int) e.StatusCode; | ||||
| 			await response.WriteAsync(e.Message); | ||||
| 			if (response.HasStarted) { | ||||
| 				Log.Warn("Response has already started, cannot write status message: " + e.Message); | ||||
| 			} | ||||
| 			else { | ||||
| 				await response.WriteAsync(e.Message); | ||||
| 			} | ||||
| 		} catch (Exception e) { | ||||
| 			Log.Error(e); | ||||
| 			response.StatusCode = (int) HttpStatusCode.InternalServerError; | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	protected abstract Task<IHttpOutput> Respond(HttpContext ctx); | ||||
| 	protected abstract Task Respond(HttpRequest request, HttpResponse response); | ||||
|  | ||||
| 	protected static async Task<JsonElement> ReadJson(HttpContext ctx) { | ||||
| 	protected static async Task<JsonElement> ReadJson(HttpRequest request) { | ||||
| 		try { | ||||
| 			return await ctx.Request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement); | ||||
| 			return await request.ReadFromJsonAsync(JsonElementContext.Default.JsonElement); | ||||
| 		} catch (JsonException) { | ||||
| 			throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON."); | ||||
| 		} | ||||
|   | ||||
| @@ -7,18 +7,13 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class GetDownloadedFileEndpoint : BaseEndpoint { | ||||
| 	public GetDownloadedFileEndpoint(IDatabaseFile db) : base(db) {} | ||||
|  | ||||
| 	protected override async Task<IHttpOutput> Respond(HttpContext ctx) { | ||||
| 		string url = WebUtility.UrlDecode((string) ctx.Request.RouteValues["url"]!); | ||||
| sealed class GetDownloadedFileEndpoint(IDatabaseFile db) : BaseEndpoint(db) { | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		string url = WebUtility.UrlDecode((string) request.RouteValues["url"]!); | ||||
| 		string normalizedUrl = DiscordCdn.NormalizeUrl(url); | ||||
| 		 | ||||
| 		if (await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl) is { Download: {} download, Data: {} data }) { | ||||
| 			return new HttpOutput.File(download.Type, data); | ||||
| 		} | ||||
| 		else { | ||||
| 			return new HttpOutput.Redirect(url, permanent: false); | ||||
| 		if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, (download, stream) => response.WriteStreamAsync(download.Type, download.Size, stream))) { | ||||
| 			response.Redirect(url, permanent: false); | ||||
| 		} | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -1,5 +1,4 @@ | ||||
| using System.Reflection; | ||||
| using System.Text; | ||||
| using System.Net.Mime; | ||||
| using System.Threading.Tasks; | ||||
| using System.Web; | ||||
| using DHT.Server.Database; | ||||
| @@ -10,25 +9,17 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class GetTrackingScriptEndpoint : BaseEndpoint { | ||||
| 	private static ResourceLoader Resources { get; } = new (Assembly.GetExecutingAssembly()); | ||||
| 	 | ||||
| 	private readonly ServerParameters serverParameters; | ||||
| 	 | ||||
| 	public GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters) : base(db) { | ||||
| 		serverParameters = parameters; | ||||
| 	} | ||||
|  | ||||
| 	protected override async Task<IHttpOutput> Respond(HttpContext ctx) { | ||||
| 		string bootstrap = await Resources.ReadTextAsync("Tracker/bootstrap.js"); | ||||
| 		string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + serverParameters.Port + ";") | ||||
| 		                         .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(serverParameters.Token)) | ||||
| 		                         .Replace("/*[IMPORTS]*/", await Resources.ReadJoinedAsync("Tracker/scripts/", '\n')) | ||||
| 		                         .Replace("/*[CSS-CONTROLLER]*/", await Resources.ReadTextAsync("Tracker/styles/controller.css")) | ||||
| 		                         .Replace("/*[CSS-SETTINGS]*/", await Resources.ReadTextAsync("Tracker/styles/settings.css")) | ||||
| 		                         .Replace("/*[DEBUGGER]*/", ctx.Request.Query.ContainsKey("debug") ? "debugger;" : ""); | ||||
| sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters, ResourceLoader resources) : BaseEndpoint(db) { | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		string bootstrap = await resources.ReadTextAsync("Tracker/bootstrap.js"); | ||||
| 		string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + parameters.Port + ";") | ||||
| 		                         .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(parameters.Token)) | ||||
| 		                         .Replace("/*[IMPORTS]*/", await resources.ReadJoinedAsync("Tracker/scripts/", '\n')) | ||||
| 		                         .Replace("/*[CSS-CONTROLLER]*/", await resources.ReadTextAsync("Tracker/styles/controller.css")) | ||||
| 		                         .Replace("/*[CSS-SETTINGS]*/", await resources.ReadTextAsync("Tracker/styles/settings.css")) | ||||
| 		                         .Replace("/*[DEBUGGER]*/", request.Query.ContainsKey("debug") ? "debugger;" : ""); | ||||
| 		 | ||||
| 		ctx.Response.Headers.Append("X-DHT", "1"); | ||||
| 		return new HttpOutput.File("text/javascript", Encoding.UTF8.GetBytes(script)); | ||||
| 		response.Headers.Append("X-DHT", "1"); | ||||
| 		await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script); | ||||
| 	} | ||||
| } | ||||
|   | ||||
| @@ -8,18 +8,14 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class TrackChannelEndpoint : BaseEndpoint { | ||||
| 	public TrackChannelEndpoint(IDatabaseFile db) : base(db) {} | ||||
|  | ||||
| 	protected override async Task<IHttpOutput> Respond(HttpContext ctx) { | ||||
| 		var root = await ReadJson(ctx); | ||||
| sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint(db) { | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		var root = await ReadJson(request); | ||||
| 		var server = ReadServer(root.RequireObject("server"), "server"); | ||||
| 		var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id); | ||||
|  | ||||
| 		await Db.Servers.Add([server]); | ||||
| 		await Db.Channels.Add([channel]); | ||||
|  | ||||
| 		return HttpOutput.None; | ||||
| 	} | ||||
|  | ||||
| 	private static Data.Server ReadServer(JsonElement json, string path) => new () { | ||||
|   | ||||
| @@ -15,14 +15,12 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class TrackMessagesEndpoint : BaseEndpoint { | ||||
| sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) { | ||||
| 	private const string HasNewMessages = "1"; | ||||
| 	private const string NoNewMessages = "0"; | ||||
|  | ||||
| 	public TrackMessagesEndpoint(IDatabaseFile db) : base(db) {} | ||||
|  | ||||
| 	protected override async Task<IHttpOutput> Respond(HttpContext ctx) { | ||||
| 		var root = await ReadJson(ctx); | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		var root = await ReadJson(request); | ||||
|  | ||||
| 		if (root.ValueKind != JsonValueKind.Array) { | ||||
| 			throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array."); | ||||
| @@ -43,7 +41,7 @@ sealed class TrackMessagesEndpoint : BaseEndpoint { | ||||
|  | ||||
| 		await Db.Messages.Add(messages); | ||||
|  | ||||
| 		return new HttpOutput.Text(anyNewMessages ? HasNewMessages : NoNewMessages); | ||||
| 		await response.WriteTextAsync(anyNewMessages ? HasNewMessages : NoNewMessages); | ||||
| 	} | ||||
|  | ||||
| 	private static Message ReadMessage(JsonElement json, string path) => new () { | ||||
|   | ||||
| @@ -8,11 +8,9 @@ using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class TrackUsersEndpoint : BaseEndpoint { | ||||
| 	public TrackUsersEndpoint(IDatabaseFile db) : base(db) {} | ||||
|  | ||||
| 	protected override async Task<IHttpOutput> Respond(HttpContext ctx) { | ||||
| 		var root = await ReadJson(ctx); | ||||
| sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) { | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		var root = await ReadJson(request); | ||||
|  | ||||
| 		if (root.ValueKind != JsonValueKind.Array) { | ||||
| 			throw new HttpException(HttpStatusCode.BadRequest, "Expected root element to be an array."); | ||||
| @@ -26,8 +24,6 @@ sealed class TrackUsersEndpoint : BaseEndpoint { | ||||
| 		} | ||||
|  | ||||
| 		await Db.Users.Add(users); | ||||
|  | ||||
| 		return HttpOutput.None; | ||||
| 	} | ||||
|  | ||||
| 	private static User ReadUser(JsonElement json, string path) => new () { | ||||
|   | ||||
							
								
								
									
										42
									
								
								app/Server/Endpoints/ViewerEndpoint.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										42
									
								
								app/Server/Endpoints/ViewerEndpoint.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,42 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Net; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using DHT.Server.Database; | ||||
| using DHT.Utils.Http; | ||||
| using DHT.Utils.Resources; | ||||
| using Microsoft.AspNetCore.Http; | ||||
| using Microsoft.AspNetCore.StaticFiles; | ||||
|  | ||||
| namespace DHT.Server.Endpoints; | ||||
|  | ||||
| sealed class ViewerEndpoint(IDatabaseFile db, ResourceLoader resources) : BaseEndpoint(db) { | ||||
| 	private static readonly FileExtensionContentTypeProvider ContentTypeProvider = new (); | ||||
|  | ||||
| 	private readonly Dictionary<string, byte[]?> cache = new (); | ||||
| 	private readonly SemaphoreSlim cacheSemaphore = new (1); | ||||
|  | ||||
| 	protected override async Task Respond(HttpRequest request, HttpResponse response) { | ||||
| 		string path = (string?) request.RouteValues["path"] ?? "index.html"; | ||||
| 		string resourcePath = "Viewer/" + path; | ||||
| 		 | ||||
| 		byte[]? resourceBytes; | ||||
|  | ||||
| 		await cacheSemaphore.WaitAsync(); | ||||
| 		try { | ||||
| 			if (!cache.TryGetValue(resourcePath, out resourceBytes)) { | ||||
| 				cache[resourcePath] = resourceBytes = await resources.ReadBytesAsyncIfExists(resourcePath); | ||||
| 			} | ||||
| 		} finally { | ||||
| 			cacheSemaphore.Release(); | ||||
| 		} | ||||
|  | ||||
| 		if (resourceBytes == null) { | ||||
| 			throw new HttpException(HttpStatusCode.NotFound, "File not found: " + path); | ||||
| 		} | ||||
| 		else { | ||||
| 			var contentType = ContentTypeProvider.TryGetContentType(path, out string? type) ? type : null; | ||||
| 			await response.WriteFileAsync(contentType, resourceBytes); | ||||
| 		} | ||||
| 	} | ||||
| } | ||||
| @@ -39,6 +39,16 @@ | ||||
|       <Link>Resources/Tracker/styles/%(RecursiveDir)%(Filename)%(Extension)</Link> | ||||
|       <Visible>false</Visible> | ||||
|     </EmbeddedResource> | ||||
|     <EmbeddedResource Include="../Resources/Viewer/**"> | ||||
|       <LogicalName>Viewer\%(RecursiveDir)%(Filename)%(Extension)</LogicalName> | ||||
|       <Link>Resources/Viewer/%(RecursiveDir)%(Filename)%(Extension)</Link> | ||||
|       <Visible>false</Visible> | ||||
|     </EmbeddedResource> | ||||
|     <EmbeddedResource Include="../Desktop/Resources/icon.ico"> | ||||
|       <LogicalName>Viewer\favicon.ico</LogicalName> | ||||
|       <Link>Resources/icon.ico</Link> | ||||
|       <Visible>false</Visible> | ||||
|     </EmbeddedResource> | ||||
|   </ItemGroup> | ||||
|    | ||||
| </Project> | ||||
|   | ||||
| @@ -1,8 +1,10 @@ | ||||
| using System; | ||||
| using System.Reflection; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using DHT.Server.Database; | ||||
| using DHT.Utils.Logging; | ||||
| using DHT.Utils.Resources; | ||||
| using Microsoft.AspNetCore.Hosting; | ||||
| using Microsoft.AspNetCore.Server.Kestrel.Core; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| @@ -57,6 +59,7 @@ public sealed class ServerManager { | ||||
| 		void AddServices(IServiceCollection services) { | ||||
| 			services.AddSingleton(typeof(IDatabaseFile), db); | ||||
| 			services.AddSingleton(typeof(ServerParameters), new ServerParameters(port, token)); | ||||
| 			services.AddSingleton(typeof(ResourceLoader), new ResourceLoader(Assembly.GetExecutingAssembly())); | ||||
| 		} | ||||
|  | ||||
| 		void SetKestrelOptions(KestrelServerOptions options) { | ||||
|   | ||||
| @@ -3,6 +3,7 @@ using System.Text.Json.Serialization; | ||||
| using DHT.Server.Database; | ||||
| using DHT.Server.Endpoints; | ||||
| using DHT.Server.Service.Middlewares; | ||||
| using DHT.Utils.Resources; | ||||
| using Microsoft.AspNetCore.Builder; | ||||
| using Microsoft.AspNetCore.Http.Json; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| @@ -33,14 +34,23 @@ sealed class Startup { | ||||
| 	} | ||||
|  | ||||
| 	[SuppressMessage("ReSharper", "UnusedMember.Global")] | ||||
| 	public void Configure(IApplicationBuilder app, IHostApplicationLifetime lifetime, IDatabaseFile db, ServerParameters parameters) { | ||||
| 	public void Configure(IApplicationBuilder app, IHostApplicationLifetime lifetime, IDatabaseFile db, ServerParameters parameters, ResourceLoader resources) { | ||||
| 		app.UseMiddleware<ServerLoggingMiddleware>(); | ||||
| 		app.UseCors(); | ||||
| 		app.UseMiddleware<ServerAuthorizationMiddleware>(); | ||||
| 		app.UseRouting(); | ||||
| 		 | ||||
| 		app.Map("/viewer", node => { | ||||
| 			node.UseRouting(); | ||||
| 			node.UseEndpoints(endpoints => { | ||||
| 				endpoints.MapGet("/{**path}", new ViewerEndpoint(db, resources).Handle); | ||||
| 			}); | ||||
| 		}); | ||||
| 		 | ||||
| 		app.UseMiddleware<ServerAuthorizationMiddleware>(); | ||||
| 		 | ||||
| 		app.UseRouting(); | ||||
| 		app.UseEndpoints(endpoints => { | ||||
| 			endpoints.MapGet("/get-tracking-script", new GetTrackingScriptEndpoint(db, parameters).Handle); | ||||
| 			endpoints.MapGet("/get-tracking-script", new GetTrackingScriptEndpoint(db, parameters, resources).Handle); | ||||
| 			endpoints.MapGet("/get-viewer-data", new GetViewerDataEndpoint(db).Handle); | ||||
| 			endpoints.MapGet("/get-downloaded-file/{url}", new GetDownloadedFileEndpoint(db).Handle); | ||||
| 			endpoints.MapPost("/track-channel", new TrackChannelEndpoint(db).Handle); | ||||
| 			endpoints.MapPost("/track-users", new TrackUsersEndpoint(db).Handle); | ||||
|   | ||||
| @@ -6,18 +6,12 @@ using DHT.Server.Service; | ||||
|  | ||||
| namespace DHT.Server; | ||||
|  | ||||
| public sealed class State : IAsyncDisposable { | ||||
| 	public static State Dummy { get; } = new (DummyDatabaseFile.Instance); | ||||
| public sealed class State(IDatabaseFile db, int? concurrentDownloads) : IAsyncDisposable { | ||||
| 	public static State Dummy { get; } = new (DummyDatabaseFile.Instance, null); | ||||
| 	 | ||||
| 	public IDatabaseFile Db { get; } | ||||
| 	public Downloader Downloader { get; } | ||||
| 	public ServerManager Server { get; } | ||||
|  | ||||
| 	public State(IDatabaseFile db) { | ||||
| 		Db = db; | ||||
| 		Downloader = new Downloader(db); | ||||
| 		Server = new ServerManager(db); | ||||
| 	} | ||||
| 	public IDatabaseFile Db { get; } = db; | ||||
| 	public Downloader Downloader { get; } = new (db, concurrentDownloads); | ||||
| 	public ServerManager Server { get; } = new (db); | ||||
|  | ||||
| 	public async ValueTask DisposeAsync() { | ||||
| 		await Downloader.Stop(); | ||||
|   | ||||
| @@ -8,7 +8,7 @@ public static class LinqExtensions { | ||||
| 		HashSet<TKey>? seenKeys = null; | ||||
|  | ||||
| 		foreach (var item in collection) { | ||||
| 			seenKeys ??= new HashSet<TKey>(); | ||||
| 			seenKeys ??= []; | ||||
|  | ||||
| 			if (seenKeys.Add(getKeyFromItem(item))) { | ||||
| 				yield return item; | ||||
|   | ||||
							
								
								
									
										33
									
								
								app/Utils/Http/HttpExtensions.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										33
									
								
								app/Utils/Http/HttpExtensions.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,33 @@ | ||||
| using System.IO; | ||||
| using System.Net.Mime; | ||||
| using System.Text; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Utils.Http;  | ||||
|  | ||||
| public static class HttpExtensions { | ||||
| 	public static Task WriteTextAsync(this HttpResponse response, string text) { | ||||
| 		return WriteTextAsync(response, MediaTypeNames.Text.Plain, text); | ||||
| 	} | ||||
| 	 | ||||
| 	public static async Task WriteTextAsync(this HttpResponse response, string contentType, string text) { | ||||
| 		response.ContentType = contentType; | ||||
| 		await response.StartAsync(); | ||||
| 		await response.WriteAsync(text, Encoding.UTF8); | ||||
| 	} | ||||
| 	 | ||||
| 	public static async Task WriteFileAsync(this HttpResponse response, string? contentType, byte[] bytes) { | ||||
| 		response.ContentType = contentType ?? string.Empty; | ||||
| 		response.ContentLength = bytes.Length; | ||||
| 		await response.StartAsync(); | ||||
| 		await response.Body.WriteAsync(bytes); | ||||
| 	} | ||||
| 	 | ||||
| 	public static async Task WriteStreamAsync(this HttpResponse response, string? contentType, ulong? contentLength, Stream source) { | ||||
| 		response.ContentType = contentType ?? string.Empty; | ||||
| 		response.ContentLength = (long?) contentLength; | ||||
| 		await response.StartAsync(); | ||||
| 		await source.CopyToAsync(response.Body); | ||||
| 	} | ||||
| } | ||||
| @@ -1,35 +0,0 @@ | ||||
| using System.Text; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Utils.Http; | ||||
|  | ||||
| public static class HttpOutput { | ||||
| 	public static IHttpOutput None { get; } = new NoneImpl(); | ||||
|  | ||||
| 	private sealed class NoneImpl : IHttpOutput { | ||||
| 		public Task WriteTo(HttpResponse response) { | ||||
| 			return Task.CompletedTask; | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	public sealed class Text(string text) : IHttpOutput { | ||||
| 		public Task WriteTo(HttpResponse response) { | ||||
| 			return response.WriteAsync(text, Encoding.UTF8); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	public sealed class File(string? contentType, byte[] bytes) : IHttpOutput { | ||||
| 		public async Task WriteTo(HttpResponse response) { | ||||
| 			response.ContentType = contentType ?? string.Empty; | ||||
| 			await response.Body.WriteAsync(bytes); | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	public sealed class Redirect(string url, bool permanent) : IHttpOutput { | ||||
| 		public Task WriteTo(HttpResponse response) { | ||||
| 			response.Redirect(url, permanent); | ||||
| 			return Task.CompletedTask; | ||||
| 		} | ||||
| 	} | ||||
| } | ||||
| @@ -1,8 +0,0 @@ | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.AspNetCore.Http; | ||||
|  | ||||
| namespace DHT.Utils.Http; | ||||
|  | ||||
| public interface IHttpOutput { | ||||
| 	Task WriteTo(HttpResponse response); | ||||
| } | ||||
| @@ -7,8 +7,9 @@ using System.Threading.Tasks; | ||||
| namespace DHT.Utils.Resources; | ||||
|  | ||||
| public sealed class ResourceLoader(Assembly assembly) { | ||||
| 	private Stream GetEmbeddedStream(string filename) { | ||||
| 	private Stream? TryGetEmbeddedStream(string filename) { | ||||
| 		Stream? stream = null; | ||||
| 		 | ||||
| 		foreach (var embeddedName in assembly.GetManifestResourceNames()) { | ||||
| 			if (embeddedName.Replace('\\', '/') == filename) { | ||||
| 				stream = assembly.GetManifestResourceStream(embeddedName); | ||||
| @@ -16,17 +17,31 @@ public sealed class ResourceLoader(Assembly assembly) { | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		return stream ?? throw new ArgumentException("Missing embedded resource: " + filename); | ||||
| 		return stream; | ||||
| 	} | ||||
| 	 | ||||
| 	private Stream GetEmbeddedStream(string filename) { | ||||
| 		return TryGetEmbeddedStream(filename) ?? throw new ArgumentException("Missing embedded resource: " + filename); | ||||
| 	} | ||||
|  | ||||
| 	private async Task<string> ReadTextAsync(Stream stream) { | ||||
| 		using var reader = new StreamReader(stream, Encoding.UTF8); | ||||
| 		return await reader.ReadToEndAsync(); | ||||
| 	} | ||||
| 	 | ||||
| 	private async Task<byte[]> ReadBytesAsync(Stream stream) { | ||||
| 		using var memoryStream = new MemoryStream(); | ||||
| 		await stream.CopyToAsync(memoryStream); | ||||
| 		return memoryStream.ToArray(); | ||||
| 	} | ||||
|  | ||||
| 	public async Task<string> ReadTextAsync(string filename) { | ||||
| 		return await ReadTextAsync(GetEmbeddedStream(filename)); | ||||
| 	} | ||||
| 	 | ||||
| 	public async Task<byte[]?> ReadBytesAsyncIfExists(string filename) { | ||||
| 		return TryGetEmbeddedStream(filename) is {} stream ? await ReadBytesAsync(stream) : null; | ||||
| 	} | ||||
|  | ||||
| 	public async Task<string> ReadJoinedAsync(string path, char separator) { | ||||
| 		StringBuilder joined = new (); | ||||
|   | ||||
| @@ -8,5 +8,5 @@ using DHT.Utils; | ||||
| namespace DHT.Utils; | ||||
|  | ||||
| static class Version { | ||||
| 	public const string Tag = "41.0.0.0"; | ||||
| 	public const string Tag = "41.2.0.0"; | ||||
| } | ||||
|   | ||||
		Reference in New Issue
	
	Block a user