Revert "Add JVM argument to disable signal handling to prevent instant termination of Minecraft processes"

This reverts commit e699513036.

Agent/Phantom.Agent.Minecraft/Instance/InstanceProcess.cs

@@ -5,9 +5,8 @@ namespace Phantom.Agent.Minecraft.Instance;
 
 public sealed class InstanceProcess : IDisposable {
 	public InstanceProperties InstanceProperties { get; }
-	public CancellableSemaphore BackupSemaphore { get; } = new (1, 1);
 	
-	private readonly RingBuffer<string> outputBuffer = new (10000);
+	private readonly RingBuffer<string> outputBuffer = new (100);
 	private event EventHandler<string>? OutputEvent;
 
 	public event EventHandler? Ended;
@@ -61,7 +60,6 @@ public sealed class InstanceProcess : IDisposable {
 
 	public void Dispose() {
 		process.Dispose();
-		BackupSemaphore.Dispose();
 		OutputEvent = null;
 		Ended = null;
 	}

Agent/Phantom.Agent.Minecraft/Java/JavaPropertiesFileEditor.cs

@@ -0,0 +1,92 @@
+using System.Text;
+using Kajabity.Tools.Java;
+
+namespace Phantom.Agent.Minecraft.Java;
+
+sealed class JavaPropertiesFileEditor {
+	private static readonly Encoding Encoding = Encoding.GetEncoding("ISO-8859-1");
+
+	private readonly Dictionary<string, string> overriddenProperties = new ();
+
+	public void Set(string key, string value) {
+		overriddenProperties[key] = value;
+	}
+
+	public async Task EditOrCreate(string filePath) {
+		if (File.Exists(filePath)) {
+			string tmpFilePath = filePath + ".tmp";
+			File.Copy(filePath, tmpFilePath, overwrite: true);
+			await EditFromCopyOrCreate(filePath, tmpFilePath);
+			File.Move(tmpFilePath, filePath, overwrite: true);
+		}
+		else {
+			await EditFromCopyOrCreate(null, filePath);
+		}
+	}
+
+	private async Task EditFromCopyOrCreate(string? sourceFilePath, string targetFilePath) {
+		var properties = new JavaProperties();
+
+		if (sourceFilePath != null) {
+			// TODO replace with custom async parser
+			await using var sourceStream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+			properties.Load(sourceStream, Encoding);
+		}
+
+		foreach (var (key, value) in overriddenProperties) {
+			properties[key] = value;
+		}
+
+		await using var targetStream = new FileStream(targetFilePath, FileMode.Create, FileAccess.Write, FileShare.Read);
+		await using var targetWriter = new StreamWriter(targetStream, Encoding);
+
+		await targetWriter.WriteLineAsync("# Properties");
+
+		foreach (var (key, value) in properties) {
+			await WriteProperty(targetWriter, key, value);
+		}
+	}
+
+	private static async Task WriteProperty(StreamWriter writer, string key, string value) {
+		await WritePropertyComponent(writer, key, escapeSpaces: true);
+		await writer.WriteAsync('=');
+		await WritePropertyComponent(writer, value, escapeSpaces: false);
+		await writer.WriteLineAsync();
+	}
+
+	private static async Task WritePropertyComponent(TextWriter writer, string component, bool escapeSpaces) {
+		for (int index = 0; index < component.Length; index++) {
+			var c = component[index];
+			switch (c) {
+				case '\\':
+				case '#':
+				case '!':
+				case '=':
+				case ':':
+				case ' ' when escapeSpaces || index == 0:
+					await writer.WriteAsync('\\');
+					await writer.WriteAsync(c);
+					break;
+				case var _ when c > 31 && c < 127:
+					await writer.WriteAsync(c);
+					break;
+				case '\t':
+					await writer.WriteAsync("\\t");
+					break;
+				case '\n':
+					await writer.WriteAsync("\\n");
+					break;
+				case '\r':
+					await writer.WriteAsync("\\r");
+					break;
+				case '\f':
+					await writer.WriteAsync("\\f");
+					break;
+				default:
+					await writer.WriteAsync("\\u");
+					await writer.WriteAsync(((int) c).ToString("X4"));
+					break;
+			}
+		}
+	}
+}

Agent/Phantom.Agent.Minecraft/Java/JavaRuntimeDiscovery.cs

@@ -79,10 +79,9 @@ public sealed class JavaRuntimeDiscovery {
 			WorkingDirectory = Path.GetDirectoryName(javaExecutablePath),
 			Arguments = "-XshowSettings:properties -version",
 			RedirectStandardInput = false,
-			RedirectStandardOutput = true,
+			RedirectStandardOutput = false,
 			RedirectStandardError = true,
-			UseShellExecute = false,
-			CreateNoWindow = false
+			UseShellExecute = false
 		};
 
 		var process = new Process { StartInfo = startInfo };

Agent/Phantom.Agent.Minecraft/Java/JvmArgumentBuilder.cs

@@ -1,5 +1,4 @@
-using System.Collections.Immutable;
-using System.Collections.ObjectModel;
+using System.Collections.ObjectModel;
 
 namespace Phantom.Agent.Minecraft.Java;
 
@@ -7,12 +6,12 @@ sealed class JvmArgumentBuilder {
 	private readonly JvmProperties basicProperties;
 	private readonly List<string> customArguments = new ();
 
-	public JvmArgumentBuilder(JvmProperties basicProperties, ImmutableArray<string> customArguments) {
+	public JvmArgumentBuilder(JvmProperties basicProperties) {
 		this.basicProperties = basicProperties;
-
-		foreach (var jvmArgument in customArguments) {
-			this.customArguments.Add(jvmArgument);
 	}
+
+	public void Add(string argument) {
+		customArguments.Add(argument);
 	}
 
 	public void AddProperty(string key, string value) {
@@ -24,8 +23,8 @@ sealed class JvmArgumentBuilder {
 			target.Add(property);
 		}
 
+		// In case of duplicate JVM arguments, typically the last one wins.
 		target.Add("-Xms" + basicProperties.InitialHeapMegabytes + "M");
 		target.Add("-Xmx" + basicProperties.MaximumHeapMegabytes + "M");
-		target.Add("-Xrs");
 	}
 }

Agent/Phantom.Agent.Minecraft/Launcher/BaseLauncher.cs

@@ -1,9 +1,7 @@
 using System.Text;
-using Kajabity.Tools.Java;
 using Phantom.Agent.Minecraft.Instance;
 using Phantom.Agent.Minecraft.Java;
 using Phantom.Agent.Minecraft.Server;
-using Phantom.Common.Minecraft;
 using Phantom.Utils.Runtime;
 using Serilog;
 
@@ -23,10 +21,6 @@ public abstract class BaseLauncher : IServerLauncher {
 			return new LaunchResult.InvalidJavaRuntime();
 		}
 
-		if (JvmArgumentsHelper.Validate(instanceProperties.JvmArguments) != null) {
-			return new LaunchResult.InvalidJvmArguments();
-		}
-
 		var vanillaServerJarPath = await services.ServerExecutables.DownloadAndGetPath(instanceProperties.LaunchProperties.ServerDownloadInfo, MinecraftVersion, downloadProgressEventHandler, cancellationToken);
 		if (vanillaServerJarPath == null) {
 			return new LaunchResult.CouldNotDownloadMinecraftServer();
@@ -62,16 +56,8 @@ public abstract class BaseLauncher : IServerLauncher {
 			UseShellExecute = false
 		};
 		
-		var jvmArguments = new JvmArgumentBuilder(instanceProperties.JvmProperties, instanceProperties.JvmArguments);
-		CustomizeJvmArguments(jvmArguments);
-
 		var processArguments = processConfigurator.ArgumentList;
-		jvmArguments.Build(processArguments);
-		
-		foreach (var extraArgument in serverJar.ExtraArgs) {
-			processArguments.Add(extraArgument);
-		}
-		
+		PrepareJvmArguments(serverJar).Build(processArguments);
 		processArguments.Add("-jar");
 		processArguments.Add(serverJar.FilePath);
 		processArguments.Add("nogui");
@@ -96,6 +82,21 @@ public abstract class BaseLauncher : IServerLauncher {
 		return new LaunchResult.Success(instanceProcess);
 	}
 
+	private JvmArgumentBuilder PrepareJvmArguments(ServerJarInfo serverJar) {
+		var builder = new JvmArgumentBuilder(instanceProperties.JvmProperties);
+
+		foreach (string argument in instanceProperties.JvmArguments) {
+			builder.Add(argument);
+		}
+
+		foreach (var argument in serverJar.ExtraArgs) {
+			builder.Add(argument);
+		}
+
+		CustomizeJvmArguments(builder);
+		return builder;
+	}
+
 	private protected virtual void CustomizeJvmArguments(JvmArgumentBuilder arguments) {}
 
 	private protected virtual Task<ServerJarInfo> PrepareServerJar(ILogger logger, string serverJarPath, CancellationToken cancellationToken) {
@@ -108,21 +109,8 @@ public abstract class BaseLauncher : IServerLauncher {
 	}
 
 	private static async Task UpdateServerProperties(InstanceProperties instanceProperties) {
-		var serverPropertiesFilePath = Path.Combine(instanceProperties.InstanceFolder, "server.properties");
-		var serverPropertiesData = new JavaProperties();
-
-		await using var fileStream = new FileStream(serverPropertiesFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
-		try {
-			serverPropertiesData.Load(fileStream);
-		} catch (ParseException e) {
-			throw new Exception("Could not parse server.properties file: " + serverPropertiesFilePath, e);
-		}
-		
-		instanceProperties.ServerProperties.SetTo(serverPropertiesData);
-
-		fileStream.Seek(0L, SeekOrigin.Begin);
-		fileStream.SetLength(0L);
-		
-		serverPropertiesData.Store(fileStream, true);
+		var serverPropertiesEditor = new JavaPropertiesFileEditor();
+		instanceProperties.ServerProperties.SetTo(serverPropertiesEditor);
+		await serverPropertiesEditor.EditOrCreate(Path.Combine(instanceProperties.InstanceFolder, "server.properties"));
 	}
 }

Agent/Phantom.Agent.Minecraft/Launcher/LaunchResult.cs

@@ -9,8 +9,6 @@ public abstract record LaunchResult {
 
 	public sealed record InvalidJavaRuntime : LaunchResult;
 
-	public sealed record InvalidJvmArguments : LaunchResult;
-
 	public sealed record CouldNotDownloadMinecraftServer : LaunchResult;
 	
 	public sealed record CouldNotPrepareMinecraftServerLauncher : LaunchResult;

Agent/Phantom.Agent.Minecraft/Phantom.Agent.Minecraft.csproj

@@ -13,7 +13,6 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Phantom.Common.Data\Phantom.Common.Data.csproj" />
     <ProjectReference Include="..\..\Common\Phantom.Common.Logging\Phantom.Common.Logging.csproj" />
-    <ProjectReference Include="..\..\Common\Phantom.Common.Minecraft\Phantom.Common.Minecraft.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Collections\Phantom.Utils.Collections.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Cryptography\Phantom.Utils.Cryptography.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.IO\Phantom.Utils.IO.csproj" />

Agent/Phantom.Agent.Minecraft/Properties/MinecraftServerProperty.cs

@@ -1,4 +1,4 @@
-using Kajabity.Tools.Java;
+using Phantom.Agent.Minecraft.Java;
 
 namespace Phantom.Agent.Minecraft.Properties; 
 
@@ -12,7 +12,7 @@ abstract class MinecraftServerProperty<T> {
 	protected abstract T Read(string value);
 	protected abstract string Write(T value);
 
-	public void Set(JavaProperties properties, T value) {
-		properties.SetProperty(key, Write(value));
+	public void Set(JavaPropertiesFileEditor properties, T value) {
+		properties.Set(key, Write(value));
 	}
 }

Agent/Phantom.Agent.Minecraft/Properties/ServerProperties.cs

@@ -1,4 +1,4 @@
-using Kajabity.Tools.Java;
+using Phantom.Agent.Minecraft.Java;
 
 namespace Phantom.Agent.Minecraft.Properties;
 
@@ -7,7 +7,7 @@ public sealed record ServerProperties(
 	ushort RconPort,
 	bool EnableRcon = true
 ) {
-	internal void SetTo(JavaProperties properties) {
+	internal void SetTo(JavaPropertiesFileEditor properties) {
 		MinecraftServerProperties.ServerPort.Set(properties, ServerPort);
 		MinecraftServerProperties.RconPort.Set(properties, RconPort);
 		MinecraftServerProperties.EnableRcon.Set(properties, EnableRcon);

Agent/Phantom.Agent.Minecraft/Server/MinecraftServerExecutables.cs

@@ -10,7 +10,7 @@ public sealed partial class MinecraftServerExecutables {
 	private static readonly ILogger Logger = PhantomLogger.Create<MinecraftServerExecutables>();
 
 	[GeneratedRegex(@"[^a-zA-Z0-9_\-\.]", RegexOptions.Compiled)]
-	private static partial Regex VersionFolderSanitizeRegex();
+	private static partial Regex SanitizePathRegex();
 
 	private readonly string basePath;
 	private readonly Dictionary<string, MinecraftServerExecutableDownloader> runningDownloadersByVersion = new ();
@@ -20,7 +20,7 @@ public sealed partial class MinecraftServerExecutables {
 	}
 
 	internal async Task<string?> DownloadAndGetPath(FileDownloadInfo? fileDownloadInfo, string minecraftVersion, EventHandler<DownloadProgressEventArgs> progressEventHandler, CancellationToken cancellationToken) {
-		string serverExecutableFolderPath = Path.Combine(basePath, VersionFolderSanitizeRegex().Replace(minecraftVersion, "_"));
+		string serverExecutableFolderPath = Path.Combine(basePath, SanitizePathRegex().IsMatch(minecraftVersion) ? SanitizePathRegex().Replace(minecraftVersion, "_") : minecraftVersion);
 		string serverExecutableFilePath = Path.Combine(serverExecutableFolderPath, "server.jar");
 
 		if (File.Exists(serverExecutableFilePath)) {

Agent/Phantom.Agent.Minecraft/Server/ServerStatusProtocol.cs

@@ -78,7 +78,7 @@ public sealed class ServerStatusProtocol {
 				return null;
 			}
 
-			string onlinePlayerCountStr = Encoding.BigEndianUnicode.GetString(messageBuffer[(separator1 + 1)..(separator2 - 1)]);
+			string onlinePlayerCountStr = Encoding.BigEndianUnicode.GetString(messageBuffer.AsSpan((separator1 + 1)..(separator2 - 1)));
 			if (!int.TryParse(onlinePlayerCountStr, out int onlinePlayerCount)) {
 				logger.Error("Could not parse online player count in response from server: {OnlinePlayerCount}.", onlinePlayerCountStr);
 				return null;

Agent/Phantom.Agent.Rpc/RpcLauncher.cs

@@ -13,7 +13,7 @@ using Serilog.Events;
 namespace Phantom.Agent.Rpc;
 
 public sealed class RpcLauncher : RpcRuntime<ClientSocket> {
-	public static async Task Launch(RpcConfiguration config, AgentAuthToken authToken, AgentInfo agentInfo, Func<RpcServerConnection, IMessageToAgentListener> listenerFactory, SemaphoreSlim disconnectSemaphore, CancellationToken receiveCancellationToken) {
+	public static Task Launch(RpcConfiguration config, AgentAuthToken authToken, AgentInfo agentInfo, Func<RpcServerConnection, IMessageToAgentListener> listenerFactory, SemaphoreSlim disconnectSemaphore, CancellationToken receiveCancellationToken) {
 		var socket = new ClientSocket();
 		var options = socket.Options;
 
@@ -21,7 +21,7 @@ public sealed class RpcLauncher : RpcRuntime<ClientSocket> {
 		options.CurveCertificate = new NetMQCertificate();
 		options.HelloMessage = MessageRegistries.ToServer.Write(new RegisterAgentMessage(authToken, agentInfo)).ToArray();
 
-		await new RpcLauncher(config, socket, agentInfo.Guid, listenerFactory, disconnectSemaphore, receiveCancellationToken).Launch();
+		return new RpcLauncher(config, socket, agentInfo.Guid, listenerFactory, disconnectSemaphore, receiveCancellationToken).Launch();
 	}
 
 	private readonly RpcConfiguration config;

Agent/Phantom.Agent.Services/AgentServiceConfiguration.cs

@@ -0,0 +1,3 @@
+namespace Phantom.Agent.Services; 
+
+public readonly record struct AgentServiceConfiguration(int MaxConcurrentCompressionTasks);

Agent/Phantom.Agent.Services/AgentServices.cs

@@ -18,10 +18,10 @@ public sealed class AgentServices {
 	internal JavaRuntimeRepository JavaRuntimeRepository { get; }
 	internal InstanceSessionManager InstanceSessionManager { get; }
 
-	public AgentServices(AgentInfo agentInfo, AgentFolders agentFolders) {
+	public AgentServices(AgentInfo agentInfo, AgentFolders agentFolders, AgentServiceConfiguration serviceConfiguration) {
 		this.AgentFolders = agentFolders;
 		this.TaskManager = new TaskManager(PhantomLogger.Create<TaskManager, AgentServices>());
-		this.BackupManager = new BackupManager(agentFolders);
+		this.BackupManager = new BackupManager(agentFolders, serviceConfiguration.MaxConcurrentCompressionTasks);
 		this.JavaRuntimeRepository = new JavaRuntimeRepository();
 		this.InstanceSessionManager = new InstanceSessionManager(agentInfo, agentFolders, JavaRuntimeRepository, TaskManager, BackupManager);
 	}
@@ -35,11 +35,11 @@ public sealed class AgentServices {
 	public async Task Shutdown() {
 		Logger.Information("Stopping services...");
 		
-		await InstanceSessionManager.StopAll();
-		InstanceSessionManager.Dispose();
-		
+		await InstanceSessionManager.DisposeAsync();
 		await TaskManager.Stop();
 		
+		BackupManager.Dispose();
+		
 		Logger.Information("Services stopped.");
 	}
 }

Agent/Phantom.Agent.Services/Backups/BackupManager.cs

@@ -5,44 +5,34 @@ using Serilog;
 
 namespace Phantom.Agent.Services.Backups;
 
-sealed class BackupManager {
+sealed class BackupManager : IDisposable {
 	private readonly string destinationBasePath;
 	private readonly string temporaryBasePath;
+	private readonly SemaphoreSlim compressionSemaphore;
 
-	public BackupManager(AgentFolders agentFolders) {
+	public BackupManager(AgentFolders agentFolders, int maxConcurrentCompressionTasks) {
 		this.destinationBasePath = agentFolders.BackupsFolderPath;
 		this.temporaryBasePath = Path.Combine(agentFolders.TemporaryFolderPath, "backups");
+		this.compressionSemaphore = new SemaphoreSlim(maxConcurrentCompressionTasks, maxConcurrentCompressionTasks);
 	}
 
-	public async Task<BackupCreationResult> CreateBackup(string loggerName, InstanceProcess process, CancellationToken cancellationToken) {
-		try {
-			if (!await process.BackupSemaphore.Wait(TimeSpan.FromSeconds(1), cancellationToken)) {
-				return new BackupCreationResult(BackupCreationResultKind.BackupAlreadyRunning);
-			}
-		} catch (ObjectDisposedException) {
-			return new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning);
-		} catch (OperationCanceledException) {
-			return new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning);
+	public Task<BackupCreationResult> CreateBackup(string loggerName, InstanceProcess process, CancellationToken cancellationToken) {
+		return new BackupCreator(this, loggerName, process, cancellationToken).CreateBackup();
 	}
 
-		try {
-			return await new BackupCreator(destinationBasePath, temporaryBasePath, loggerName, process, cancellationToken).CreateBackup();
-		} finally {
-			process.BackupSemaphore.Release();
-		}
+	public void Dispose() {
+		compressionSemaphore.Dispose();
 	}
 
 	private sealed class BackupCreator {
-		private readonly string destinationBasePath;
-		private readonly string temporaryBasePath;
+		private readonly BackupManager manager;
 		private readonly string loggerName;
 		private readonly ILogger logger;
 		private readonly InstanceProcess process;
 		private readonly CancellationToken cancellationToken;
 
-		public BackupCreator(string destinationBasePath, string temporaryBasePath, string loggerName, InstanceProcess process, CancellationToken cancellationToken) {
-			this.destinationBasePath = destinationBasePath;
-			this.temporaryBasePath = temporaryBasePath;
+		public BackupCreator(BackupManager manager, string loggerName, InstanceProcess process, CancellationToken cancellationToken) {
+			this.manager = manager;
 			this.loggerName = loggerName;
 			this.logger = PhantomLogger.Create<BackupManager>(loggerName);
 			this.process = process;
@@ -72,7 +62,7 @@ sealed class BackupManager {
 			try {
 				await dispatcher.DisableAutomaticSaving();
 				await dispatcher.SaveAllChunks();
-				return await new BackupArchiver(destinationBasePath, temporaryBasePath, loggerName, process.InstanceProperties, cancellationToken).ArchiveWorld(resultBuilder);
+				return await new BackupArchiver(manager.destinationBasePath, manager.temporaryBasePath, loggerName, process.InstanceProperties, cancellationToken).ArchiveWorld(resultBuilder);
 			} catch (OperationCanceledException) {
 				resultBuilder.Kind = BackupCreationResultKind.BackupCancelled;
 				logger.Warning("Backup creation was cancelled.");
@@ -85,7 +75,7 @@ sealed class BackupManager {
 				try {
 					await dispatcher.EnableAutomaticSaving();
 				} catch (OperationCanceledException) {
-					// ignore
+					// Ignore.
 				} catch (Exception e) {
 					resultBuilder.Warnings |= BackupCreationWarnings.CouldNotRestoreAutomaticSaving;
 					logger.Error(e, "Caught exception while enabling automatic saving after creating an instance backup.");
@@ -94,10 +84,20 @@ sealed class BackupManager {
 		}
 		
 		private async Task CompressWorldArchive(string filePath, BackupCreationResult.Builder resultBuilder) {
+			if (!await manager.compressionSemaphore.WaitAsync(TimeSpan.FromSeconds(1), cancellationToken)) {
+				logger.Information("Too many compression tasks running, waiting for one of them to complete...");
+				await manager.compressionSemaphore.WaitAsync(cancellationToken);
+			}
+
+			logger.Information("Compressing backup...");
+			try {
 				var compressedFilePath = await BackupCompressor.Compress(filePath, cancellationToken);
 				if (compressedFilePath == null) {
 					resultBuilder.Warnings |= BackupCreationWarnings.CouldNotCompressWorldArchive;
 				}
+			} finally {
+				manager.compressionSemaphore.Release();
+			}
 		}
 
 		private void LogBackupResult(BackupCreationResult result) {

Agent/Phantom.Agent.Services/Backups/BackupScheduler.cs

@@ -1,5 +1,7 @@
 using Phantom.Agent.Minecraft.Instance;
 using Phantom.Agent.Minecraft.Server;
+using Phantom.Agent.Services.Instances;
+using Phantom.Agent.Services.Instances.Procedures;
 using Phantom.Common.Data.Backups;
 using Phantom.Common.Logging;
 using Phantom.Utils.Runtime;
@@ -12,21 +14,23 @@ sealed class BackupScheduler : CancellableBackgroundTask {
 	private static readonly TimeSpan BackupInterval = TimeSpan.FromMinutes(30);
 	private static readonly TimeSpan BackupFailureRetryDelay = TimeSpan.FromMinutes(5);
 
-	private readonly string loggerName;
 	private readonly BackupManager backupManager;
 	private readonly InstanceProcess process;
+	private readonly IInstanceContext context;
+	private readonly SemaphoreSlim backupSemaphore = new (1, 1);
 	private readonly int serverPort;
 	private readonly ServerStatusProtocol serverStatusProtocol;
 	private readonly ManualResetEventSlim serverOutputWhileWaitingForOnlinePlayers = new ();
 	
 	public event EventHandler<BackupCreationResult>? BackupCompleted; 
 
-	public BackupScheduler(TaskManager taskManager, BackupManager backupManager, InstanceProcess process, int serverPort, string loggerName) : base(PhantomLogger.Create<BackupScheduler>(loggerName), taskManager, "Backup scheduler for " + loggerName) {
-		this.loggerName = loggerName;
+	public BackupScheduler(TaskManager taskManager, BackupManager backupManager, InstanceProcess process, IInstanceContext context, int serverPort) : base(PhantomLogger.Create<BackupScheduler>(context.ShortName), taskManager, "Backup scheduler for " + context.ShortName) {
 		this.backupManager = backupManager;
 		this.process = process;
+		this.context = context;
 		this.serverPort = serverPort;
-		this.serverStatusProtocol = new ServerStatusProtocol(loggerName);
+		this.serverStatusProtocol = new ServerStatusProtocol(context.ShortName);
+		Start();
 	}
 
 	protected override async Task RunTask() {
@@ -50,7 +54,17 @@ sealed class BackupScheduler : CancellableBackgroundTask {
 	}
 
 	private async Task<BackupCreationResult> CreateBackup() {
-		return await backupManager.CreateBackup(loggerName, process, CancellationToken.None);
+		if (!await backupSemaphore.WaitAsync(TimeSpan.FromSeconds(1))) {
+			return new BackupCreationResult(BackupCreationResultKind.BackupAlreadyRunning);
+		}
+		
+		try {
+			var procedure = new BackupInstanceProcedure(backupManager);
+			context.EnqueueProcedure(procedure);
+			return await procedure.Result;
+		} finally {
+			backupSemaphore.Release();
+		}
 	}
 
 	private async Task WaitForOnlinePlayers() {
@@ -93,4 +107,9 @@ sealed class BackupScheduler : CancellableBackgroundTask {
 			Logger.Debug("Detected server output, signalling to check for online players again.");
 		}
 	}
+
+	protected override void Dispose() {
+		backupSemaphore.Dispose();
+		serverOutputWhileWaitingForOnlinePlayers.Dispose();
+	}
 }

Agent/Phantom.Agent.Services/Instances/IInstanceContext.cs

@@ -0,0 +1,25 @@
+using Phantom.Agent.Services.Instances.Procedures;
+using Phantom.Agent.Services.Instances.States;
+using Phantom.Common.Data.Instance;
+using Serilog;
+
+namespace Phantom.Agent.Services.Instances; 
+
+interface IInstanceContext {
+	string ShortName { get; }
+	ILogger Logger { get; }
+	
+	InstanceServices Services { get; }
+	IInstanceState CurrentState { get; }
+
+	void SetStatus(IInstanceStatus newStatus);
+	void ReportEvent(IInstanceEvent instanceEvent);
+	void EnqueueProcedure(IInstanceProcedure procedure, bool immediate = false);
+}
+
+static class InstanceContextExtensions {
+	public static void SetLaunchFailedStatusAndReportEvent(this IInstanceContext context, InstanceLaunchFailReason reason) {
+		context.SetStatus(InstanceStatus.Failed(reason));
+		context.ReportEvent(new InstanceLaunchFailedEvent(reason));
+	}
+}

Agent/Phantom.Agent.Services/Instances/Instance.cs

@@ -1,5 +1,6 @@
 using Phantom.Agent.Minecraft.Launcher;
 using Phantom.Agent.Rpc;
+using Phantom.Agent.Services.Instances.Procedures;
 using Phantom.Agent.Services.Instances.States;
 using Phantom.Common.Data.Instance;
 using Phantom.Common.Data.Minecraft;
@@ -10,18 +11,12 @@ using Serilog;
 
 namespace Phantom.Agent.Services.Instances;
 
-sealed class Instance : IDisposable {
-	private static uint loggerSequenceId = 0;
-
-	private static string GetLoggerName(Guid guid) {
-		var prefix = guid.ToString();
-		return prefix[..prefix.IndexOf('-')] + "/" + Interlocked.Increment(ref loggerSequenceId);
-	}
-
+sealed class Instance : IAsyncDisposable {
 	private InstanceServices Services { get; }
 	
 	public InstanceConfiguration Configuration { get; private set; }
 	private IServerLauncher Launcher { get; set; }
+	private readonly SemaphoreSlim configurationSemaphore = new (1, 1);
 
 	private readonly string shortName;
 	private readonly ILogger logger;
@@ -30,14 +25,14 @@ sealed class Instance : IDisposable {
 	private int statusUpdateCounter;
 	
 	private IInstanceState currentState;
-	private readonly SemaphoreSlim stateTransitioningActionSemaphore = new (1, 1);
-
 	public bool IsRunning => currentState is not InstanceNotRunningState;
 	
 	public event EventHandler? IsRunningChanged; 
 	
-	public Instance(InstanceServices services, InstanceConfiguration configuration, IServerLauncher launcher) {
-		this.shortName = GetLoggerName(configuration.InstanceGuid);
+	private readonly InstanceProcedureManager procedureManager;
+
+	public Instance(string shortName, InstanceServices services, InstanceConfiguration configuration, IServerLauncher launcher) {
+		this.shortName = shortName;
 		this.logger = PhantomLogger.Create<Instance>(shortName);
 
 		this.Services = services;
@@ -46,6 +41,8 @@ sealed class Instance : IDisposable {
 		
 		this.currentState = new InstanceNotRunningState();
 		this.currentStatus = InstanceStatus.NotRunning;
+		
+		this.procedureManager = new InstanceProcedureManager(this, new Context(this), services.TaskManager);
 	}
 
 	private void TryUpdateStatus(string taskName, Func<Task> getUpdateTask) {
@@ -66,8 +63,10 @@ sealed class Instance : IDisposable {
 
 	private void ReportAndSetStatus(IInstanceStatus status) {
 		TryUpdateStatus("Report status of instance " + shortName + " as " + status.GetType().Name, async () => {
+			if (status != currentStatus) {
 				currentStatus = status;
 				await ServerMessaging.Send(new ReportInstanceStatusMessage(Configuration.InstanceGuid, status));
+			}
 		});
 	}
 
@@ -76,7 +75,7 @@ sealed class Instance : IDisposable {
 		Services.TaskManager.Run("Report event for instance " + shortName, async () => await ServerMessaging.Send(message));
 	}
 	
-	private void TransitionState(IInstanceState newState) {
+	internal void TransitionState(IInstanceState newState) {
 		if (currentState == newState) {
 			return;
 		}
@@ -96,114 +95,94 @@ sealed class Instance : IDisposable {
 		}
 	}
 
-	private T TransitionStateAndReturn<T>((IInstanceState State, T Result) newStateAndResult) {
-		TransitionState(newStateAndResult.State);
-		return newStateAndResult.Result;
-	}
-
 	public async Task Reconfigure(InstanceConfiguration configuration, IServerLauncher launcher, CancellationToken cancellationToken) {
-		await stateTransitioningActionSemaphore.WaitAsync(cancellationToken);
+		await configurationSemaphore.WaitAsync(cancellationToken);
 		try {
 			Configuration = configuration;
 			Launcher = launcher;
 		} finally {
-			stateTransitioningActionSemaphore.Release();
+			configurationSemaphore.Release();
 		}
 	}
 
-	public async Task<LaunchInstanceResult> Launch(CancellationToken shutdownCancellationToken) {
-		await stateTransitioningActionSemaphore.WaitAsync(shutdownCancellationToken);
+	public async Task<LaunchInstanceResult> Launch(CancellationToken cancellationToken) {
+		if (IsRunning) {
+			return LaunchInstanceResult.InstanceAlreadyRunning;
+		}
+
+		if (await procedureManager.GetCurrentProcedure(cancellationToken) is LaunchInstanceProcedure) {
+			return LaunchInstanceResult.InstanceAlreadyLaunching;
+		}
+		
+		LaunchInstanceProcedure procedure;
+		
+		await configurationSemaphore.WaitAsync(cancellationToken);
 		try {
-			return TransitionStateAndReturn(currentState.Launch(new InstanceContextImpl(this, shutdownCancellationToken)));
-		} catch (Exception e) {
-			logger.Error(e, "Caught exception while launching instance.");
-			return LaunchInstanceResult.UnknownError;
+			procedure = new LaunchInstanceProcedure(Configuration, Launcher);
 		} finally {
-			stateTransitioningActionSemaphore.Release();
-		}
+			configurationSemaphore.Release();
 		}
 		
-	public async Task<StopInstanceResult> Stop(MinecraftStopStrategy stopStrategy) {
-		await stateTransitioningActionSemaphore.WaitAsync();
-		try {
-			return TransitionStateAndReturn(currentState.Stop(stopStrategy));
-		} catch (Exception e) {
-			logger.Error(e, "Caught exception while stopping instance.");
-			return StopInstanceResult.UnknownError;
-		} finally {
-			stateTransitioningActionSemaphore.Release();
-		}
+		ReportAndSetStatus(InstanceStatus.Launching);
+		await procedureManager.Enqueue(procedure);
+		return LaunchInstanceResult.LaunchInitiated;
 	}
 
-	public async Task StopAndWait(TimeSpan waitTime) {
-		await Stop(MinecraftStopStrategy.Instant);
-
-		using var waitTokenSource = new CancellationTokenSource(waitTime);
-		var waitToken = waitTokenSource.Token;
-
-		while (currentState is not InstanceNotRunningState) {
-			await Task.Delay(TimeSpan.FromMilliseconds(250), waitToken);
+	public async Task<StopInstanceResult> Stop(MinecraftStopStrategy stopStrategy, CancellationToken cancellationToken) {
+		if (!IsRunning) {
+			return StopInstanceResult.InstanceAlreadyStopped;
 		}
+		
+		if (await procedureManager.GetCurrentProcedure(cancellationToken) is StopInstanceProcedure) {
+			return StopInstanceResult.InstanceAlreadyStopping;
+		}
+		
+		ReportAndSetStatus(InstanceStatus.Stopping);
+		await procedureManager.Enqueue(new StopInstanceProcedure(stopStrategy));
+		return StopInstanceResult.StopInitiated;
 	}
 
 	public async Task<bool> SendCommand(string command, CancellationToken cancellationToken) {
 		return await currentState.SendCommand(command, cancellationToken);
 	}
 
-	private sealed class InstanceContextImpl : InstanceContext {
-		private readonly Instance instance;
-		private readonly CancellationToken shutdownCancellationToken;
+	public async ValueTask DisposeAsync() {
+		await procedureManager.DisposeAsync();
 
-		public InstanceContextImpl(Instance instance, CancellationToken shutdownCancellationToken) : base(instance.Services, instance.Configuration, instance.Launcher) {
-			this.instance = instance;
-			this.shutdownCancellationToken = shutdownCancellationToken;
+		while (currentState is not InstanceNotRunningState) {
+			await Task.Delay(TimeSpan.FromMilliseconds(250), CancellationToken.None);
 		}
 
-		public override ILogger Logger => instance.logger;
-		public override string ShortName => instance.shortName;
-
-		public override void SetStatus(IInstanceStatus newStatus) {
-			instance.ReportAndSetStatus(newStatus);
-		}
-
-		public override void ReportEvent(IInstanceEvent instanceEvent) {
-			instance.ReportEvent(instanceEvent);
-		}
-
-		public override void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus) {
-			instance.stateTransitioningActionSemaphore.Wait(CancellationToken.None);
-			try {
-				var (state, status) = newStateAndStatus();
-				
-				if (!instance.IsRunning) {
-					// Only InstanceSessionManager is allowed to transition an instance out of a non-running state.
-					instance.logger.Debug("Cancelled state transition to {State} because instance is not running.", state.GetType().Name);
-					return;
-				}
-				
-				if (state is not InstanceNotRunningState && shutdownCancellationToken.IsCancellationRequested) {
-					instance.logger.Debug("Cancelled state transition to {State} due to Agent shutdown.", state.GetType().Name);
-					return;
-				}
-
-				if (status != null) {
-					SetStatus(status);
-				}
-
-				instance.TransitionState(state);
-			} catch (Exception e) {
-				instance.logger.Error(e, "Caught exception during state transition.");
-			} finally {
-				instance.stateTransitioningActionSemaphore.Release();
-			}
-		}
-	}
-
-	public void Dispose() {
-		stateTransitioningActionSemaphore.Dispose();
-		
 		if (currentState is IDisposable disposable) {
 			disposable.Dispose();
 		}
+		
+		configurationSemaphore.Dispose();
+	}
+
+	private sealed class Context : IInstanceContext {
+		public string ShortName => instance.shortName;
+		public ILogger Logger => instance.logger;
+		
+		public InstanceServices Services => instance.Services;
+		public IInstanceState CurrentState => instance.currentState;
+
+		private readonly Instance instance;
+		
+		public Context(Instance instance) {
+			this.instance = instance;
+		}
+
+		public void SetStatus(IInstanceStatus newStatus) {
+			instance.ReportAndSetStatus(newStatus);
+		}
+
+		public void ReportEvent(IInstanceEvent instanceEvent) {
+			instance.ReportEvent(instanceEvent);
+		}
+
+		public void EnqueueProcedure(IInstanceProcedure procedure, bool immediate) {
+			Services.TaskManager.Run("Enqueue procedure for instance " + instance.shortName, () => instance.procedureManager.Enqueue(procedure, immediate));
+		}
 	}
 }

Agent/Phantom.Agent.Services/Instances/InstanceContext.cs

@@ -1,35 +0,0 @@
-using Phantom.Agent.Minecraft.Launcher;
-using Phantom.Agent.Services.Instances.States;
-using Phantom.Common.Data.Instance;
-using Serilog;
-
-namespace Phantom.Agent.Services.Instances;
-
-abstract class InstanceContext {
-	public InstanceServices Services { get; }
-	public InstanceConfiguration Configuration { get; }
-	public IServerLauncher Launcher { get; }
-	
-	public abstract ILogger Logger { get; }
-	public abstract string ShortName { get; }
-
-	protected InstanceContext(InstanceServices services, InstanceConfiguration configuration, IServerLauncher launcher) {
-		Services = services;
-		Configuration = configuration;
-		Launcher = launcher;
-	}
-
-	public abstract void SetStatus(IInstanceStatus newStatus);
-
-	public void SetLaunchFailedStatusAndReportEvent(InstanceLaunchFailReason reason) {
-		SetStatus(InstanceStatus.Failed(reason));
-		ReportEvent(new InstanceLaunchFailedEvent(reason));
-	}
-
-	public abstract void ReportEvent(IInstanceEvent instanceEvent);
-	public abstract void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus);
-
-	public void TransitionState(IInstanceState newState, IInstanceStatus? newStatus = null) {
-		TransitionState(() => (newState, newStatus));
-	}
-}

Agent/Phantom.Agent.Services/Instances/InstanceLogSender.cs

@@ -1,36 +1,62 @@
 using System.Collections.Immutable;
+using System.Threading.Channels;
 using Phantom.Agent.Rpc;
 using Phantom.Common.Logging;
 using Phantom.Common.Messages.ToServer;
-using Phantom.Utils.Collections;
 using Phantom.Utils.Runtime;
 
 namespace Phantom.Agent.Services.Instances;
 
 sealed class InstanceLogSender : CancellableBackgroundTask {
+	private static readonly BoundedChannelOptions BufferOptions = new (capacity: 64) {
+		SingleReader = true,
+		SingleWriter = true,
+		FullMode = BoundedChannelFullMode.DropNewest
+	};
+	
 	private static readonly TimeSpan SendDelay = TimeSpan.FromMilliseconds(200);
 
 	private readonly Guid instanceGuid;
+	private readonly Channel<string> outputChannel;
 	
-	private readonly SemaphoreSlim semaphore = new (1, 1);
-	private readonly RingBuffer<string> buffer = new (1000);
+	private int droppedLinesSinceLastSend;
 
 	public InstanceLogSender(TaskManager taskManager, Guid instanceGuid, string loggerName) : base(PhantomLogger.Create<InstanceLogSender>(loggerName), taskManager, "Instance log sender for " + loggerName) {
 		this.instanceGuid = instanceGuid;
+		this.outputChannel = Channel.CreateBounded<string>(BufferOptions, OnLineDropped);
+		Start();
 	}
 	
 	protected override async Task RunTask() {
+		var lineReader = outputChannel.Reader;
+		var lineBuilder = ImmutableArray.CreateBuilder<string>();
+		
 		try {
-			while (!CancellationToken.IsCancellationRequested) {
-				await SendOutputToServer(await DequeueOrThrow());
+			while (await lineReader.WaitToReadAsync(CancellationToken)) {
 				await Task.Delay(SendDelay, CancellationToken);
+				await SendOutputToServer(ReadLinesFromChannel(lineReader, lineBuilder));
 			}
 		} catch (OperationCanceledException) {
 			// Ignore.
 		}
 
 		// Flush remaining lines.
-		await SendOutputToServer(DequeueWithoutSemaphore());
+		await SendOutputToServer(ReadLinesFromChannel(lineReader, lineBuilder));
+	}
+
+	private ImmutableArray<string> ReadLinesFromChannel(ChannelReader<string> reader, ImmutableArray<string>.Builder builder) {
+		builder.Clear();
+		
+		while (reader.TryRead(out string? line)) {
+			builder.Add(line);
+		}
+		
+		int droppedLines = Interlocked.Exchange(ref droppedLinesSinceLastSend, 0);
+		if (droppedLines > 0) {
+			builder.Add($"Dropped {droppedLines} {(droppedLines == 1 ? "line" : "lines")} due to buffer overflow.");
+		}
+		
+		return builder.ToImmutable();
 	}
 
 	private async Task SendOutputToServer(ImmutableArray<string> lines) {
@@ -39,33 +65,18 @@ sealed class InstanceLogSender : CancellableBackgroundTask {
 		}
 	}
 
-	private ImmutableArray<string> DequeueWithoutSemaphore() {
-		ImmutableArray<string> lines = buffer.Count > 0 ? buffer.EnumerateLast(uint.MaxValue).ToImmutableArray() : ImmutableArray<string>.Empty;
-		buffer.Clear();
-		return lines;
-	}
-
-	private async Task<ImmutableArray<string>> DequeueOrThrow() {
-		await semaphore.WaitAsync(CancellationToken);
-
-		try {
-			return DequeueWithoutSemaphore();
-		} finally {
-			semaphore.Release();
-		}
+	private void OnLineDropped(string line) {
+		Logger.Warning("Buffer is full, dropped line: {Line}", line);
+		Interlocked.Increment(ref droppedLinesSinceLastSend);
 	}
 
 	public void Enqueue(string line) {
-		try {
-			semaphore.Wait(CancellationToken);
-		} catch (Exception) {
-			return;
+		outputChannel.Writer.TryWrite(line);
 	}
 
-		try {
-			buffer.Add(line);
-		} finally {
-			semaphore.Release();
+	protected override void Dispose() {
+		if (!outputChannel.Writer.TryComplete()) {
+			Logger.Error("Could not mark channel as completed.");
 		}
 	}
 }

Agent/Phantom.Agent.Services/Instances/InstanceProcedureManager.cs

@@ -0,0 +1,85 @@
+using Phantom.Agent.Services.Instances.Procedures;
+using Phantom.Common.Data.Minecraft;
+using Phantom.Utils.Collections;
+using Phantom.Utils.Runtime;
+
+namespace Phantom.Agent.Services.Instances;
+
+sealed class InstanceProcedureManager : IAsyncDisposable {
+	private readonly record struct CurrentProcedure(IInstanceProcedure Procedure, CancellationTokenSource CancellationTokenSource);
+
+	private readonly ThreadSafeStructRef<CurrentProcedure> currentProcedure = new ();
+	private readonly ThreadSafeLinkedList<IInstanceProcedure> procedureQueue = new ();
+	private readonly AutoResetEvent procedureQueueReady = new (false);
+	private readonly ManualResetEventSlim procedureQueueFinished = new (false);
+
+	private readonly Instance instance;
+	private readonly IInstanceContext context;
+	private readonly CancellationTokenSource shutdownCancellationTokenSource = new ();
+
+	public InstanceProcedureManager(Instance instance, IInstanceContext context, TaskManager taskManager) {
+		this.instance = instance;
+		this.context = context;
+		taskManager.Run("Procedure manager for instance " + context.ShortName, Run);
+	}
+
+	public async Task Enqueue(IInstanceProcedure procedure, bool immediate = false) {
+		await procedureQueue.Add(procedure, toFront: immediate, shutdownCancellationTokenSource.Token);
+		procedureQueueReady.Set();
+	}
+
+	public async Task<IInstanceProcedure?> GetCurrentProcedure(CancellationToken cancellationToken) {
+		return (await currentProcedure.Get(cancellationToken))?.Procedure;
+	}
+
+	private async Task Run() {
+		try {
+			var shutdownCancellationToken = shutdownCancellationTokenSource.Token;
+			while (true) {
+				await procedureQueueReady.WaitOneAsync(shutdownCancellationToken);
+				while (await procedureQueue.TryTakeFromFront(shutdownCancellationToken) is {} nextProcedure) {
+					using var procedureCancellationTokenSource = new CancellationTokenSource();
+					await currentProcedure.Set(new CurrentProcedure(nextProcedure, procedureCancellationTokenSource), shutdownCancellationToken);
+					await RunProcedure(nextProcedure, procedureCancellationTokenSource.Token);
+					await currentProcedure.Set(null, shutdownCancellationToken);
+				}
+			}
+		} catch (OperationCanceledException) {
+			// Ignore.
+		}
+
+		await RunProcedure(new StopInstanceProcedure(MinecraftStopStrategy.Instant), CancellationToken.None);
+		procedureQueueFinished.Set();
+	}
+
+	private async Task RunProcedure(IInstanceProcedure procedure, CancellationToken cancellationToken) {
+		var procedureName = procedure.GetType().Name;
+
+		context.Logger.Debug("Started procedure: {Procedure}", procedureName);
+		try {
+			var newState = await procedure.Run(context, cancellationToken);
+			context.Logger.Debug("Finished procedure: {Procedure}", procedureName);
+
+			if (newState != null) {
+				instance.TransitionState(newState);
+			}
+		} catch (OperationCanceledException) {
+			context.Logger.Debug("Cancelled procedure: {Procedure}", procedureName);
+		} catch (Exception e) {
+			context.Logger.Error(e, "Caught exception while running procedure: {Procedure}", procedureName);
+		}
+	}
+
+	public async ValueTask DisposeAsync() {
+		shutdownCancellationTokenSource.Cancel();
+
+		(await currentProcedure.Get(CancellationToken.None))?.CancellationTokenSource.Cancel();
+		await procedureQueueFinished.WaitHandle.WaitOneAsync();
+
+		currentProcedure.Dispose();
+		procedureQueue.Dispose();
+		procedureQueueReady.Dispose();
+		procedureQueueFinished.Dispose();
+		shutdownCancellationTokenSource.Dispose();
+	}
+}

Agent/Phantom.Agent.Services/Instances/InstanceSessionManager.cs

@@ -21,7 +21,7 @@ using Serilog;
 
 namespace Phantom.Agent.Services.Instances;
 
-sealed class InstanceSessionManager : IDisposable {
+sealed class InstanceSessionManager : IAsyncDisposable {
 	private static readonly ILogger Logger = PhantomLogger.Create<InstanceSessionManager>();
 
 	private readonly AgentInfo agentInfo;
@@ -34,6 +34,8 @@ sealed class InstanceSessionManager : IDisposable {
 	private readonly CancellationToken shutdownCancellationToken;
 	private readonly SemaphoreSlim semaphore = new (1, 1);
 
+	private uint instanceLoggerSequenceId = 0;
+
 	public InstanceSessionManager(AgentInfo agentInfo, AgentFolders agentFolders, JavaRuntimeRepository javaRuntimeRepository, TaskManager taskManager, BackupManager backupManager) {
 		this.agentInfo = agentInfo;
 		this.basePath = agentFolders.InstancesFolderPath;
@@ -109,7 +111,7 @@ sealed class InstanceSessionManager : IDisposable {
 				}
 			}
 			else {
-				instances[instanceGuid] = instance = new Instance(instanceServices, configuration, launcher);
+				instances[instanceGuid] = instance = new Instance(GetInstanceLoggerName(instanceGuid), instanceServices, configuration, launcher);
 				Logger.Information("Created instance \"{Name}\" (GUID {Guid}).", configuration.InstanceName, configuration.InstanceGuid);
 
 				instance.ReportLastStatus();
@@ -124,6 +126,11 @@ sealed class InstanceSessionManager : IDisposable {
 		});
 	}
 
+	private string GetInstanceLoggerName(Guid guid) {
+		var prefix = guid.ToString();
+		return prefix[..prefix.IndexOf('-')] + "/" + Interlocked.Increment(ref instanceLoggerSequenceId);
+	}
+
 	private ImmutableArray<Instance> GetRunningInstancesInternal() {
 		return instances.Values.Where(static instance => instance.IsRunning).ToImmutableArray();
 	}
@@ -167,38 +174,23 @@ sealed class InstanceSessionManager : IDisposable {
 	}
 
 	public Task<InstanceActionResult<StopInstanceResult>> Stop(Guid instanceGuid, MinecraftStopStrategy stopStrategy) {
-		return AcquireSemaphoreAndRunWithInstance(instanceGuid, instance => instance.Stop(stopStrategy));
+		return AcquireSemaphoreAndRunWithInstance(instanceGuid, instance => instance.Stop(stopStrategy, shutdownCancellationToken));
 	}
 
 	public Task<InstanceActionResult<SendCommandToInstanceResult>> SendCommand(Guid instanceGuid, string command) {
 		return AcquireSemaphoreAndRunWithInstance(instanceGuid, async instance => await instance.SendCommand(command, shutdownCancellationToken) ? SendCommandToInstanceResult.Success : SendCommandToInstanceResult.UnknownError);
 	}
 
-	public async Task StopAll() {
-		shutdownCancellationTokenSource.Cancel();
-		
+	public async ValueTask DisposeAsync() {
 		Logger.Information("Stopping all instances...");
 		
-		await semaphore.WaitAsync(CancellationToken.None);
-		try {
-			await Task.WhenAll(instances.Values.Select(static instance => instance.StopAndWait(TimeSpan.FromSeconds(30))));
-			DisposeAllInstances();
-		} finally {
-			semaphore.Release();
-		}
-	}
+		shutdownCancellationTokenSource.Cancel();
+
+		await semaphore.WaitAsync(CancellationToken.None);
+		await Task.WhenAll(instances.Values.Select(static instance => instance.DisposeAsync().AsTask()));
+		instances.Clear();
 		
-	public void Dispose() {
-		DisposeAllInstances();
 		shutdownCancellationTokenSource.Dispose();
 		semaphore.Dispose();
 	}
-
-	private void DisposeAllInstances() {
-		foreach (var (_, instance) in instances) {
-			instance.Dispose();
-		}
-
-		instances.Clear();
-	}
 }

Agent/Phantom.Agent.Services/Instances/Procedures/BackupInstanceProcedure.cs

@@ -0,0 +1,29 @@
+using Phantom.Agent.Services.Backups;
+using Phantom.Agent.Services.Instances.States;
+using Phantom.Common.Data.Backups;
+
+namespace Phantom.Agent.Services.Instances.Procedures;
+
+sealed record BackupInstanceProcedure(BackupManager BackupManager) : IInstanceProcedure {
+	private readonly TaskCompletionSource<BackupCreationResult> resultCompletionSource = new ();
+
+	public Task<BackupCreationResult> Result => resultCompletionSource.Task;
+
+	public async Task<IInstanceState?> Run(IInstanceContext context, CancellationToken cancellationToken) {
+		if (context.CurrentState is not InstanceRunningState runningState || runningState.Process.HasEnded) {
+			resultCompletionSource.SetResult(new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning));
+			return null;
+		}
+
+		try {
+			var result = await BackupManager.CreateBackup(context.ShortName, runningState.Process, cancellationToken);
+			resultCompletionSource.SetResult(result);
+		} catch (OperationCanceledException) {
+			resultCompletionSource.SetCanceled(cancellationToken);
+		} catch (Exception e) {
+			resultCompletionSource.SetException(e);
+		}
+
+		return null;
+	}
+}

Agent/Phantom.Agent.Services/Instances/Procedures/IInstanceProcedure.cs

@@ -0,0 +1,7 @@
+using Phantom.Agent.Services.Instances.States;
+
+namespace Phantom.Agent.Services.Instances.Procedures; 
+
+interface IInstanceProcedure {
+	Task<IInstanceState?> Run(IInstanceContext context, CancellationToken cancellationToken);
+}

Agent/Phantom.Agent.Services/Instances/Procedures/LaunchInstanceProcedure.cs

@@ -0,0 +1,97 @@
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Minecraft.Launcher;
+using Phantom.Agent.Minecraft.Server;
+using Phantom.Agent.Services.Instances.States;
+using Phantom.Common.Data.Instance;
+
+namespace Phantom.Agent.Services.Instances.Procedures;
+
+sealed record LaunchInstanceProcedure(InstanceConfiguration Configuration, IServerLauncher Launcher, bool IsRestarting = false) : IInstanceProcedure {
+	public async Task<IInstanceState?> Run(IInstanceContext context, CancellationToken cancellationToken) {
+		if (!IsRestarting && context.CurrentState is InstanceRunningState) {
+			return null;
+		}
+		
+		context.SetStatus(IsRestarting ? InstanceStatus.Restarting : InstanceStatus.Launching);
+
+		InstanceLaunchFailReason? failReason = context.Services.PortManager.Reserve(Configuration) switch {
+			PortManager.Result.ServerPortNotAllowed   => InstanceLaunchFailReason.ServerPortNotAllowed,
+			PortManager.Result.ServerPortAlreadyInUse => InstanceLaunchFailReason.ServerPortAlreadyInUse,
+			PortManager.Result.RconPortNotAllowed     => InstanceLaunchFailReason.RconPortNotAllowed,
+			PortManager.Result.RconPortAlreadyInUse   => InstanceLaunchFailReason.RconPortAlreadyInUse,
+			_                                         => null
+		};
+
+		if (failReason is {} reason) {
+			context.SetLaunchFailedStatusAndReportEvent(reason);
+			return new InstanceNotRunningState();
+		}
+		
+		context.Logger.Information("Session starting...");
+		try {
+			InstanceProcess process = await DoLaunch(context, cancellationToken);
+			return new InstanceRunningState(Configuration, Launcher, process, context);
+		} catch (OperationCanceledException) {
+			context.SetStatus(InstanceStatus.NotRunning);
+		} catch (LaunchFailureException e) {
+			context.Logger.Error(e.LogMessage);
+			context.SetLaunchFailedStatusAndReportEvent(e.Reason);
+		} catch (Exception e) {
+			context.Logger.Error(e, "Caught exception while launching instance.");
+			context.SetLaunchFailedStatusAndReportEvent(InstanceLaunchFailReason.UnknownError);
+		}
+		
+		context.Services.PortManager.Release(Configuration);
+		return new InstanceNotRunningState();
+	}
+
+	private async Task<InstanceProcess> DoLaunch(IInstanceContext context, CancellationToken cancellationToken) {
+		cancellationToken.ThrowIfCancellationRequested();
+
+		byte lastDownloadProgress = byte.MaxValue;
+
+		void OnDownloadProgress(object? sender, DownloadProgressEventArgs args) {
+			byte progress = (byte) Math.Min(args.DownloadedBytes * 100 / args.TotalBytes, 100);
+
+			if (lastDownloadProgress != progress) {
+				lastDownloadProgress = progress;
+				context.SetStatus(InstanceStatus.Downloading(progress));
+			}
+		}
+
+		var launchResult = await Launcher.Launch(context.Logger, context.Services.LaunchServices, OnDownloadProgress, cancellationToken);
+		if (launchResult is LaunchResult.InvalidJavaRuntime) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.JavaRuntimeNotFound, "Session failed to launch, invalid Java runtime.");
+		}
+		else if (launchResult is LaunchResult.CouldNotDownloadMinecraftServer) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotDownloadMinecraftServer, "Session failed to launch, could not download Minecraft server.");
+		}
+		else if (launchResult is LaunchResult.CouldNotPrepareMinecraftServerLauncher) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotPrepareMinecraftServerLauncher, "Session failed to launch, could not prepare Minecraft server launcher.");
+		}
+		else if (launchResult is LaunchResult.CouldNotConfigureMinecraftServer) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotConfigureMinecraftServer, "Session failed to launch, could not configure Minecraft server.");
+		}
+		else if (launchResult is LaunchResult.CouldNotStartMinecraftServer) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotStartMinecraftServer, "Session failed to launch, could not start Minecraft server.");
+		}
+
+		if (launchResult is not LaunchResult.Success launchSuccess) {
+			throw new LaunchFailureException(InstanceLaunchFailReason.UnknownError, "Session failed to launch.");
+		}
+
+		context.SetStatus(InstanceStatus.Running);
+		context.ReportEvent(InstanceEvent.LaunchSucceded);
+		return launchSuccess.Process;
+	}
+
+	private sealed class LaunchFailureException : Exception {
+		public InstanceLaunchFailReason Reason { get; }
+		public string LogMessage { get; }
+
+		public LaunchFailureException(InstanceLaunchFailReason reason, string logMessage) {
+			this.Reason = reason;
+			this.LogMessage = logMessage;
+		}
+	}
+}

Agent/Phantom.Agent.Services/Instances/Procedures/SetInstanceToNotRunningStateProcedure.cs

@@ -0,0 +1,17 @@
+using Phantom.Agent.Services.Instances.States;
+using Phantom.Common.Data.Instance;
+
+namespace Phantom.Agent.Services.Instances.Procedures;
+
+sealed record SetInstanceToNotRunningStateProcedure(IInstanceStatus Status) : IInstanceProcedure {
+	public Task<IInstanceState?> Run(IInstanceContext context, CancellationToken cancellationToken) {
+		if (context.CurrentState is InstanceRunningState { Process.HasEnded: true }) {
+			context.SetStatus(Status);
+			context.ReportEvent(InstanceEvent.Stopped);
+			return Task.FromResult<IInstanceState?>(new InstanceNotRunningState());
+		}
+		else {
+			return Task.FromResult<IInstanceState?>(null);
+		}
+	}
+}

Agent/Phantom.Agent.Services/Instances/Procedures/StopInstanceProcedure.cs

@@ -0,0 +1,103 @@
+using System.Diagnostics;
+using Phantom.Agent.Minecraft.Command;
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Services.Instances.States;
+using Phantom.Common.Data.Instance;
+using Phantom.Common.Data.Minecraft;
+
+namespace Phantom.Agent.Services.Instances.Procedures;
+
+sealed record StopInstanceProcedure(MinecraftStopStrategy StopStrategy) : IInstanceProcedure {
+	private static readonly ushort[] Stops = { 60, 30, 10, 5, 4, 3, 2, 1, 0 };
+
+	public async Task<IInstanceState?> Run(IInstanceContext context, CancellationToken cancellationToken) {
+		if (context.CurrentState is not InstanceRunningState runningState) {
+			return null;
+		}
+
+		var process = runningState.Process;
+
+		runningState.IsStopping = true;
+		context.SetStatus(InstanceStatus.Stopping);
+
+		var seconds = StopStrategy.Seconds;
+		if (seconds > 0) {
+			try {
+				await CountDownWithAnnouncements(context, process, seconds, cancellationToken);
+			} catch (OperationCanceledException) {
+				runningState.IsStopping = false;
+				return null;
+			}
+		}
+
+		try {
+			// Too late to cancel the stop procedure now.
+			if (!process.HasEnded) {
+				context.Logger.Information("Session stopping now.");
+				await DoStop(context, process);
+			}
+		} finally {
+			context.Logger.Information("Session stopped.");
+			context.SetStatus(InstanceStatus.NotRunning);
+			context.ReportEvent(InstanceEvent.Stopped);
+		}
+
+		return new InstanceNotRunningState();
+	}
+
+	private async Task CountDownWithAnnouncements(IInstanceContext context, InstanceProcess process, ushort seconds, CancellationToken cancellationToken) {
+		context.Logger.Information("Session stopping in {Seconds} seconds.", seconds);
+
+		foreach (var stop in Stops) {
+			// TODO change to event-based cancellation
+			if (process.HasEnded) {
+				return;
+			}
+
+			if (seconds > stop) {
+				await process.SendCommand(GetCountDownAnnouncementCommand(seconds), cancellationToken);
+				await Task.Delay(TimeSpan.FromSeconds(seconds - stop), cancellationToken);
+				seconds = stop;
+			}
+		}
+	}
+
+	private static string GetCountDownAnnouncementCommand(ushort seconds) {
+		return MinecraftCommand.Say("Server shutting down in " + seconds + (seconds == 1 ? " second." : " seconds."));
+	}
+
+	private async Task DoStop(IInstanceContext context, InstanceProcess process) {
+		context.Logger.Information("Sending stop command...");
+		await TrySendStopCommand(context, process);
+
+		context.Logger.Information("Waiting for session to end...");
+		await WaitForSessionToEnd(context, process);
+	}
+
+	private async Task TrySendStopCommand(IInstanceContext context, InstanceProcess process) {
+		using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+		try {
+			await process.SendCommand(MinecraftCommand.Stop, timeout.Token);
+		} catch (OperationCanceledException) {
+			// Ignore.
+		} catch (ObjectDisposedException e) when (e.ObjectName == typeof(Process).FullName && process.HasEnded) {
+			// Ignore.
+		} catch (Exception e) {
+			context.Logger.Warning(e, "Caught exception while sending stop command.");
+		}
+	}
+
+	private async Task WaitForSessionToEnd(IInstanceContext context, InstanceProcess process) {
+		using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(55));
+		try {
+			await process.WaitForExit(timeout.Token);
+		} catch (OperationCanceledException) {
+			try {
+				context.Logger.Warning("Waiting timed out, killing session...");
+				process.Kill();
+			} catch (Exception e) {
+				context.Logger.Error(e, "Caught exception while killing session.");
+			}
+		}
+	}
+}

Agent/Phantom.Agent.Services/Instances/Sessions/InstanceSession.cs

@@ -1,28 +0,0 @@
-using Phantom.Agent.Minecraft.Instance;
-
-namespace Phantom.Agent.Services.Instances.Sessions; 
-
-sealed class InstanceSession : IDisposable {
-	private readonly InstanceProcess process;
-	private readonly InstanceContext context;
-	private readonly InstanceLogSender logSender;
-
-	public InstanceSession(InstanceProcess process, InstanceContext context) {
-		this.process = process;
-		this.context = context;
-		this.logSender = new InstanceLogSender(context.Services.TaskManager, context.Configuration.InstanceGuid, context.ShortName);
-		
-		this.process.AddOutputListener(SessionOutput);
-	}
-	
-	private void SessionOutput(object? sender, string line) {
-		context.Logger.Debug("[Server] {Line}", line);
-		logSender.Enqueue(line);
-	}
-	
-	public void Dispose() {
-		logSender.Stop();
-		process.Dispose();
-		context.Services.PortManager.Release(context.Configuration);
-	}
-}

Agent/Phantom.Agent.Services/Instances/States/IInstanceState.cs

@@ -1,11 +1,6 @@
-using Phantom.Common.Data.Minecraft;
-using Phantom.Common.Data.Replies;
-
-namespace Phantom.Agent.Services.Instances.States; 
+namespace Phantom.Agent.Services.Instances.States; 
 
 interface IInstanceState {
 	void Initialize();
-	(IInstanceState, LaunchInstanceResult) Launch(InstanceContext context);
-	(IInstanceState, StopInstanceResult) Stop(MinecraftStopStrategy stopStrategy);
 	Task<bool> SendCommand(string command, CancellationToken cancellationToken);
 }

Agent/Phantom.Agent.Services/Instances/States/InstanceLaunchingState.cs

@@ -1,127 +0,0 @@
-using Phantom.Agent.Minecraft.Instance;
-using Phantom.Agent.Minecraft.Launcher;
-using Phantom.Agent.Minecraft.Server;
-using Phantom.Agent.Services.Instances.Sessions;
-using Phantom.Common.Data.Instance;
-using Phantom.Common.Data.Minecraft;
-using Phantom.Common.Data.Replies;
-
-namespace Phantom.Agent.Services.Instances.States;
-
-sealed class InstanceLaunchingState : IInstanceState, IDisposable {
-	private readonly InstanceContext context;
-	private readonly CancellationTokenSource cancellationTokenSource = new ();
-	private byte lastDownloadProgress = byte.MaxValue;
-
-	public InstanceLaunchingState(InstanceContext context) {
-		this.context = context;
-	}
-
-	public void Initialize() {
-		context.Logger.Information("Session starting...");
-
-		var launchTask = context.Services.TaskManager.Run("Launch procedure for instance " + context.ShortName, DoLaunch);
-		launchTask.ContinueWith(OnLaunchSuccess, CancellationToken.None, TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
-		launchTask.ContinueWith(OnLaunchFailure, CancellationToken.None, TaskContinuationOptions.NotOnRanToCompletion, TaskScheduler.Default);
-	}
-
-	private async Task<InstanceProcess> DoLaunch() {
-		var cancellationToken = cancellationTokenSource.Token;
-		cancellationToken.ThrowIfCancellationRequested();
-
-		void OnDownloadProgress(object? sender, DownloadProgressEventArgs args) {
-			byte progress = (byte) Math.Min(args.DownloadedBytes * 100 / args.TotalBytes, 100);
-
-			if (lastDownloadProgress != progress) {
-				lastDownloadProgress = progress;
-				context.SetStatus(InstanceStatus.Downloading(progress));
-			}
-		}
-
-		var launchResult = await context.Launcher.Launch(context.Logger, context.Services.LaunchServices, OnDownloadProgress, cancellationToken);
-		if (launchResult is LaunchResult.InvalidJavaRuntime) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.JavaRuntimeNotFound, "Session failed to launch, invalid Java runtime.");
-		}
-		else if (launchResult is LaunchResult.InvalidJvmArguments) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.InvalidJvmArguments, "Session failed to launch, invalid JVM arguments.");
-		}
-		else if (launchResult is LaunchResult.CouldNotDownloadMinecraftServer) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotDownloadMinecraftServer, "Session failed to launch, could not download Minecraft server.");
-		}
-		else if (launchResult is LaunchResult.CouldNotPrepareMinecraftServerLauncher) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotPrepareMinecraftServerLauncher, "Session failed to launch, could not prepare Minecraft server launcher.");
-		}
-		else if (launchResult is LaunchResult.CouldNotConfigureMinecraftServer) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotConfigureMinecraftServer, "Session failed to launch, could not configure Minecraft server.");
-		}
-		else if (launchResult is LaunchResult.CouldNotStartMinecraftServer) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.CouldNotStartMinecraftServer, "Session failed to launch, could not start Minecraft server.");
-		}
-
-		if (launchResult is not LaunchResult.Success launchSuccess) {
-			throw new LaunchFailureException(InstanceLaunchFailReason.UnknownError, "Session failed to launch.");
-		}
-
-		context.SetStatus(InstanceStatus.Launching);
-		return launchSuccess.Process;
-	}
-
-	private void OnLaunchSuccess(Task<InstanceProcess> task) {
-		context.TransitionState(() => {
-			context.ReportEvent(InstanceEvent.LaunchSucceded);
-			
-			var process = task.Result;
-			var session = new InstanceSession(process, context);
-			
-			if (cancellationTokenSource.IsCancellationRequested) {
-				return (new InstanceStoppingState(context, process, session), InstanceStatus.Stopping);
-			}
-			else {
-				return (new InstanceRunningState(context, process, session), null);
-			}
-		});
-	}
-
-	private void OnLaunchFailure(Task task) {
-		if (task.IsFaulted) {
-			if (task.Exception is { InnerException: LaunchFailureException e }) {
-				context.Logger.Error(e.LogMessage);
-				context.SetLaunchFailedStatusAndReportEvent(e.Reason);
-			}
-			else {
-				context.Logger.Error(task.Exception, "Caught exception while launching instance.");
-				context.SetLaunchFailedStatusAndReportEvent(InstanceLaunchFailReason.UnknownError);
-			}
-		}
-
-		context.Services.PortManager.Release(context.Configuration);
-		context.TransitionState(new InstanceNotRunningState());
-	}
-
-	private sealed class LaunchFailureException : Exception {
-		public InstanceLaunchFailReason Reason { get; }
-		public string LogMessage { get; }
-
-		public LaunchFailureException(InstanceLaunchFailReason reason, string logMessage) {
-			this.Reason = reason;
-			this.LogMessage = logMessage;
-		}
-	}
-
-	public (IInstanceState, LaunchInstanceResult) Launch(InstanceContext context) {
-		return (this, LaunchInstanceResult.InstanceAlreadyLaunching);
-	}
-
-	public (IInstanceState, StopInstanceResult) Stop(MinecraftStopStrategy stopStrategy) {
-		cancellationTokenSource.Cancel();
-		return (this, StopInstanceResult.StopInitiated);
-	}
-
-	public Task<bool> SendCommand(string command, CancellationToken cancellationToken) {
-		return Task.FromResult(false);
-	}
-
-	public void Dispose() {
-		cancellationTokenSource.Dispose();
-	}
-}

Agent/Phantom.Agent.Services/Instances/States/InstanceNotRunningState.cs

@@ -1,34 +1,8 @@
-using Phantom.Common.Data.Instance;
-using Phantom.Common.Data.Minecraft;
-using Phantom.Common.Data.Replies;
-
-namespace Phantom.Agent.Services.Instances.States; 
+namespace Phantom.Agent.Services.Instances.States; 
 
 sealed class InstanceNotRunningState : IInstanceState {
 	public void Initialize() {}
 
-	public (IInstanceState, LaunchInstanceResult) Launch(InstanceContext context) {
-		InstanceLaunchFailReason? failReason = context.Services.PortManager.Reserve(context.Configuration) switch {
-			PortManager.Result.ServerPortNotAllowed   => InstanceLaunchFailReason.ServerPortNotAllowed,
-			PortManager.Result.ServerPortAlreadyInUse => InstanceLaunchFailReason.ServerPortAlreadyInUse,
-			PortManager.Result.RconPortNotAllowed     => InstanceLaunchFailReason.RconPortNotAllowed,
-			PortManager.Result.RconPortAlreadyInUse   => InstanceLaunchFailReason.RconPortAlreadyInUse,
-			_                                         => null
-		};
-
-		if (failReason is {} reason) {
-			context.SetLaunchFailedStatusAndReportEvent(reason);
-			return (this, LaunchInstanceResult.LaunchInitiated);
-		}
-		
-		context.SetStatus(InstanceStatus.Launching);
-		return (new InstanceLaunchingState(context), LaunchInstanceResult.LaunchInitiated);
-	}
-
-	public (IInstanceState, StopInstanceResult) Stop(MinecraftStopStrategy stopStrategy) {
-		return (this, StopInstanceResult.InstanceAlreadyStopped);
-	}
-
 	public Task<bool> SendCommand(string command, CancellationToken cancellationToken) {
 		return Task.FromResult(false);
 	}

Agent/Phantom.Agent.Services/Instances/States/InstanceRunningState.cs

@@ -1,133 +1,81 @@
-using Phantom.Agent.Minecraft.Command;
-using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Minecraft.Launcher;
 using Phantom.Agent.Services.Backups;
-using Phantom.Agent.Services.Instances.Sessions;
+using Phantom.Agent.Services.Instances.Procedures;
 using Phantom.Common.Data.Backups;
 using Phantom.Common.Data.Instance;
-using Phantom.Common.Data.Minecraft;
-using Phantom.Common.Data.Replies;
 
 namespace Phantom.Agent.Services.Instances.States;
 
-sealed class InstanceRunningState : IInstanceState {
-	private readonly InstanceContext context;
-	private readonly InstanceProcess process;
+sealed class InstanceRunningState : IInstanceState, IDisposable {
+	public InstanceProcess Process { get; }
+
+	internal bool IsStopping { get; set; }
+
+	private readonly InstanceConfiguration configuration;
+	private readonly IServerLauncher launcher;
+	private readonly IInstanceContext context;
+
+	private readonly InstanceLogSender logSender;
 	private readonly BackupScheduler backupScheduler;
-	private readonly RunningSessionDisposer runningSessionDisposer;
 
-	private readonly CancellationTokenSource delayedStopCancellationTokenSource = new ();
-	private bool stateOwnsDelayedStopCancellationTokenSource = true;
-	private bool isStopping;
+	private bool isDisposed;
 
-	public InstanceRunningState(InstanceContext context, InstanceProcess process, InstanceSession session) {
+	public InstanceRunningState(InstanceConfiguration configuration, IServerLauncher launcher, InstanceProcess process, IInstanceContext context) {
+		this.configuration = configuration;
+		this.launcher = launcher;
 		this.context = context;
-		this.process = process;
-		this.backupScheduler = new BackupScheduler(context.Services.TaskManager, context.Services.BackupManager, process, context.Configuration.ServerPort, context.ShortName);
+		this.Process = process;
+
+		this.logSender = new InstanceLogSender(context.Services.TaskManager, configuration.InstanceGuid, context.ShortName);
+
+		this.backupScheduler = new BackupScheduler(context.Services.TaskManager, context.Services.BackupManager, process, context, configuration.ServerPort);
 		this.backupScheduler.BackupCompleted += OnScheduledBackupCompleted;
-		this.runningSessionDisposer = new RunningSessionDisposer(this, session);
 	}
 
 	public void Initialize() {
-		process.Ended += ProcessEnded;
+		Process.Ended += ProcessEnded;
 
-		if (process.HasEnded) {
-			if (runningSessionDisposer.Dispose()) {
+		if (Process.HasEnded) {
+			if (TryDispose()) {
 				context.Logger.Warning("Session ended immediately after it was started.");
-				context.ReportEvent(InstanceEvent.Stopped);
-				context.Services.TaskManager.Run("Transition state of instance " + context.ShortName + " to not running", () => context.TransitionState(new InstanceNotRunningState(), InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)));
+				context.EnqueueProcedure(new SetInstanceToNotRunningStateProcedure(InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)), immediate: true);
 			}
 		}
 		else {
-			context.SetStatus(InstanceStatus.Running);
 			context.Logger.Information("Session started.");
+			Process.AddOutputListener(SessionOutput);
 		}
 	}
 
+	private void SessionOutput(object? sender, string line) {
+		context.Logger.Debug("[Server] {Line}", line);
+		logSender.Enqueue(line);
+	}
+
 	private void ProcessEnded(object? sender, EventArgs e) {
-		if (!runningSessionDisposer.Dispose()) {
+		if (!TryDispose()) {
 			return;
 		}
 
-		if (isStopping) {
-			context.Logger.Information("Session ended.");
-			context.ReportEvent(InstanceEvent.Stopped);
-			context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
+		if (IsStopping) {
+			context.EnqueueProcedure(new SetInstanceToNotRunningStateProcedure(InstanceStatus.NotRunning), immediate: true);
 		}
 		else {
 			context.Logger.Information("Session ended unexpectedly, restarting...");
 			context.ReportEvent(InstanceEvent.Crashed);
-			context.TransitionState(new InstanceLaunchingState(context), InstanceStatus.Restarting);
+			context.EnqueueProcedure(new LaunchInstanceProcedure(configuration, launcher, IsRestarting: true));
 		}
 	}
 
-	public (IInstanceState, LaunchInstanceResult) Launch(InstanceContext context) {
-		return (this, LaunchInstanceResult.InstanceAlreadyRunning);
-	}
-
-	public (IInstanceState, StopInstanceResult) Stop(MinecraftStopStrategy stopStrategy) {
-		if (stopStrategy == MinecraftStopStrategy.Instant) {
-			CancelDelayedStop();
-			return (PrepareStoppedState(), StopInstanceResult.StopInitiated);
-		}
-
-		if (isStopping) {
-			// TODO change delay or something
-			return (this, StopInstanceResult.InstanceAlreadyStopping);
-		}
-		
-		isStopping = true;
-		context.Services.TaskManager.Run("Delayed stop timer for instance " + context.ShortName, () => StopLater(stopStrategy.Seconds));
-		return (this, StopInstanceResult.StopInitiated);
-	}
-
-	private IInstanceState PrepareStoppedState() {
-		process.Ended -= ProcessEnded;
-		backupScheduler.Stop();
-		return new InstanceStoppingState(context, process, runningSessionDisposer);
-	}
-
-	private void CancelDelayedStop() {
-		try {
-			delayedStopCancellationTokenSource.Cancel();
-		} catch (ObjectDisposedException) {
-			// ignore
-		}
-	}
-
-	private async Task StopLater(int seconds) {
-		var cancellationToken = delayedStopCancellationTokenSource.Token;
-
-		try {
-			stateOwnsDelayedStopCancellationTokenSource = false;
-
-			int[] stops = { 60, 30, 10, 5, 4, 3, 2, 1, 0 };
-			
-			foreach (var stop in stops) {
-				if (seconds > stop) {
-					await SendCommand(MinecraftCommand.Say("Server shutting down in " + seconds + (seconds == 1 ? " second." : " seconds.")), cancellationToken);
-					await Task.Delay(TimeSpan.FromSeconds(seconds - stop), cancellationToken);
-					seconds = stop;
-				}
-			}
-		} catch (OperationCanceledException) {
-			context.Logger.Debug("Cancelled delayed stop.");
-			return;
-		} catch (ObjectDisposedException) {
-			return;
-		} catch (Exception e) {
-			context.Logger.Warning(e, "Caught exception during delayed stop.");
-			return;
-		} finally {
-			delayedStopCancellationTokenSource.Dispose();
-		}
-
-		context.TransitionState(PrepareStoppedState());
+	private void OnScheduledBackupCompleted(object? sender, BackupCreationResult e) {
+		context.ReportEvent(new InstanceBackupCompletedEvent(e.Kind, e.Warnings));
 	}
 
 	public async Task<bool> SendCommand(string command, CancellationToken cancellationToken) {
 		try {
 			context.Logger.Information("Sending command: {Command}", command);
-			await process.SendCommand(command, cancellationToken);
+			await Process.SendCommand(command, cancellationToken);
 			return true;
 		} catch (OperationCanceledException) {
 			return false;
@@ -137,21 +85,7 @@ sealed class InstanceRunningState : IInstanceState {
 		}
 	}
 
-	private void OnScheduledBackupCompleted(object? sender, BackupCreationResult e) {
-		context.ReportEvent(new InstanceBackupCompletedEvent(e.Kind, e.Warnings));
-	}
-
-	private sealed class RunningSessionDisposer : IDisposable {
-		private readonly InstanceRunningState state;
-		private readonly InstanceSession session;
-		private bool isDisposed;
-
-		public RunningSessionDisposer(InstanceRunningState state, InstanceSession session) {
-			this.state = state;
-			this.session = session;
-		}
-
-		public bool Dispose() {
+	private bool TryDispose() {
 		lock (this) {
 			if (isDisposed) {
 				return false;
@@ -160,19 +94,16 @@ sealed class InstanceRunningState : IInstanceState {
 			isDisposed = true;
 		}
 
-			if (state.stateOwnsDelayedStopCancellationTokenSource) {
-				state.delayedStopCancellationTokenSource.Dispose();
-			}
-			else {
-				state.CancelDelayedStop();
-			}
+		logSender.Stop();
+		backupScheduler.Stop();
+		
+		Process.Dispose();
+		context.Services.PortManager.Release(configuration);
 		
-			session.Dispose();
 		return true;
 	}
 
-		void IDisposable.Dispose() {
-			Dispose();
-		}
+	public void Dispose() {
+		TryDispose();
 	}
 }

Agent/Phantom.Agent.Services/Instances/States/InstanceStoppingState.cs

@@ -1,89 +0,0 @@
-using System.Diagnostics;
-using Phantom.Agent.Minecraft.Command;
-using Phantom.Agent.Minecraft.Instance;
-using Phantom.Common.Data.Instance;
-using Phantom.Common.Data.Minecraft;
-using Phantom.Common.Data.Replies;
-
-namespace Phantom.Agent.Services.Instances.States; 
-
-sealed class InstanceStoppingState : IInstanceState, IDisposable {
-	private readonly InstanceContext context;
-	private readonly InstanceProcess process;
-	private readonly IDisposable sessionDisposer;
-
-	public InstanceStoppingState(InstanceContext context, InstanceProcess process, IDisposable sessionDisposer) {
-		this.context = context;
-		this.process = process;
-		this.sessionDisposer = sessionDisposer;
-	}
-
-	public void Initialize() {
-		context.Logger.Information("Session stopping.");
-		context.SetStatus(InstanceStatus.Stopping);
-		context.Services.TaskManager.Run("Stop procedure for instance " + context.ShortName, DoStop);
-	}
-
-	private async Task DoStop() {
-		try {
-			// Do not release the semaphore after this point.
-			if (!await process.BackupSemaphore.CancelAndWait(TimeSpan.FromSeconds(1))) {
-				context.Logger.Information("Waiting for backup to finish...");
-				await process.BackupSemaphore.CancelAndWait(Timeout.InfiniteTimeSpan);
-			}
-			
-			context.Logger.Information("Sending stop command...");
-			await DoSendStopCommand();
-
-			context.Logger.Information("Waiting for session to end...");
-			await DoWaitForSessionToEnd();
-		} finally {
-			context.Logger.Information("Session stopped.");
-			context.ReportEvent(InstanceEvent.Stopped);
-			context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
-		}
-	}
-
-	private async Task DoSendStopCommand() {
-		using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
-		try {
-			await process.SendCommand(MinecraftCommand.Stop, timeout.Token);
-		} catch (OperationCanceledException) {
-			// ignore
-		} catch (ObjectDisposedException e) when (e.ObjectName == typeof(Process).FullName && process.HasEnded) {
-			// ignore
-		} catch (Exception e) {
-			context.Logger.Warning(e, "Caught exception while sending stop command.");
-		}
-	}
-
-	private async Task DoWaitForSessionToEnd() {
-		using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(55));
-		try {
-			await process.WaitForExit(timeout.Token);
-		} catch (OperationCanceledException) {
-			try {
-				context.Logger.Warning("Waiting timed out, killing session...");
-				process.Kill();
-			} catch (Exception e) {
-				context.Logger.Error(e, "Caught exception while killing session.");
-			}
-		}
-	}
-
-	public (IInstanceState, LaunchInstanceResult) Launch(InstanceContext context) {
-		return (this, LaunchInstanceResult.InstanceIsStopping);
-	}
-
-	public (IInstanceState, StopInstanceResult) Stop(MinecraftStopStrategy stopStrategy) {
-		return (this, StopInstanceResult.InstanceAlreadyStopping); // TODO maybe provide a way to kill?
-	}
-
-	public Task<bool> SendCommand(string command, CancellationToken cancellationToken) {
-		return Task.FromResult(false);
-	}
-
-	public void Dispose() {
-		sessionDisposer.Dispose();
-	}
-}

Agent/Phantom.Agent/Program.cs

@@ -17,13 +17,15 @@ PosixSignals.RegisterCancellation(shutdownCancellationTokenSource, static () =>
 	PhantomLogger.Root.InformationHeading("Stopping Phantom Panel agent...");
 });
 
+ThreadPool.SetMinThreads(workerThreads: 2, completionPortThreads: 1);
+
 try {
 	var fullVersion = AssemblyAttributes.GetFullVersion(Assembly.GetExecutingAssembly());
 
 	PhantomLogger.Root.InformationHeading("Initializing Phantom Panel agent...");
 	PhantomLogger.Root.Information("Agent version: {Version}", fullVersion);
 
-	var (serverHost, serverPort, javaSearchPath, agentKeyToken, agentKeyFilePath, agentName, maxInstances, maxMemory, allowedServerPorts, allowedRconPorts) = Variables.LoadOrExit();
+	var (serverHost, serverPort, javaSearchPath, agentKeyToken, agentKeyFilePath, agentName, maxInstances, maxMemory, allowedServerPorts, allowedRconPorts, maxConcurrentBackupCompressionTasks) = Variables.LoadOrExit();
 
 	var agentKey = await AgentKey.Load(agentKeyToken, agentKeyFilePath);
 	if (agentKey == null) {
@@ -42,7 +44,7 @@ try {
 
 	var (serverCertificate, agentToken) = agentKey.Value;
 	var agentInfo = new AgentInfo(agentGuid.Value, agentName, ProtocolVersion, fullVersion, maxInstances, maxMemory, allowedServerPorts, allowedRconPorts);
-	var agentServices = new AgentServices(agentInfo, folders);
+	var agentServices = new AgentServices(agentInfo, folders, new AgentServiceConfiguration(maxConcurrentBackupCompressionTasks));
 
 	MessageListener MessageListenerFactory(RpcServerConnection connection) {
 		return new MessageListener(connection, agentServices, shutdownCancellationTokenSource);

Agent/Phantom.Agent/Variables.cs

@@ -15,7 +15,8 @@ sealed record Variables(
 	ushort MaxInstances,
 	RamAllocationUnits MaxMemory,
 	AllowedPorts AllowedServerPorts,
-	AllowedPorts AllowedRconPorts
+	AllowedPorts AllowedRconPorts,
+	ushort MaxConcurrentBackupCompressionTasks
 ) {
 	private static Variables LoadOrThrow() {
 		var (agentKeyToken, agentKeyFilePath) = EnvironmentVariables.GetEitherString("AGENT_KEY", "AGENT_KEY_FILE").Require;
@@ -31,7 +32,8 @@ sealed record Variables(
 			(ushort) EnvironmentVariables.GetInteger("MAX_INSTANCES", min: 1, max: 10000).Require,
 			EnvironmentVariables.GetString("MAX_MEMORY").MapParse(RamAllocationUnits.FromString).Require,
 			EnvironmentVariables.GetString("ALLOWED_SERVER_PORTS").MapParse(AllowedPorts.FromString).Require,
-			EnvironmentVariables.GetString("ALLOWED_RCON_PORTS").MapParse(AllowedPorts.FromString).Require
+			EnvironmentVariables.GetString("ALLOWED_RCON_PORTS").MapParse(AllowedPorts.FromString).Require,
+			(ushort) EnvironmentVariables.GetInteger("MAX_CONCURRENT_BACKUP_COMPRESSION_TASKS", min: 1, max: 10000).WithDefault(1)
 		);
 	}
 

Common/Phantom.Common.Data/AllowedPorts.cs

@@ -53,6 +53,6 @@ public sealed partial class AllowedPorts {
 	}
 
 	public static AllowedPorts FromString(string definitions) {
-		return FromString((ReadOnlySpan<char>) definitions);
+		return FromString(definitions.AsSpan());
 	}
 }

Common/Phantom.Common.Data/Instance/InstanceLaunchFailReason.cs

@@ -1,17 +1,16 @@
 namespace Phantom.Common.Data.Instance;
 
 public enum InstanceLaunchFailReason : byte {
-	UnknownError,
-	ServerPortNotAllowed,
-	ServerPortAlreadyInUse,
-	RconPortNotAllowed,
-	RconPortAlreadyInUse,
-	JavaRuntimeNotFound,
-	InvalidJvmArguments,
-	CouldNotDownloadMinecraftServer,
-	CouldNotConfigureMinecraftServer,
-	CouldNotPrepareMinecraftServerLauncher,
-	CouldNotStartMinecraftServer
+	UnknownError                           = 0,
+	ServerPortNotAllowed                   = 1,
+	ServerPortAlreadyInUse                 = 2,
+	RconPortNotAllowed                     = 3,
+	RconPortAlreadyInUse                   = 4,
+	JavaRuntimeNotFound                    = 5,
+	CouldNotDownloadMinecraftServer        = 6,
+	CouldNotConfigureMinecraftServer       = 7,
+	CouldNotPrepareMinecraftServerLauncher = 8,
+	CouldNotStartMinecraftServer           = 9
 }
 
 public static class InstanceLaunchFailReasonExtensions {
@@ -22,7 +21,6 @@ public static class InstanceLaunchFailReasonExtensions {
 			InstanceLaunchFailReason.RconPortNotAllowed                     => "Rcon port not allowed.",
 			InstanceLaunchFailReason.RconPortAlreadyInUse                   => "Rcon port already in use.",
 			InstanceLaunchFailReason.JavaRuntimeNotFound                    => "Java runtime not found.",
-			InstanceLaunchFailReason.InvalidJvmArguments                    => "Invalid JVM arguments.",
 			InstanceLaunchFailReason.CouldNotDownloadMinecraftServer        => "Could not download Minecraft server.",
 			InstanceLaunchFailReason.CouldNotConfigureMinecraftServer       => "Could not configure Minecraft server.",
 			InstanceLaunchFailReason.CouldNotPrepareMinecraftServerLauncher => "Could not prepare Minecraft server launcher.",

Common/Phantom.Common.Data/RamAllocationUnits.cs

@@ -98,6 +98,6 @@ public readonly partial record struct RamAllocationUnits(
 	/// </summary>
 	/// <exception cref="ArgumentOutOfRangeException">If the <paramref name="definition"/> is in the incorrect format, or the value cannot be converted via <see cref="FromMegabytes"/>.</exception>
 	public static RamAllocationUnits FromString(string definition) {
-		return FromString((ReadOnlySpan<char>) definition);
+		return FromString(definition.AsSpan());
 	}
 }

Common/Phantom.Common.Data/Replies/LaunchInstanceResult.cs

@@ -1,13 +1,11 @@
 namespace Phantom.Common.Data.Replies;
 
 public enum LaunchInstanceResult : byte {
-	UnknownError,
-	LaunchInitiated,
-	InstanceAlreadyLaunching,
-	InstanceAlreadyRunning,
-	InstanceIsStopping,
-	InstanceLimitExceeded,
-	MemoryLimitExceeded
+	LaunchInitiated          = 1,
+	InstanceAlreadyLaunching = 2,
+	InstanceAlreadyRunning   = 3,
+	InstanceLimitExceeded    = 4,
+	MemoryLimitExceeded      = 5
 }
 
 public static class LaunchInstanceResultExtensions {
@@ -16,7 +14,6 @@ public static class LaunchInstanceResultExtensions {
 			LaunchInstanceResult.LaunchInitiated          => "Launch initiated.",
 			LaunchInstanceResult.InstanceAlreadyLaunching => "Instance is already launching.",
 			LaunchInstanceResult.InstanceAlreadyRunning   => "Instance is already running.",
-			LaunchInstanceResult.InstanceIsStopping       => "Instance is stopping.",
 			LaunchInstanceResult.InstanceLimitExceeded    => "Agent does not have any more available instances.",
 			LaunchInstanceResult.MemoryLimitExceeded      => "Agent does not have enough available memory.",
 			_                                             => "Unknown error."

Common/Phantom.Common.Data/Replies/StopInstanceResult.cs

@@ -1,10 +1,9 @@
 namespace Phantom.Common.Data.Replies;
 
 public enum StopInstanceResult : byte {
-	UnknownError,
-	StopInitiated,
-	InstanceAlreadyStopping,
-	InstanceAlreadyStopped
+	StopInitiated           = 1,
+	InstanceAlreadyStopping = 2,
+	InstanceAlreadyStopped  = 3
 }
 
 public static class StopInstanceResultExtensions {

Common/Phantom.Common.Messages/ToServer/ReportAgentStatusMessage.cs

@@ -5,7 +5,7 @@ using Phantom.Utils.Rpc.Message;
 namespace Phantom.Common.Messages.ToServer; 
 
 [MemoryPackable]
-public partial record ReportAgentStatusMessage(
+public sealed partial record ReportAgentStatusMessage(
 	[property: MemoryPackOrder(0)] int RunningInstanceCount,
 	[property: MemoryPackOrder(1)] RamAllocationUnits RunningInstanceMemory
 ) : IMessageToServer {

Common/Phantom.Common.Minecraft/Phantom.Common.Minecraft.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-  
-</Project>

Packages.props

@@ -1,11 +1,11 @@
 <Project>
   
   <ItemGroup>
-    <PackageReference Update="Microsoft.AspNetCore.Components.Authorization"     Version="7.0.1" />
-    <PackageReference Update="Microsoft.AspNetCore.Components.Web"               Version="7.0.1" />
-    <PackageReference Update="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.1" />
-    <PackageReference Update="Microsoft.EntityFrameworkCore.Tools"               Version="7.0.1" />
-    <PackageReference Update="Npgsql.EntityFrameworkCore.PostgreSQL"             Version="7.0.1" />
+    <PackageReference Update="Microsoft.AspNetCore.Components.Authorization"     Version="7.0.3" />
+    <PackageReference Update="Microsoft.AspNetCore.Components.Web"               Version="7.0.3" />
+    <PackageReference Update="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.3" />
+    <PackageReference Update="Microsoft.EntityFrameworkCore.Tools"               Version="7.0.3" />
+    <PackageReference Update="Npgsql.EntityFrameworkCore.PostgreSQL"             Version="7.0.3" />
   </ItemGroup>
   
   <ItemGroup>
@@ -13,7 +13,7 @@
   </ItemGroup>
   
   <ItemGroup>
-    <PackageReference Update="MemoryPack" Version="1.9.7" />
+    <PackageReference Update="MemoryPack" Version="1.9.13" />
     <PackageReference Update="NetMQ"      Version="4.0.1.10" />
   </ItemGroup>
   
@@ -26,10 +26,10 @@
   
   <ItemGroup>
     <PackageReference Update="coverlet.collector"     Version="3.2.0" />
-    <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Update="Microsoft.NET.Test.Sdk" Version="17.5.0" />
     <PackageReference Update="NUnit"                  Version="3.13.3" />
-    <PackageReference Update="NUnit.Analyzers"        Version="3.5.0" />
-    <PackageReference Update="NUnit3TestAdapter"      Version="4.3.1" />
+    <PackageReference Update="NUnit.Analyzers"        Version="3.6.0" />
+    <PackageReference Update="NUnit3TestAdapter"      Version="4.4.2" />
   </ItemGroup>
 
 </Project>

PhantomPanel.sln

@@ -26,8 +26,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Phantom.Common.Data.Tests",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Phantom.Common.Logging", "Common\Phantom.Common.Logging\Phantom.Common.Logging.csproj", "{D7F55010-B3ED-42A5-8D83-E754FFC5F2A2}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Phantom.Common.Minecraft", "Common\Phantom.Common.Minecraft\Phantom.Common.Minecraft.csproj", "{48278E42-17BB-442B-9877-ACCE0C02C268}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Phantom.Common.Messages", "Common\Phantom.Common.Messages\Phantom.Common.Messages.csproj", "{95B55357-F8F0-48C2-A1C2-5EA997651783}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Phantom.Server", "Server\Phantom.Server\Phantom.Server.csproj", "{A0F1C595-96B6-4DBF-8C16-6B99223F8F35}"
@@ -100,10 +98,6 @@ Global
 		{D7F55010-B3ED-42A5-8D83-E754FFC5F2A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D7F55010-B3ED-42A5-8D83-E754FFC5F2A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D7F55010-B3ED-42A5-8D83-E754FFC5F2A2}.Release|Any CPU.Build.0 = Release|Any CPU
-		{48278E42-17BB-442B-9877-ACCE0C02C268}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{48278E42-17BB-442B-9877-ACCE0C02C268}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{48278E42-17BB-442B-9877-ACCE0C02C268}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{48278E42-17BB-442B-9877-ACCE0C02C268}.Release|Any CPU.Build.0 = Release|Any CPU
 		{95B55357-F8F0-48C2-A1C2-5EA997651783}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{95B55357-F8F0-48C2-A1C2-5EA997651783}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{95B55357-F8F0-48C2-A1C2-5EA997651783}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -184,7 +178,6 @@ Global
 		{AEE8B77E-AB07-423F-9981-8CD829ACB834} = {F5878792-64C8-4ECF-A075-66341FF97127}
 		{6C3DB1E5-F695-4D70-8F3A-78C2957274BE} = {01CB1A81-8950-471C-BFDF-F135FDDB2C18}
 		{D7F55010-B3ED-42A5-8D83-E754FFC5F2A2} = {01CB1A81-8950-471C-BFDF-F135FDDB2C18}
-		{48278E42-17BB-442B-9877-ACCE0C02C268} = {01CB1A81-8950-471C-BFDF-F135FDDB2C18}
 		{95B55357-F8F0-48C2-A1C2-5EA997651783} = {01CB1A81-8950-471C-BFDF-F135FDDB2C18}
 		{435D7981-DFDA-46A0-8CD8-CD8C117935D7} = {D781E00D-8563-4102-A0CD-477A679193B5}
 		{A0F1C595-96B6-4DBF-8C16-6B99223F8F35} = {8AC8FB6C-033A-4626-820F-ED0F908756B2}

README.md

@@ -92,6 +92,7 @@ Use volumes to persist either the whole `/data` folder, or just `/data/data` if
 * **Agent Configuration**
   - `MAX_INSTANCES` is the number of instances that can be created.
   - `MAX_MEMORY` is the maximum amount of RAM that can be distributed among all instances. Use a positive integer with an optional suffix 'M' for MB, or 'G' for GB. Examples: `4096M`, `16G`
+  - `MAX_CONCURRENT_BACKUP_COMPRESSION_TASKS` is how many backup compression tasks can run at the same time. Limiting concurrent compression tasks limits memory usage of compression, but it increases time between backups because the next backup is only scheduled once the current one completes. Default: `1`
 * **Minecraft Configuration**
   - `JAVA_SEARCH_PATH` is a path to a folder which will be searched for Java runtime installations. Linux default: `/usr/lib/jvm`
   - `ALLOWED_SERVER_PORTS` is a comma-separated list of ports and port ranges that can be used as Minecraft Server ports. Example: `25565,25900,26000-27000`

Server/Phantom.Server.Database/Enums/AuditLogEventType.cs

@@ -15,7 +15,7 @@ public enum AuditLogEventType {
 	InstanceCommandExecuted
 }
 
-public static class AuditLogEventTypeExtensions {
+static class AuditLogEventTypeExtensions {
 	private static readonly Dictionary<AuditLogEventType, AuditLogSubjectType> SubjectTypes = new () {
 		{ AuditLogEventType.AdministratorUserCreated,  AuditLogSubjectType.User },
 		{ AuditLogEventType.AdministratorUserModified, AuditLogSubjectType.User },

Server/Phantom.Server.Database/Enums/EventLogEventType.cs

@@ -10,7 +10,7 @@ public enum EventLogEventType {
 	InstanceBackupFailed,
 }
 
-internal static class EventLogEventTypeExtensions {
+static class EventLogEventTypeExtensions {
 	private static readonly Dictionary<EventLogEventType, EventLogSubjectType> SubjectTypes = new () {
 		{ EventLogEventType.InstanceLaunchSucceded, EventLogSubjectType.Instance },
 		{ EventLogEventType.InstanceLaunchFailed, EventLogSubjectType.Instance },

Server/Phantom.Server.Minecraft/JvmArgumentsHelper.cs

@@ -1,17 +1,21 @@
 using System.Collections.Immutable;
 
-namespace Phantom.Common.Minecraft;
+namespace Phantom.Server.Minecraft;
 
 public static class JvmArgumentsHelper {
 	public static ImmutableArray<string> Split(string arguments) {
 		return arguments.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToImmutableArray();
 	}
 
+	public static string Join(ImmutableArray<string> arguments) {
+		return string.Join('\n', arguments);
+	}
+
 	public static ValidationError? Validate(string arguments) {
 		return Validate(Split(arguments));
 	}
 
-	public static ValidationError? Validate(ImmutableArray<string> arguments) {
+	private static ValidationError? Validate(ImmutableArray<string> arguments) {
 		if (!arguments.All(static argument => argument.StartsWith('-'))) {
 			return ValidationError.InvalidFormat;
 		}
@@ -28,10 +32,6 @@ public static class JvmArgumentsHelper {
 		return null;
 	}
 
-	public static string Join(ImmutableArray<string> arguments) {
-		return string.Join('\n', arguments);
-	}
-
 	public enum ValidationError {
 		InvalidFormat,
 		XmxNotAllowed,

Server/Phantom.Server.Rpc/RpcLauncher.cs

@@ -11,14 +11,14 @@ using Serilog.Events;
 namespace Phantom.Server.Rpc;
 
 public sealed class RpcLauncher : RpcRuntime<ServerSocket> {
-	public static async Task Launch(RpcConfiguration config, Func<RpcClientConnection, IMessageToServerListener> listenerFactory, CancellationToken cancellationToken) {
+	public static Task Launch(RpcConfiguration config, Func<RpcClientConnection, IMessageToServerListener> listenerFactory, CancellationToken cancellationToken) {
 		var socket = new ServerSocket();
 		var options = socket.Options;
 
 		options.CurveServer = true;
 		options.CurveCertificate = config.ServerCertificate;
 
-		await new RpcLauncher(config, socket, listenerFactory, cancellationToken).Launch();
+		return new RpcLauncher(config, socket, listenerFactory, cancellationToken).Launch();
 	}
 
 	private readonly RpcConfiguration config;

Server/Phantom.Server.Services/Audit/AuditLog.EventTypes.cs

@@ -25,9 +25,7 @@ public sealed partial class AuditLog {
 	}
 
 	public Task AddUserRolesChangedEvent(IdentityUser user, List<string> addedToRoles, List<string> removedFromRoles) {
-		var extra = new Dictionary<string, object?> {
-			{ "username", user.UserName },
-		};
+		var extra = new Dictionary<string, object?>();
 		
 		if (addedToRoles.Count > 0) {
 			extra["addedToRoles"] = addedToRoles;
@@ -37,7 +35,7 @@ public sealed partial class AuditLog {
 			extra["removedFromRoles"] = removedFromRoles;
 		}
 		
-		return AddItem(AuditLogEventType.UserDeleted, user.Id, extra);
+		return AddItem(AuditLogEventType.UserRolesChanged, user.Id, extra);
 	}
 	
 	public Task AddUserDeletedEvent(IdentityUser user) {

Server/Phantom.Server.Services/Instances/InstanceManager.cs

@@ -2,12 +2,12 @@
 using System.Diagnostics.CodeAnalysis;
 using Phantom.Common.Data;
 using Phantom.Common.Data.Instance;
+using Phantom.Common.Data.Java;
 using Phantom.Common.Data.Minecraft;
 using Phantom.Common.Data.Replies;
 using Phantom.Common.Logging;
 using Phantom.Common.Messages;
 using Phantom.Common.Messages.ToAgent;
-using Phantom.Common.Minecraft;
 using Phantom.Server.Database;
 using Phantom.Server.Database.Entities;
 using Phantom.Server.Minecraft;

Server/Phantom.Server.Services/Phantom.Server.Services.csproj

@@ -12,7 +12,6 @@
   
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Phantom.Common.Data\Phantom.Common.Data.csproj" />
-    <ProjectReference Include="..\..\Common\Phantom.Common.Minecraft\Phantom.Common.Minecraft.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Collections\Phantom.Utils.Collections.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Events\Phantom.Utils.Events.csproj" />
     <ProjectReference Include="..\Phantom.Server.Database\Phantom.Server.Database.csproj" />

Server/Phantom.Server.Web.Identity/Authentication/RevalidatingIdentityAuthenticationStateProvider.cs

@@ -2,8 +2,6 @@
 using Microsoft.AspNetCore.Components.Authorization;
 using Microsoft.AspNetCore.Components.Server;
 using Microsoft.AspNetCore.Identity;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace Phantom.Server.Web.Identity.Authentication;

Server/Phantom.Server.Web.Identity/PhantomIdentityExtensions.cs

@@ -1,10 +1,7 @@
 using Microsoft.AspNetCore.Authentication.Cookies;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Components.Authorization;
-using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Identity;
-using Microsoft.Extensions.DependencyInjection;
 using Phantom.Server.Database;
 using Phantom.Server.Web.Identity.Authentication;
 using Phantom.Server.Web.Identity.Authorization;

Server/Phantom.Server.Web.Identity/PhantomIdentityMiddleware.cs

@@ -1,5 +1,4 @@
 using System.Diagnostics.CodeAnalysis;
-using Microsoft.AspNetCore.Http;
 using Phantom.Server.Web.Identity.Authentication;
 using Phantom.Server.Web.Identity.Interfaces;
 

Server/Phantom.Server.Web/Shared/InstanceAddOrEditForm.razor

@@ -1,5 +1,4 @@
 @using Phantom.Common.Data.Minecraft
-@using Phantom.Common.Minecraft
 @using Phantom.Server.Minecraft
 @using Phantom.Server.Services.Agents
 @using Phantom.Server.Services.Audit
@@ -7,10 +6,10 @@
 @using System.Collections.Immutable
 @using System.ComponentModel.DataAnnotations
 @using System.Diagnostics.CodeAnalysis
+@using Phantom.Common.Data.Java
 @using Phantom.Server.Web.Components.Utils
 @using Phantom.Server.Web.Identity.Interfaces
 @using Phantom.Common.Data.Instance
-@using Phantom.Common.Data.Java
 @using Phantom.Common.Data
 @inject INavigation Nav
 @inject MinecraftVersions MinecraftVersions

Utils/Phantom.Utils.Collections/Table.cs

@@ -68,10 +68,14 @@ public sealed class Table<TRow, TKey> : IReadOnlyList<TRow>, IReadOnlyDictionary
 		}
 	}
 
-	public IEnumerator<TRow> GetEnumerator() {
+	public List<TRow>.Enumerator GetEnumerator() {
 		return rowList.GetEnumerator();
 	}
 
+	IEnumerator<TRow> IEnumerable<TRow>.GetEnumerator() {
+		return GetEnumerator();
+	}
+
 	IEnumerator IEnumerable.GetEnumerator() {
 		return GetEnumerator();
 	}

Utils/Phantom.Utils.Collections/ThreadSafeLinkedList.cs

@@ -0,0 +1,39 @@
+namespace Phantom.Utils.Collections; 
+
+public sealed class ThreadSafeLinkedList<T> : IDisposable {
+	private readonly LinkedList<T> list = new ();
+	private readonly SemaphoreSlim semaphore = new (1, 1);
+
+	public async Task Add(T item, bool toFront, CancellationToken cancellationToken) {
+		await semaphore.WaitAsync(cancellationToken);
+		try {
+			if (toFront) {
+				list.AddFirst(item);
+			}
+			else {
+				list.AddLast(item);
+			}
+		} finally {
+			semaphore.Release();
+		}
+	}
+
+	public async Task<T?> TryTakeFromFront(CancellationToken cancellationToken) {
+		await semaphore.WaitAsync(cancellationToken);
+		try {
+			var firstNode = list.First;
+			if (firstNode == null) {
+				return default;
+			}
+
+			list.RemoveFirst();
+			return firstNode.Value;
+		} finally {
+			semaphore.Release();
+		}
+	}
+
+	public void Dispose() {
+		semaphore.Dispose();
+	}
+}

Utils/Phantom.Utils.Rpc/Message/MessageHandler.cs

@@ -19,7 +19,7 @@ public abstract class MessageHandler<TListener> {
 	
 	internal void Enqueue<TMessage, TReply>(uint sequenceId, TMessage message) where TMessage : IMessage<TListener, TReply> {
 		cancellationToken.ThrowIfCancellationRequested();
-		taskManager.Run("Handle message {Type}" + message.GetType().Name, async () => {
+		taskManager.Run("Handle message " + message.GetType().Name, async () => {
 			try {
 				await Handle<TMessage, TReply>(sequenceId, message);
 			} catch (Exception e) {

Utils/Phantom.Utils.Rpc/Message/MessageSerializer.cs

@@ -8,7 +8,7 @@ static class MessageSerializer {
 	private static readonly MemoryPackSerializerOptions SerializerOptions = MemoryPackSerializerOptions.Utf8;
 
 	public static byte[] Serialize<T>(T message) {
-		return MemoryPackSerializer.Serialize(typeof(T), message, SerializerOptions);
+		return MemoryPackSerializer.Serialize(message, SerializerOptions);
 	}
 
 	public static void Serialize<T>(IBufferWriter<byte> destination, T message) {

Utils/Phantom.Utils.Runtime/CancellableBackgroundTask.cs

@@ -8,9 +8,18 @@ public abstract class CancellableBackgroundTask {
 	protected ILogger Logger { get; }
 	protected CancellationToken CancellationToken { get; }
 
+	private readonly TaskManager taskManager;
+	private readonly string taskName;
+	
 	protected CancellableBackgroundTask(ILogger logger, TaskManager taskManager, string taskName) {
 		this.Logger = logger;
 		this.CancellationToken = cancellationTokenSource.Token;
+		
+		this.taskManager = taskManager;
+		this.taskName = taskName;
+	}
+
+	protected void Start() {
 		taskManager.Run(taskName, Run);
 	}
 
@@ -25,12 +34,15 @@ public abstract class CancellableBackgroundTask {
 			Logger.Fatal(e, "Caught exception in task.");
 		} finally {
 			cancellationTokenSource.Dispose();
+			Dispose();
 			Logger.Debug("Task stopped.");
 		}
 	}
 	
 	protected abstract Task RunTask();
 	
+	protected abstract void Dispose();
+
 	public void Stop() {
 		try {
 			cancellationTokenSource.Cancel();

Utils/Phantom.Utils.Runtime/CancellableSemaphore.cs

@@ -1,28 +0,0 @@
-namespace Phantom.Utils.Runtime; 
-
-public sealed class CancellableSemaphore : IDisposable {
-	private readonly SemaphoreSlim semaphore;
-	private readonly CancellationTokenSource cancellationTokenSource = new ();
-	
-	public CancellableSemaphore(int initialCount, int maxCount) {
-		this.semaphore = new SemaphoreSlim(initialCount, maxCount);
-	}
-
-	public async Task<bool> Wait(TimeSpan timeout, CancellationToken cancellationToken) {
-		return await semaphore.WaitAsync(timeout, cancellationTokenSource.Token).WaitAsync(cancellationToken);
-	}
-
-	public async Task<bool> CancelAndWait(TimeSpan timeout) {
-		cancellationTokenSource.Cancel();
-		return await semaphore.WaitAsync(timeout);
-	}
-
-	public void Release() {
-		semaphore.Release();
-	}
-	
-	public void Dispose() {
-		semaphore.Dispose();
-		cancellationTokenSource.Dispose();
-	}
-}

Utils/Phantom.Utils.Runtime/ThreadSafeStructRef.cs

@@ -0,0 +1,28 @@
+namespace Phantom.Utils.Runtime; 
+
+public sealed class ThreadSafeStructRef<T> : IDisposable where T : struct {
+	private T? value;
+	private readonly SemaphoreSlim semaphore = new (1, 1);
+	
+	public async Task<T?> Get(CancellationToken cancellationToken) {
+		await semaphore.WaitAsync(cancellationToken);
+		try {
+			return value;
+		} finally {
+			semaphore.Release();
+		}
+	}
+
+	public async Task Set(T? value, CancellationToken cancellationToken) {
+		await semaphore.WaitAsync(cancellationToken);
+		try {
+			this.value = value;
+		} finally {
+			semaphore.Release();
+		}
+	}
+
+	public void Dispose() {
+		semaphore.Dispose();
+	}
+}