Change backup world copying to use the dedicated temporary folder

Agent/Phantom.Agent.Minecraft/Instance/InstanceSession.cs

@@ -1,10 +1,12 @@
 using System.Diagnostics;
 using Phantom.Utils.Collections;
+using Phantom.Utils.Runtime;
 
 namespace Phantom.Agent.Minecraft.Instance; 
 
 public sealed class InstanceSession : IDisposable {
 	public InstanceProperties InstanceProperties { get; }
+	public CancellableSemaphore BackupSemaphore { get; } = new (1, 1);
 	
 	private readonly RingBuffer<string> outputBuffer = new (10000);
 	private event EventHandler<string>? OutputEvent;
@@ -64,6 +66,7 @@ public sealed class InstanceSession : IDisposable {
 
 	public void Dispose() {
 		process.Dispose();
+		BackupSemaphore.Dispose();
 		OutputEvent = null;
 		SessionEnded = null;
 	}

Agent/Phantom.Agent.Minecraft/Launcher/LaunchServices.cs

@@ -1,7 +1,6 @@
 using Phantom.Agent.Minecraft.Java;
 using Phantom.Agent.Minecraft.Server;
-using Phantom.Utils.Runtime;
 
 namespace Phantom.Agent.Minecraft.Launcher; 
 
-public sealed record LaunchServices(TaskManager TaskManager, MinecraftServerExecutables ServerExecutables, JavaRuntimeRepository JavaRuntimeRepository);
+public sealed record LaunchServices(MinecraftServerExecutables ServerExecutables, JavaRuntimeRepository JavaRuntimeRepository);

Agent/Phantom.Agent.Minecraft/Server/ServerStatusProtocol.cs

@@ -0,0 +1,93 @@
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using Phantom.Common.Logging;
+using Serilog;
+
+namespace Phantom.Agent.Minecraft.Server; 
+
+public sealed class ServerStatusProtocol {
+	private readonly ILogger logger;
+
+	public ServerStatusProtocol(string loggerName) {
+		this.logger = PhantomLogger.Create<ServerStatusProtocol>(loggerName);
+	}
+
+	public async Task<int?> GetOnlinePlayerCount(int serverPort, CancellationToken cancellationToken) {
+		try {
+			return await GetOnlinePlayerCountOrThrow(serverPort, cancellationToken);
+		} catch (Exception e) {
+			logger.Error(e, "Caught exception while checking if players are online.");
+			return null;
+		}
+	}
+
+	private async Task<int?> GetOnlinePlayerCountOrThrow(int serverPort, CancellationToken cancellationToken) {
+		using var tcpClient = new TcpClient();
+		await tcpClient.ConnectAsync(IPAddress.Loopback, serverPort, cancellationToken);
+		var tcpStream = tcpClient.GetStream();
+		
+		// https://wiki.vg/Server_List_Ping
+		tcpStream.WriteByte(0xFE);
+		await tcpStream.FlushAsync(cancellationToken);
+
+		short? messageLength = await ReadStreamHeader(tcpStream, cancellationToken);
+		return messageLength == null ? null : await ReadOnlinePlayerCount(tcpStream, messageLength.Value * 2, cancellationToken);
+	}
+
+	private async Task<short?> ReadStreamHeader(NetworkStream tcpStream, CancellationToken cancellationToken) {
+		var headerBuffer = ArrayPool<byte>.Shared.Rent(3);
+		try {
+			await tcpStream.ReadExactlyAsync(headerBuffer, 0, 3, cancellationToken);
+
+			if (headerBuffer[0] != 0xFF) {
+				logger.Error("Unexpected first byte in response from server: {FirstByte}.", headerBuffer[0]);
+				return null;
+			}
+
+			short messageLength = BinaryPrimitives.ReadInt16BigEndian(headerBuffer.AsSpan(1));
+			if (messageLength <= 0) {
+				logger.Error("Unexpected message length in response from server: {MessageLength}.", messageLength);
+				return null;
+			}
+			
+			return messageLength;
+		} finally {
+			ArrayPool<byte>.Shared.Return(headerBuffer);
+		}
+	}
+
+	private async Task<int?> ReadOnlinePlayerCount(NetworkStream tcpStream, int messageLength, CancellationToken cancellationToken) {
+		var messageBuffer = ArrayPool<byte>.Shared.Rent(messageLength);
+		try {
+			await tcpStream.ReadExactlyAsync(messageBuffer, 0, messageLength, cancellationToken);
+
+			// Valid response separator encoded in UTF-16BE is 0x00 0xA7 (§).
+			const byte SeparatorSecondByte = 0xA7;
+			
+			static bool IsValidSeparator(ReadOnlySpan<byte> buffer, int index) {
+				return index > 0 && buffer[index - 1] == 0x00;
+			}
+			
+			int separator2 = Array.LastIndexOf(messageBuffer, SeparatorSecondByte);
+			int separator1 = separator2 == -1 ? -1 : Array.LastIndexOf(messageBuffer, SeparatorSecondByte, separator2 - 1);
+			if (!IsValidSeparator(messageBuffer, separator1) || !IsValidSeparator(messageBuffer, separator2)) {
+				logger.Error("Could not find message separators in response from server.");
+				return null;
+			}
+
+			string onlinePlayerCountStr = Encoding.BigEndianUnicode.GetString(messageBuffer[(separator1 + 1)..(separator2 - 1)]);
+			if (!int.TryParse(onlinePlayerCountStr, out int onlinePlayerCount)) {
+				logger.Error("Could not parse online player count in response from server: {OnlinePlayerCount}.", onlinePlayerCountStr);
+				return null;
+			}
+			
+			logger.Verbose("Detected {OnlinePlayerCount} online player(s).", onlinePlayerCount);
+			return onlinePlayerCount;
+		} finally {
+			ArrayPool<byte>.Shared.Return(messageBuffer);
+		}
+	}
+}

Agent/Phantom.Agent.Services/AgentServices.cs

@@ -23,7 +23,7 @@ public sealed class AgentServices {
 		this.TaskManager = new TaskManager(PhantomLogger.Create<TaskManager, AgentServices>());
 		this.BackupManager = new BackupManager(agentFolders);
 		this.JavaRuntimeRepository = new JavaRuntimeRepository();
-		this.InstanceSessionManager = new InstanceSessionManager(agentInfo, agentFolders, JavaRuntimeRepository, TaskManager);
+		this.InstanceSessionManager = new InstanceSessionManager(agentInfo, agentFolders, JavaRuntimeRepository, TaskManager, BackupManager);
 	}
 
 	public async Task Initialize() {

Agent/Phantom.Agent.Services/Backups/BackupArchiver.cs

@@ -10,11 +10,15 @@ using Serilog;
 namespace Phantom.Agent.Services.Backups; 
 
 sealed class BackupArchiver {
+	private readonly string destinationBasePath;
+	private readonly string temporaryBasePath;
 	private readonly ILogger logger;
 	private readonly InstanceProperties instanceProperties;
 	private readonly CancellationToken cancellationToken;
 	
-	public BackupArchiver(string loggerName, InstanceProperties instanceProperties, CancellationToken cancellationToken) {
+	public BackupArchiver(string destinationBasePath, string temporaryBasePath, string loggerName, InstanceProperties instanceProperties, CancellationToken cancellationToken) {
+		this.destinationBasePath = destinationBasePath;
+		this.temporaryBasePath = temporaryBasePath;
 		this.logger = PhantomLogger.Create<BackupArchiver>(loggerName);
 		this.instanceProperties = instanceProperties;
 		this.cancellationToken = cancellationToken;
@@ -40,12 +44,14 @@ sealed class BackupArchiver {
 		return false;
 	}
 
-	public async Task ArchiveWorld(string destinationPath, BackupCreationResult.Builder resultBuilder) {
+	public async Task ArchiveWorld(BackupCreationResult.Builder resultBuilder) {
-		string backupFolderPath = Path.Combine(destinationPath, instanceProperties.InstanceGuid.ToString());
+		string guid = instanceProperties.InstanceGuid.ToString();
-		string backupFilePath = Path.Combine(backupFolderPath, DateTime.Now.ToString("yyyyMMdd-HHmmss") + ".tar");
+		string currentDateTime = DateTime.Now.ToString("yyyyMMdd-HHmmss");
+		string backupFolderPath = Path.Combine(destinationBasePath, guid);
+		string backupFilePath = Path.Combine(backupFolderPath, currentDateTime + ".tar");
 		
 		if (File.Exists(backupFilePath)) {
-			resultBuilder.Kind = BackupCreationResultKind.BackupAlreadyExists;
+			resultBuilder.Kind = BackupCreationResultKind.BackupFileAlreadyExists;
 			logger.Warning("Skipping backup, file already exists: {File}", backupFilePath);
 			return;
 		}
@@ -58,7 +64,8 @@ sealed class BackupArchiver {
 			return;
 		}
 
-		if (!await CopyWorldAndCreateTarArchive(backupFolderPath, backupFilePath, resultBuilder)) {
+		string temporaryFolderPath = Path.Combine(temporaryBasePath, guid + "_" + currentDateTime);
+		if (!await CopyWorldAndCreateTarArchive(temporaryFolderPath, backupFilePath, resultBuilder)) {
 			return;
 		}
 		
@@ -68,9 +75,7 @@ sealed class BackupArchiver {
 		}
 	}
 
-	private async Task<bool> CopyWorldAndCreateTarArchive(string backupFolderPath, string backupFilePath, BackupCreationResult.Builder resultBuilder) {
+	private async Task<bool> CopyWorldAndCreateTarArchive(string temporaryFolderPath, string backupFilePath, BackupCreationResult.Builder resultBuilder) {
-		string temporaryFolderPath = Path.Combine(backupFolderPath, "temp");
-		
 		try {
 			if (!await CopyWorldToTemporaryFolder(temporaryFolderPath)) {
 				resultBuilder.Kind = BackupCreationResultKind.CouldNotCopyWorldToTemporaryFolder;
@@ -109,10 +114,21 @@ sealed class BackupArchiver {
 			return true;
 		} catch (Exception e) {
 			logger.Error(e, "Could not create archive.");
+			DeleteBrokenArchiveFile(backupFilePath);
 			return false;
 		}
 	}
 
+	private void DeleteBrokenArchiveFile(string filePath) {
+		if (File.Exists(filePath)) {
+			try {
+				File.Delete(filePath);
+			} catch (Exception e) {
+				logger.Error(e, "Could not delete broken archive: {File}", filePath);
+			}
+		}
+	}
+
 	private async Task CopyDirectory(DirectoryInfo sourceFolder, string destinationFolderPath, ImmutableList<string> relativePath) {
 		cancellationToken.ThrowIfCancellationRequested();
 

Agent/Phantom.Agent.Services/Backups/BackupManager.cs

@@ -8,26 +8,44 @@ using Serilog;
 namespace Phantom.Agent.Services.Backups;
 
 sealed partial class BackupManager {
-	private readonly string basePath;
+	private readonly string destinationBasePath;
+	private readonly string temporaryBasePath;
 
 	public BackupManager(AgentFolders agentFolders) {
-		this.basePath = agentFolders.BackupsFolderPath;
+		this.destinationBasePath = agentFolders.BackupsFolderPath;
+		this.temporaryBasePath = Path.Combine(agentFolders.TemporaryFolderPath, "backups");
 	}
 
 	public async Task<BackupCreationResult> CreateBackup(string loggerName, InstanceSession session, CancellationToken cancellationToken) {
-		return await new BackupCreator(basePath, loggerName, session, cancellationToken).CreateBackup();
+		try {
+			if (!await session.BackupSemaphore.Wait(TimeSpan.FromSeconds(1), cancellationToken)) {
+				return new BackupCreationResult(BackupCreationResultKind.BackupAlreadyRunning);
+			}
+		} catch (ObjectDisposedException) {
+			return new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning);
+		} catch (OperationCanceledException) {
+			return new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning);
+		}
+
+		try {
+			return await new BackupCreator(destinationBasePath, temporaryBasePath, loggerName, session, cancellationToken).CreateBackup();
+		} finally {
+			session.BackupSemaphore.Release();
+		}
 	}
 
 	private sealed class BackupCreator {
-		private readonly string basePath;
+		private readonly string destinationBasePath;
+		private readonly string temporaryBasePath;
 		private readonly string loggerName;
 		private readonly ILogger logger;
 		private readonly InstanceSession session;
 		private readonly BackupCommandListener listener;
 		private readonly CancellationToken cancellationToken;
 
-		public BackupCreator(string basePath, string loggerName, InstanceSession session, CancellationToken cancellationToken) {
+		public BackupCreator(string destinationBasePath, string temporaryBasePath, string loggerName, InstanceSession session, CancellationToken cancellationToken) {
-			this.basePath = basePath;
+			this.destinationBasePath = destinationBasePath;
+			this.temporaryBasePath = temporaryBasePath;
 			this.loggerName = loggerName;
 			this.logger = PhantomLogger.Create<BackupManager>(loggerName);
 			this.session = session;
@@ -36,21 +54,38 @@ sealed partial class BackupManager {
 		}
 
 		public async Task<BackupCreationResult> CreateBackup() {
+			logger.Information("Backup started.");
 			session.AddOutputListener(listener.OnOutput, 0);
 			try {
 				var resultBuilder = new BackupCreationResult.Builder();
-				await RunBackupProcess(resultBuilder);
+				
-				return resultBuilder.Build();
+				await RunBackupProcedure(resultBuilder);
+				
+				var result = resultBuilder.Build();
+				if (result.Kind == BackupCreationResultKind.Success) {
+					var warningCount = result.Warnings.Count();
+					if (warningCount == 0) {
+						logger.Information("Backup finished successfully.");
+					}
+					else {
+						logger.Warning("Backup finished with {Warnings} warning(s).", warningCount);
+					}
+				}
+				else {
+					logger.Warning("Backup failed: {Reason}", result.Kind.ToSentence());
+				}
+				
+				return result;
 			} finally {
 				session.RemoveOutputListener(listener.OnOutput);
 			}
 		}
 		
-		private async Task RunBackupProcess(BackupCreationResult.Builder resultBuilder) {
+		private async Task RunBackupProcedure(BackupCreationResult.Builder resultBuilder) {
 			try {
 				await DisableAutomaticSaving();
 				await SaveAllChunks();
-				await new BackupArchiver(loggerName, session.InstanceProperties, cancellationToken).ArchiveWorld(basePath, resultBuilder);
+				await new BackupArchiver(destinationBasePath, temporaryBasePath, loggerName, session.InstanceProperties, cancellationToken).ArchiveWorld(resultBuilder);
 			} catch (OperationCanceledException) {
 				resultBuilder.Kind = BackupCreationResultKind.BackupCancelled;
 				logger.Warning("Backup creation was cancelled.");

Agent/Phantom.Agent.Services/Backups/BackupScheduler.cs

@@ -0,0 +1,75 @@
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Minecraft.Server;
+using Phantom.Common.Data.Backups;
+using Phantom.Common.Logging;
+using Phantom.Utils.Runtime;
+
+namespace Phantom.Agent.Services.Backups;
+
+sealed class BackupScheduler : CancellableBackgroundTask {
+	// TODO make configurable
+	private static readonly TimeSpan InitialDelay = TimeSpan.FromMinutes(2);
+	private static readonly TimeSpan BackupInterval = TimeSpan.FromMinutes(30);
+	private static readonly TimeSpan BackupFailureRetryDelay = TimeSpan.FromMinutes(5);
+	private static readonly TimeSpan OnlinePlayersCheckInterval = TimeSpan.FromMinutes(1);
+
+	private readonly string loggerName;
+	private readonly BackupManager backupManager;
+	private readonly InstanceSession session;
+	private readonly int serverPort;
+	private readonly ServerStatusProtocol serverStatusProtocol;
+
+	public BackupScheduler(TaskManager taskManager, BackupManager backupManager, InstanceSession session, int serverPort, string loggerName) : base(PhantomLogger.Create<BackupScheduler>(loggerName), taskManager, "Backup scheduler for " + loggerName) {
+		this.loggerName = loggerName;
+		this.backupManager = backupManager;
+		this.session = session;
+		this.serverPort = serverPort;
+		this.serverStatusProtocol = new ServerStatusProtocol(loggerName);
+	}
+
+	protected override async Task RunTask() {
+		await Task.Delay(InitialDelay, CancellationToken);
+		Logger.Information("Starting a new backup after server launched.");
+			
+		while (!CancellationToken.IsCancellationRequested) {
+			var result = await CreateBackup();
+			if (result.Kind.ShouldRetry()) {
+				Logger.Warning("Scheduled backup failed, retrying in {Minutes} minutes.", BackupFailureRetryDelay.TotalMinutes);
+				await Task.Delay(BackupFailureRetryDelay, CancellationToken);
+			}
+			else {
+				Logger.Warning("Scheduling next backup in {Minutes} minutes.", BackupInterval.TotalMinutes);
+				await Task.Delay(BackupInterval, CancellationToken);
+				await WaitForOnlinePlayers();
+			}
+		}
+	}
+
+	private async Task<BackupCreationResult> CreateBackup() {
+		return await backupManager.CreateBackup(loggerName, session, CancellationToken.None);
+	}
+
+	private async Task WaitForOnlinePlayers() {
+		bool needsToLogOfflinePlayersMessage = true;
+		
+		while (!CancellationToken.IsCancellationRequested) {
+			var onlinePlayerCount = await serverStatusProtocol.GetOnlinePlayerCount(serverPort, CancellationToken);
+			if (onlinePlayerCount == null) {
+				Logger.Warning("Could not detect whether any players are online, starting a new backup.");
+				break;
+			}
+			
+			if (onlinePlayerCount > 0) {
+				Logger.Information("Players are online, starting a new backup.");
+				break;
+			}
+			
+			if (needsToLogOfflinePlayersMessage) {
+				needsToLogOfflinePlayersMessage = false;
+				Logger.Information("No players are online, waiting for someone to join before starting a new backup.");
+			}
+
+			await Task.Delay(OnlinePlayersCheckInterval, CancellationToken);
+		}
+	}
+}

Agent/Phantom.Agent.Services/Instances/Instance.cs

@@ -18,21 +18,19 @@ sealed class Instance : IDisposable {
 		return prefix[..prefix.IndexOf('-')] + "/" + Interlocked.Increment(ref loggerSequenceId);
 	}
 
-	public static async Task<Instance> Create(InstanceConfiguration configuration, BaseLauncher launcher, LaunchServices launchServices, PortManager portManager) {
+	public static async Task<Instance> Create(InstanceConfiguration configuration, InstanceServices services, BaseLauncher launcher) {
-		var instance = new Instance(configuration, launcher, launchServices, portManager);
+		var instance = new Instance(configuration, services, launcher);
 		await instance.ReportLastStatus();
 		return instance;
 	}
 
 	public InstanceConfiguration Configuration { get; private set; }
+	private InstanceServices Services { get; }
 	private BaseLauncher Launcher { get; set; }
 
 	private readonly string shortName;
 	private readonly ILogger logger;
 
-	private readonly LaunchServices launchServices;
-	private readonly PortManager portManager;
-
 	private IInstanceStatus currentStatus;
 	private IInstanceState currentState;
 	private readonly SemaphoreSlim stateTransitioningActionSemaphore = new (1, 1);
@@ -41,15 +39,14 @@ sealed class Instance : IDisposable {
 	
 	public event EventHandler? IsRunningChanged; 
 
-	private Instance(InstanceConfiguration configuration, BaseLauncher launcher, LaunchServices launchServices, PortManager portManager) {
+	private Instance(InstanceConfiguration configuration, InstanceServices services, BaseLauncher launcher) {
 		this.shortName = GetLoggerName(configuration.InstanceGuid);
 		this.logger = PhantomLogger.Create<Instance>(shortName);
 
 		this.Configuration = configuration;
+		this.Services = services;
 		this.Launcher = launcher;
 		
-		this.launchServices = launchServices;
-		this.portManager = portManager;
 		this.currentState = new InstanceNotRunningState();
 		this.currentStatus = InstanceStatus.NotRunning;
 	}
@@ -139,20 +136,18 @@ sealed class Instance : IDisposable {
 		
 		private int statusUpdateCounter;
 
-		public InstanceContextImpl(Instance instance, CancellationToken shutdownCancellationToken) : base(instance.Configuration, instance.Launcher) {
+		public InstanceContextImpl(Instance instance, CancellationToken shutdownCancellationToken) : base(instance.Configuration, instance.Services, instance.Launcher) {
 			this.instance = instance;
 			this.shutdownCancellationToken = shutdownCancellationToken;
 		}
 
-		public override LaunchServices LaunchServices => instance.launchServices;
-		public override PortManager PortManager => instance.portManager;
 		public override ILogger Logger => instance.logger;
 		public override string ShortName => instance.shortName;
 
 		public override void ReportStatus(IInstanceStatus newStatus) {
 			int myStatusUpdateCounter = Interlocked.Increment(ref statusUpdateCounter);
 			
-			instance.launchServices.TaskManager.Run("Report status of instance " + instance.shortName + " as " + newStatus.GetType().Name, async () => {
+			instance.Services.TaskManager.Run("Report status of instance " + instance.shortName + " as " + newStatus.GetType().Name, async () => {
 				if (myStatusUpdateCounter == statusUpdateCounter) {
 					instance.currentStatus = newStatus;
 					await ServerMessaging.Send(new ReportInstanceStatusMessage(Configuration.InstanceGuid, newStatus));

Agent/Phantom.Agent.Services/Instances/InstanceContext.cs

@@ -7,16 +7,16 @@ namespace Phantom.Agent.Services.Instances;
 
 abstract class InstanceContext {
 	public InstanceConfiguration Configuration { get; }
+	public InstanceServices Services { get; }
 	public BaseLauncher Launcher { get; }
 	
-	public abstract LaunchServices LaunchServices { get; }
-	public abstract PortManager PortManager { get; }
 	public abstract ILogger Logger { get; }
 	public abstract string ShortName { get; }
 
-	protected InstanceContext(InstanceConfiguration configuration, BaseLauncher launcher) {
+	protected InstanceContext(InstanceConfiguration configuration, InstanceServices services, BaseLauncher launcher) {
 		Configuration = configuration;
 		Launcher = launcher;
+		Services = services;
 	}
 
 	public abstract void ReportStatus(IInstanceStatus newStatus);

Agent/Phantom.Agent.Services/Instances/InstanceLogSender.cs

@@ -4,50 +4,33 @@ using Phantom.Common.Logging;
 using Phantom.Common.Messages.ToServer;
 using Phantom.Utils.Collections;
 using Phantom.Utils.Runtime;
-using Serilog;
 
 namespace Phantom.Agent.Services.Instances;
 
-sealed class InstanceLogSender {
+sealed class InstanceLogSender : CancellableBackgroundTask {
 	private static readonly TimeSpan SendDelay = TimeSpan.FromMilliseconds(200);
 
 	private readonly Guid instanceGuid;
-	private readonly ILogger logger;
-	private readonly CancellationTokenSource cancellationTokenSource;
-	private readonly CancellationToken cancellationToken;
 
 	private readonly SemaphoreSlim semaphore = new (1, 1);
 	private readonly RingBuffer<string> buffer = new (1000);
 
-	public InstanceLogSender(TaskManager taskManager, Guid instanceGuid, string name) {
+	public InstanceLogSender(TaskManager taskManager, Guid instanceGuid, string loggerName) : base(PhantomLogger.Create<InstanceLogSender>(loggerName), taskManager, "Instance log sender for " + loggerName) {
 		this.instanceGuid = instanceGuid;
-		this.logger = PhantomLogger.Create<InstanceLogSender>(name);
-		this.cancellationTokenSource = new CancellationTokenSource();
-		this.cancellationToken = cancellationTokenSource.Token;
-		taskManager.Run("Instance log sender for " + name, Run);
 	}
 	
-	private async Task Run() {
+	protected override async Task RunTask() {
-		logger.Verbose("Task started.");
-
 		try {
-			try {
+			while (!CancellationToken.IsCancellationRequested) {
-				while (!cancellationToken.IsCancellationRequested) {
+				await SendOutputToServer(await DequeueOrThrow());
-					await SendOutputToServer(await DequeueOrThrow());
+				await Task.Delay(SendDelay, CancellationToken);
-					await Task.Delay(SendDelay, cancellationToken);
-				}
-			} catch (OperationCanceledException) {
-				// Ignore.
 			}
-
+		} catch (OperationCanceledException) {
-			// Flush remaining lines.
+			// Ignore.
-			await SendOutputToServer(DequeueWithoutSemaphore());
-		} catch (Exception e) {
-			logger.Error(e, "Caught exception in task.");
-		} finally {
-			cancellationTokenSource.Dispose();
-			logger.Verbose("Task stopped.");
 		}
+
+		// Flush remaining lines.
+		await SendOutputToServer(DequeueWithoutSemaphore());
 	}
 
 	private async Task SendOutputToServer(ImmutableArray<string> lines) {
@@ -63,7 +46,7 @@ sealed class InstanceLogSender {
 	}
 
 	private async Task<ImmutableArray<string>> DequeueOrThrow() {
-		await semaphore.WaitAsync(cancellationToken);
+		await semaphore.WaitAsync(CancellationToken);
 
 		try {
 			return DequeueWithoutSemaphore();
@@ -74,7 +57,7 @@ sealed class InstanceLogSender {
 
 	public void Enqueue(string line) {
 		try {
-			semaphore.Wait(cancellationToken);
+			semaphore.Wait(CancellationToken);
 		} catch (Exception) {
 			return;
 		}
@@ -85,12 +68,4 @@ sealed class InstanceLogSender {
 			semaphore.Release();
 		}
 	}
-
-	public void Cancel() {
-		try {
-			cancellationTokenSource.Cancel();
-		} catch (ObjectDisposedException) {
-			// Ignore.
-		}
-	}
 }

Agent/Phantom.Agent.Services/Instances/InstanceServices.cs

@@ -0,0 +1,7 @@
+using Phantom.Agent.Minecraft.Launcher;
+using Phantom.Agent.Services.Backups;
+using Phantom.Utils.Runtime;
+
+namespace Phantom.Agent.Services.Instances; 
+
+sealed record InstanceServices(TaskManager TaskManager, PortManager PortManager, BackupManager BackupManager, LaunchServices LaunchServices);

Agent/Phantom.Agent.Services/Instances/InstanceSessionManager.cs

@@ -7,6 +7,7 @@ using Phantom.Agent.Minecraft.Launcher.Types;
 using Phantom.Agent.Minecraft.Properties;
 using Phantom.Agent.Minecraft.Server;
 using Phantom.Agent.Rpc;
+using Phantom.Agent.Services.Backups;
 using Phantom.Common.Data;
 using Phantom.Common.Data.Agent;
 using Phantom.Common.Data.Instance;
@@ -27,21 +28,22 @@ sealed class InstanceSessionManager : IDisposable {
 	private readonly string basePath;
 
 	private readonly MinecraftServerExecutables minecraftServerExecutables;
-	private readonly LaunchServices launchServices;
+	private readonly InstanceServices instanceServices;
-	private readonly PortManager portManager;
 	private readonly Dictionary<Guid, Instance> instances = new ();
 
 	private readonly CancellationTokenSource shutdownCancellationTokenSource = new ();
 	private readonly CancellationToken shutdownCancellationToken;
 	private readonly SemaphoreSlim semaphore = new (1, 1);
 
-	public InstanceSessionManager(AgentInfo agentInfo, AgentFolders agentFolders, JavaRuntimeRepository javaRuntimeRepository, TaskManager taskManager) {
+	public InstanceSessionManager(AgentInfo agentInfo, AgentFolders agentFolders, JavaRuntimeRepository javaRuntimeRepository, TaskManager taskManager, BackupManager backupManager) {
 		this.agentInfo = agentInfo;
 		this.basePath = agentFolders.InstancesFolderPath;
 		this.minecraftServerExecutables = new MinecraftServerExecutables(agentFolders.ServerExecutableFolderPath);
-		this.launchServices = new LaunchServices(taskManager, minecraftServerExecutables, javaRuntimeRepository);
-		this.portManager = new PortManager(agentInfo.AllowedServerPorts, agentInfo.AllowedRconPorts);
 		this.shutdownCancellationToken = shutdownCancellationTokenSource.Token;
+		
+		var launchServices = new LaunchServices(minecraftServerExecutables, javaRuntimeRepository);
+		var portManager = new PortManager(agentInfo.AllowedServerPorts, agentInfo.AllowedRconPorts);
+		this.instanceServices = new InstanceServices(taskManager, portManager, backupManager, launchServices);
 	}
 
 	private async Task<InstanceActionResult<T>> AcquireSemaphoreAndRun<T>(Func<Task<InstanceActionResult<T>>> func) {
@@ -98,7 +100,7 @@ sealed class InstanceSessionManager : IDisposable {
 				Logger.Information("Reconfigured instance \"{Name}\" (GUID {Guid}).", configuration.InstanceName, configuration.InstanceGuid);
 			}
 			else {
-				instances[instanceGuid] = instance = await Instance.Create(configuration, launcher, launchServices, portManager);
+				instances[instanceGuid] = instance = await Instance.Create(configuration, instanceServices, launcher);
 				instance.IsRunningChanged += OnInstanceIsRunningChanged;
 				Logger.Information("Created instance \"{Name}\" (GUID {Guid}).", configuration.InstanceName, configuration.InstanceGuid);
 			}
@@ -116,7 +118,7 @@ sealed class InstanceSessionManager : IDisposable {
 	}
 
 	private void OnInstanceIsRunningChanged(object? sender, EventArgs e) {
-		launchServices.TaskManager.Run("Handle instance running state changed event", RefreshAgentStatus);
+		instanceServices.TaskManager.Run("Handle instance running state changed event", RefreshAgentStatus);
 	}
 
 	public async Task RefreshAgentStatus() {

Agent/Phantom.Agent.Services/Instances/States/InstanceLaunchingState.cs

@@ -19,7 +19,7 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
 	public void Initialize() {
 		context.Logger.Information("Session starting...");
 
-		var launchTask = context.LaunchServices.TaskManager.Run("Launch procedure for instance " + context.ShortName, DoLaunch);
+		var launchTask = context.Services.TaskManager.Run("Launch procedure for instance " + context.ShortName, DoLaunch);
 		launchTask.ContinueWith(OnLaunchSuccess, CancellationToken.None, TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Default);
 		launchTask.ContinueWith(OnLaunchFailure, CancellationToken.None, TaskContinuationOptions.NotOnRanToCompletion, TaskScheduler.Default);
 	}
@@ -37,7 +37,7 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
 			}
 		}
 
-		var launchResult = await context.Launcher.Launch(context.Logger, context.LaunchServices, OnDownloadProgress, cancellationToken);
+		var launchResult = await context.Launcher.Launch(context.Logger, context.Services.LaunchServices, OnDownloadProgress, cancellationToken);
 		if (launchResult is LaunchResult.InvalidJavaRuntime) {
 			throw new LaunchFailureException(InstanceLaunchFailReason.JavaRuntimeNotFound, "Session failed to launch, invalid Java runtime.");
 		}
@@ -65,7 +65,7 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
 	private void OnLaunchSuccess(Task<InstanceSession> task) {
 		context.TransitionState(() => {
 			if (cancellationTokenSource.IsCancellationRequested) {
-				context.PortManager.Release(context.Configuration);
+				context.Services.PortManager.Release(context.Configuration);
 				return (new InstanceNotRunningState(), InstanceStatus.NotRunning);
 			}
 			else {
@@ -83,7 +83,7 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
 			context.ReportStatus(InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError));
 		}
 
-		context.PortManager.Release(context.Configuration);
+		context.Services.PortManager.Release(context.Configuration);
 		context.TransitionState(new InstanceNotRunningState());
 	}
 

Agent/Phantom.Agent.Services/Instances/States/InstanceNotRunningState.cs

@@ -8,7 +8,7 @@ sealed class InstanceNotRunningState : IInstanceState {
 	public void Initialize() {}
 
 	public (IInstanceState, LaunchInstanceResult) Launch(InstanceContext context) {
-		InstanceLaunchFailReason? failReason = context.PortManager.Reserve(context.Configuration) switch {
+		InstanceLaunchFailReason? failReason = context.Services.PortManager.Reserve(context.Configuration) switch {
 			PortManager.Result.ServerPortNotAllowed   => InstanceLaunchFailReason.ServerPortNotAllowed,
 			PortManager.Result.ServerPortAlreadyInUse => InstanceLaunchFailReason.ServerPortAlreadyInUse,
 			PortManager.Result.RconPortNotAllowed     => InstanceLaunchFailReason.RconPortNotAllowed,

Agent/Phantom.Agent.Services/Instances/States/InstanceRunningState.cs

@@ -1,5 +1,6 @@
 using Phantom.Agent.Minecraft.Command;
 using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Services.Backups;
 using Phantom.Common.Data.Instance;
 using Phantom.Common.Data.Minecraft;
 using Phantom.Common.Data.Replies;
@@ -10,6 +11,7 @@ sealed class InstanceRunningState : IInstanceState {
 	private readonly InstanceContext context;
 	private readonly InstanceSession session;
 	private readonly InstanceLogSender logSender;
+	private readonly BackupScheduler backupScheduler;
 	private readonly SessionObjects sessionObjects;
 	
 	private readonly CancellationTokenSource delayedStopCancellationTokenSource = new ();
@@ -19,7 +21,8 @@ sealed class InstanceRunningState : IInstanceState {
 	public InstanceRunningState(InstanceContext context, InstanceSession session) {
 		this.context = context;
 		this.session = session;
-		this.logSender = new InstanceLogSender(context.LaunchServices.TaskManager, context.Configuration.InstanceGuid, context.ShortName);
+		this.logSender = new InstanceLogSender(context.Services.TaskManager, context.Configuration.InstanceGuid, context.ShortName);
+		this.backupScheduler = new BackupScheduler(context.Services.TaskManager, context.Services.BackupManager, session, context.Configuration.ServerPort, context.ShortName);
 		this.sessionObjects = new SessionObjects(this);
 	}
 
@@ -30,7 +33,7 @@ sealed class InstanceRunningState : IInstanceState {
 		if (session.HasEnded) {
 			if (sessionObjects.Dispose()) {
 				context.Logger.Warning("Session ended immediately after it was started.");
-				context.LaunchServices.TaskManager.Run("Transition state of instance " + context.ShortName + " to not running", () => context.TransitionState(new InstanceNotRunningState(), InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)));
+				context.Services.TaskManager.Run("Transition state of instance " + context.ShortName + " to not running", () => context.TransitionState(new InstanceNotRunningState(), InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)));
 			}
 		}
 		else {
@@ -39,9 +42,9 @@ sealed class InstanceRunningState : IInstanceState {
 		}
 	}
 
-	private void SessionOutput(object? sender, string e) {
+	private void SessionOutput(object? sender, string line) {
-		context.Logger.Verbose("[Server] {Line}", e);
+		context.Logger.Verbose("[Server] {Line}", line);
-		logSender.Enqueue(e);
+		logSender.Enqueue(line);
 	}
 
 	private void SessionEnded(object? sender, EventArgs e) {
@@ -75,12 +78,13 @@ sealed class InstanceRunningState : IInstanceState {
 		}
 		
 		isStopping = true;
-		context.LaunchServices.TaskManager.Run("Delayed stop timer for instance " + context.ShortName, () => StopLater(stopStrategy.Seconds));
+		context.Services.TaskManager.Run("Delayed stop timer for instance " + context.ShortName, () => StopLater(stopStrategy.Seconds));
 		return (this, StopInstanceResult.StopInitiated);
 	}
 
 	private IInstanceState PrepareStoppedState() {
 		session.SessionEnded -= SessionEnded;
+		backupScheduler.Stop();
 		return new InstanceStoppingState(context, session, sessionObjects);
 	}
 
@@ -159,9 +163,9 @@ sealed class InstanceRunningState : IInstanceState {
 				state.CancelDelayedStop();
 			}
 			
-			state.logSender.Cancel();
+			state.logSender.Stop();
 			state.session.Dispose();
-			state.context.PortManager.Release(state.context.Configuration);
+			state.context.Services.PortManager.Release(state.context.Configuration);
 			return true;
 		}
 	}

Agent/Phantom.Agent.Services/Instances/States/InstanceStoppingState.cs

@@ -21,11 +21,17 @@ sealed class InstanceStoppingState : IInstanceState, IDisposable {
 	public void Initialize() {
 		context.Logger.Information("Session stopping.");
 		context.ReportStatus(InstanceStatus.Stopping);
-		context.LaunchServices.TaskManager.Run("Stop procedure for instance " + context.ShortName, DoStop);
+		context.Services.TaskManager.Run("Stop procedure for instance " + context.ShortName, DoStop);
 	}
 
 	private async Task DoStop() {
 		try {
+			// Do not release the semaphore after this point.
+			if (!await session.BackupSemaphore.CancelAndWait(TimeSpan.FromSeconds(1))) {
+				context.Logger.Information("Waiting for backup to finish...");
+				await session.BackupSemaphore.CancelAndWait(Timeout.InfiniteTimeSpan);
+			}
+			
 			context.Logger.Information("Sending stop command...");
 			await DoSendStopCommand();
 

Common/Phantom.Common.Data/Backups/BackupCreationResultKind.cs

@@ -3,23 +3,35 @@
 public enum BackupCreationResultKind : byte {
 	UnknownError,
 	Success,
+	InstanceNotRunning,
 	BackupCancelled,
-	BackupAlreadyExists,
+	BackupAlreadyRunning,
+	BackupFileAlreadyExists,
 	CouldNotCreateBackupFolder,
 	CouldNotCopyWorldToTemporaryFolder,
 	CouldNotCreateWorldArchive
 }
 
 public static class BackupCreationResultSummaryExtensions {
+	public static bool ShouldRetry(this BackupCreationResultKind kind) {
+		return kind != BackupCreationResultKind.Success &&
+		       kind != BackupCreationResultKind.InstanceNotRunning &&
+		       kind != BackupCreationResultKind.BackupCancelled &&
+		       kind != BackupCreationResultKind.BackupAlreadyRunning &&
+		       kind != BackupCreationResultKind.BackupFileAlreadyExists;
+	}
+
 	public static string ToSentence(this BackupCreationResultKind kind) {
 		return kind switch {
 			BackupCreationResultKind.Success                            => "Backup created successfully.",
+			BackupCreationResultKind.InstanceNotRunning                 => "Instance is not running.",
 			BackupCreationResultKind.BackupCancelled                    => "Backup cancelled.",
-			BackupCreationResultKind.BackupAlreadyExists                => "Backup with the same name already exists.",
+			BackupCreationResultKind.BackupAlreadyRunning               => "A backup is already being created.",
+			BackupCreationResultKind.BackupFileAlreadyExists            => "Backup with the same name already exists.",
 			BackupCreationResultKind.CouldNotCreateBackupFolder         => "Could not create backup folder.",
 			BackupCreationResultKind.CouldNotCopyWorldToTemporaryFolder => "Could not copy world to temporary folder.",
 			BackupCreationResultKind.CouldNotCreateWorldArchive         => "Could not create world archive.",
-			_                                                              => "Unknown error."
+			_                                                           => "Unknown error."
 		};
 	}
 }

Common/Phantom.Common.Data/Backups/BackupCreationWarnings.cs

@@ -1,4 +1,6 @@
-namespace Phantom.Common.Data.Backups; 
+using System.Numerics;
+
+namespace Phantom.Common.Data.Backups; 
 
 [Flags]
 public enum BackupCreationWarnings : byte {
@@ -7,3 +9,9 @@ public enum BackupCreationWarnings : byte {
 	CouldNotCompressWorldArchive = 1 << 1,
 	CouldNotRestoreAutomaticSaving = 1 << 2
 }
+
+public static class BackupCreationWarningsExtensions {
+	public static int Count(this BackupCreationWarnings warnings) {
+		return BitOperations.PopCount((byte) warnings);
+	}
+}

Dockerfile

@@ -59,7 +59,8 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
     rm -f /etc/apt/apt.conf.d/docker-clean                   && \
     apt-get update                                           && \
     apt-get install -y                                          \
-    openjdk-18-jre-headless
+    openjdk-18-jre-headless                                     \
+    zstd
 
 RUN mkdir /data && chmod 777 /data
 WORKDIR /data

Utils/Phantom.Utils.Runtime/CancellableBackgroundTask.cs

@@ -0,0 +1,41 @@
+using Serilog;
+
+namespace Phantom.Utils.Runtime; 
+
+public abstract class CancellableBackgroundTask {
+	private readonly CancellationTokenSource cancellationTokenSource = new ();
+
+	protected ILogger Logger { get; }
+	protected CancellationToken CancellationToken { get; }
+
+	protected CancellableBackgroundTask(ILogger logger, TaskManager taskManager, string taskName) {
+		this.Logger = logger;
+		this.CancellationToken = cancellationTokenSource.Token;
+		taskManager.Run(taskName, Run);
+	}
+
+	private async Task Run() {
+		Logger.Verbose("Task started.");
+
+		try {
+			await RunTask();
+		} catch (OperationCanceledException) {
+			// Ignore.
+		} catch (Exception e) {
+			Logger.Fatal(e, "Caught exception in task.");
+		} finally {
+			cancellationTokenSource.Dispose();
+			Logger.Verbose("Task stopped.");
+		}
+	}
+	
+	protected abstract Task RunTask();
+
+	public void Stop() {
+		try {
+			cancellationTokenSource.Cancel();
+		} catch (ObjectDisposedException) {
+			// Ignore.
+		}
+	}
+}

Utils/Phantom.Utils.Runtime/CancellableSemaphore.cs

@@ -0,0 +1,28 @@
+namespace Phantom.Utils.Runtime; 
+
+public sealed class CancellableSemaphore : IDisposable {
+	private readonly SemaphoreSlim semaphore;
+	private readonly CancellationTokenSource cancellationTokenSource = new ();
+	
+	public CancellableSemaphore(int initialCount, int maxCount) {
+		this.semaphore = new SemaphoreSlim(initialCount, maxCount);
+	}
+
+	public async Task<bool> Wait(TimeSpan timeout, CancellationToken cancellationToken) {
+		return await semaphore.WaitAsync(timeout, cancellationTokenSource.Token).WaitAsync(cancellationToken);
+	}
+
+	public async Task<bool> CancelAndWait(TimeSpan timeout) {
+		cancellationTokenSource.Cancel();
+		return await semaphore.WaitAsync(timeout);
+	}
+
+	public void Release() {
+		semaphore.Release();
+	}
+	
+	public void Dispose() {
+		semaphore.Dispose();
+		cancellationTokenSource.Dispose();
+	}
+}