Fix instance stop timeout not killing the process

Caused by 72ddaf91.

Agent/Phantom.Agent.Minecraft/Properties/MinecraftServerProperties.cs

@@ -18,4 +18,5 @@ static class MinecraftServerProperties {
 	public static readonly MinecraftServerProperty<ushort> ServerPort = new UnsignedShort("server-port");
 	public static readonly MinecraftServerProperty<ushort> RconPort = new UnsignedShort("rcon.port");
 	public static readonly MinecraftServerProperty<bool> EnableRcon = new Boolean("enable-rcon");
+	public static readonly MinecraftServerProperty<bool> SyncChunkWrites = new Boolean("sync-chunk-writes");
 }

Agent/Phantom.Agent.Minecraft/Properties/ServerProperties.cs

@@ -5,11 +5,13 @@ namespace Phantom.Agent.Minecraft.Properties;
 public sealed record ServerProperties(
 	ushort ServerPort,
 	ushort RconPort,
-	bool EnableRcon = true
+	bool EnableRcon = true,
+	bool SyncChunkWrites = false
 ) {
 	internal void SetTo(JavaPropertiesFileEditor properties) {
 		MinecraftServerProperties.ServerPort.Set(properties, ServerPort);
 		MinecraftServerProperties.RconPort.Set(properties, RconPort);
 		MinecraftServerProperties.EnableRcon.Set(properties, EnableRcon);
+		MinecraftServerProperties.SyncChunkWrites.Set(properties, SyncChunkWrites);
 	}
 }

Agent/Phantom.Agent.Minecraft/Server/ServerStatusProtocol.cs

@@ -19,7 +19,7 @@ public sealed class ServerStatusProtocol {
 		try {
 			return await GetOnlinePlayerCountOrThrow(serverPort, cancellationToken);
 		} catch (Exception e) {
-			logger.Error(e, "Caught exception while checking if players are online.");
+			logger.Error(e, "Caught exception checking online player count.");
 			return null;
 		}
 	}

Agent/Phantom.Agent.Services/Backups/BackupScheduler.cs

@@ -1,10 +1,8 @@
-using Phantom.Agent.Minecraft.Instance;
-using Phantom.Agent.Minecraft.Server;
-using Phantom.Agent.Services.Instances;
+using Phantom.Agent.Services.Instances;
+using Phantom.Agent.Services.Instances.State;
 using Phantom.Common.Data.Backups;
 using Phantom.Utils.Logging;
 using Phantom.Utils.Tasks;
-using Phantom.Utils.Threading;
 
 namespace Phantom.Agent.Services.Backups;
 
@@ -16,20 +14,16 @@ sealed class BackupScheduler : CancellableBackgroundTask {
 
 	private readonly BackupManager backupManager;
 	private readonly InstanceContext context;
-	private readonly InstanceProcess process;
 	private readonly SemaphoreSlim backupSemaphore = new (1, 1);
-	private readonly int serverPort;
-	private readonly ServerStatusProtocol serverStatusProtocol;
 	private readonly ManualResetEventSlim serverOutputWhileWaitingForOnlinePlayers = new ();
+	private readonly InstancePlayerCountTracker playerCountTracker;
 	
 	public event EventHandler<BackupCreationResult>? BackupCompleted;
 
-	public BackupScheduler(InstanceContext context, InstanceProcess process, int serverPort) : base(PhantomLogger.Create<BackupScheduler>(context.ShortName)) {
+	public BackupScheduler(InstanceContext context, InstancePlayerCountTracker playerCountTracker) : base(PhantomLogger.Create<BackupScheduler>(context.ShortName)) {
 		this.backupManager = context.Services.BackupManager;
 		this.context = context;
-		this.process = process;
-		this.serverPort = serverPort;
-		this.serverStatusProtocol = new ServerStatusProtocol(context.ShortName);
+		this.playerCountTracker = playerCountTracker;
 		Start();
 	}
 
@@ -69,43 +63,18 @@ sealed class BackupScheduler : CancellableBackgroundTask {
 	}
 
 	private async Task WaitForOnlinePlayers() {
-		bool needsToLogOfflinePlayersMessage = true;
-		
-		process.AddOutputListener(ServerOutputListener, maxLinesToReadFromHistory: 0);
-		try {
-			while (!CancellationToken.IsCancellationRequested) {
-				serverOutputWhileWaitingForOnlinePlayers.Reset();
-				
-				var onlinePlayerCount = await serverStatusProtocol.GetOnlinePlayerCount(serverPort, CancellationToken);
-				if (onlinePlayerCount == null) {
-					Logger.Warning("Could not detect whether any players are online, starting a new backup.");
-					break;
-				}
-
-				if (onlinePlayerCount > 0) {
-					Logger.Information("Players are online, starting a new backup.");
-					break;
-				}
-
-				if (needsToLogOfflinePlayersMessage) {
-					needsToLogOfflinePlayersMessage = false;
-					Logger.Information("No players are online, waiting for someone to join before starting a new backup.");
-				}
-
-				await Task.Delay(TimeSpan.FromSeconds(10), CancellationToken);
-				
-				Logger.Debug("Waiting for server output before checking for online players again...");
-				await serverOutputWhileWaitingForOnlinePlayers.WaitHandle.WaitOneAsync(CancellationToken);
-			}
-		} finally {
-			process.RemoveOutputListener(ServerOutputListener);
+		var task = playerCountTracker.WaitForOnlinePlayers(CancellationToken);
+		if (!task.IsCompleted) {
+			Logger.Information("Waiting for someone to join before starting a new backup.");
 		}
-	}
 		
-	private void ServerOutputListener(object? sender, string line) {
-		if (!serverOutputWhileWaitingForOnlinePlayers.IsSet) {
-			serverOutputWhileWaitingForOnlinePlayers.Set();
-			Logger.Debug("Detected server output, signalling to check for online players again.");
+		try {
+			await task;
+			Logger.Information("Players are online, starting a new backup.");
+		} catch (OperationCanceledException) {
+			throw;
+		} catch (Exception) {
+			Logger.Warning("Could not detect whether any players are online, starting a new backup.");
 		}
 	}
 

Agent/Phantom.Agent.Services/Backups/BackupServerCommandDispatcher.cs

@@ -55,7 +55,6 @@ sealed partial class BackupServerCommandDispatcher : IDisposable {
 	}
 
 	public async Task SaveAllChunks() {
-		// TODO Try if not flushing and waiting a few seconds before flushing reduces lag.
 		await process.SendCommand(MinecraftCommand.SaveAll(flush: true), cancellationToken);
 		await savedTheGame.Task.WaitAsync(TimeSpan.FromMinutes(1), cancellationToken);
 	}

Agent/Phantom.Agent.Services/Instances/State/InstancePlayerCountTracker.cs

@@ -0,0 +1,121 @@
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Agent.Minecraft.Server;
+using Phantom.Utils.Logging;
+using Phantom.Utils.Tasks;
+using Phantom.Utils.Threading;
+
+namespace Phantom.Agent.Services.Instances.State;
+
+sealed class InstancePlayerCountTracker : CancellableBackgroundTask {
+	private readonly InstanceProcess process;
+	private readonly ushort serverPort;
+
+	private readonly ServerStatusProtocol serverStatusProtocol;
+
+	private readonly TaskCompletionSource firstDetection = AsyncTasks.CreateCompletionSource();
+	private readonly ManualResetEventSlim serverOutputEvent = new ();
+
+	private int? onlinePlayerCount;
+
+	public int? OnlinePlayerCount {
+		get {
+			lock (this) {
+				return onlinePlayerCount;
+			}
+		}
+		private set {
+			EventHandler<int?>? onlinePlayerCountChanged;
+			lock (this) {
+				if (onlinePlayerCount == value) {
+					return;
+				}
+				
+				onlinePlayerCount = value;
+				onlinePlayerCountChanged = OnlinePlayerCountChanged;
+			}
+
+			onlinePlayerCountChanged?.Invoke(this, value);
+		}
+	}
+
+	private event EventHandler<int?>? OnlinePlayerCountChanged;
+
+	private bool isDisposed = false;
+
+	public InstancePlayerCountTracker(InstanceContext context, InstanceProcess process, ushort serverPort) : base(PhantomLogger.Create<InstancePlayerCountTracker>(context.ShortName)) {
+		this.process = process;
+		this.serverPort = serverPort;
+		this.serverStatusProtocol = new ServerStatusProtocol(context.ShortName);
+		Start();
+	}
+
+	protected override async Task RunTask() {
+		// Give the server time to start accepting connections.
+		await Task.Delay(TimeSpan.FromSeconds(10), CancellationToken);
+
+		serverOutputEvent.Set();
+		process.AddOutputListener(OnOutput, maxLinesToReadFromHistory: 0);
+		
+		while (!CancellationToken.IsCancellationRequested) {
+			serverOutputEvent.Reset();
+
+			OnlinePlayerCount = await serverStatusProtocol.GetOnlinePlayerCount(serverPort, CancellationToken);
+			
+			if (!firstDetection.Task.IsCompleted) {
+				firstDetection.SetResult();
+			}
+
+			await Task.Delay(TimeSpan.FromSeconds(10), CancellationToken);
+			await serverOutputEvent.WaitHandle.WaitOneAsync(CancellationToken);
+			await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken);
+		}
+	}
+
+	public async Task WaitForOnlinePlayers(CancellationToken cancellationToken) {
+		await firstDetection.Task.WaitAsync(cancellationToken);
+
+		var onlinePlayersDetected = AsyncTasks.CreateCompletionSource();
+
+		lock (this) {
+			if (onlinePlayerCount == null) {
+				throw new InvalidOperationException();
+			}
+			else if (onlinePlayerCount > 0) {
+				return;
+			}
+
+			OnlinePlayerCountChanged += OnOnlinePlayerCountChanged;
+
+			void OnOnlinePlayerCountChanged(object? sender, int? newPlayerCount) {
+				if (newPlayerCount == null) {
+					onlinePlayersDetected.TrySetException(new InvalidOperationException());
+					OnlinePlayerCountChanged -= OnOnlinePlayerCountChanged;
+				}
+				else if (newPlayerCount > 0) {
+					onlinePlayersDetected.TrySetResult();
+					OnlinePlayerCountChanged -= OnOnlinePlayerCountChanged;
+				}
+			}
+		}
+
+		await onlinePlayersDetected.Task;
+	}
+
+	private void OnOutput(object? sender, string? line) {
+		lock (this) {
+			if (!isDisposed) {
+				serverOutputEvent.Set();
+			}
+		}
+	}
+
+	protected override void Dispose() {
+		lock (this) {
+			isDisposed = true;
+			onlinePlayerCount = null;
+		}
+
+		process.RemoveOutputListener(OnOutput);
+		serverOutputEvent.Dispose();
+	}
+}

Agent/Phantom.Agent.Services/Instances/State/InstanceRunningState.cs

@@ -19,6 +19,7 @@ sealed class InstanceRunningState : IDisposable {
 	private readonly CancellationToken cancellationToken;
 
 	private readonly InstanceLogSender logSender;
+	private readonly InstancePlayerCountTracker playerCountTracker;
 	private readonly BackupScheduler backupScheduler;
 
 	private bool isDisposed;
@@ -32,8 +33,9 @@ sealed class InstanceRunningState : IDisposable {
 		this.cancellationToken = cancellationToken;
 
 		this.logSender = new InstanceLogSender(context.Services.ControllerConnection, context.InstanceGuid, context.ShortName);
+		this.playerCountTracker = new InstancePlayerCountTracker(context, process, configuration.ServerPort);
 
-		this.backupScheduler = new BackupScheduler(context, process, configuration.ServerPort);
+		this.backupScheduler = new BackupScheduler(context, playerCountTracker);
 		this.backupScheduler.BackupCompleted += OnScheduledBackupCompleted;
 	}
 
@@ -93,6 +95,11 @@ sealed class InstanceRunningState : IDisposable {
 		}
 	}
 
+	public void OnStopInitiated() {
+		backupScheduler.Stop();
+		playerCountTracker.Stop();
+	}
+	
 	private bool TryDispose() {
 		lock (this) {
 			if (isDisposed) {
@@ -102,8 +109,8 @@ sealed class InstanceRunningState : IDisposable {
 			isDisposed = true;
 		}
 
+		OnStopInitiated();
 		logSender.Stop();
-		backupScheduler.Stop();
 		
 		Process.Dispose();
 		

Agent/Phantom.Agent.Services/Instances/State/InstanceStopProcedure.cs

@@ -25,6 +25,8 @@ static class InstanceStopProcedure {
 
 		try {
 			// Too late to cancel the stop procedure now.
+			runningState.OnStopInitiated();
+			
 			if (!process.HasEnded) {
 				context.Logger.Information("Session stopping now.");
 				await DoStop(context, process);
@@ -85,7 +87,7 @@ static class InstanceStopProcedure {
 	private static async Task WaitForSessionToEnd(InstanceContext context, InstanceProcess process) {
 		try {
 			await process.WaitForExit(TimeSpan.FromSeconds(55));
-		} catch (OperationCanceledException) {
+		} catch (TimeoutException) {
 			try {
 				context.Logger.Warning("Waiting timed out, killing session...");
 				process.Kill();