From 5ce4eefb87657f5bd63c2d95f10dbdc870da4155 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:34:48 +0200 Subject: [PATCH 1/5] feat(lobby): add mesh retries and gate QuickMatch startup Retry incomplete full-mesh checks with correlated attempts and gate QuickMatch startup on a successful mesh result. Preserve legacy client compatibility and requeue players when automatic setup fails. --- GenOnlineService/Constants.cs | 25 +- .../WebSocket/WebSocketController.cs | 22 +- GenOnlineService/LobbyManager.cs | 229 ++++++++++++++- GenOnlineService/MatchmakingManager.cs | 269 ++++++++++++++---- 4 files changed, 455 insertions(+), 90 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index 55aac44..e04f7f6 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -2843,10 +2843,12 @@ public enum EWebSocketMessageID SOCIAL_FRIENDS_LIST_DIRTY = 37, SOCIAL_CANT_ADD_FRIEND_LIST_FULL = 38, PROBE_RESP = 39, - AC_REGISTER_PLAYER = 40, - AC_DEREGISTER_PLAYER = 41, - WS_KEEPALIVE = 42, + AC_REGISTER_PLAYER = 40, + AC_DEREGISTER_PLAYER = 41, + WS_KEEPALIVE = 42, WS_KEEPALIVE_CLIENT = 43, + MATCHMAKING_ACTION_REQUEUE = 44, + MATCHMAKING_ACTION_SETUP_PROGRESS = 45, MODERATION_NOTICE = 46, MODERATION_COMMAND = 47, MODERATION_COMMAND_RESULT = 48 @@ -2985,8 +2987,16 @@ public class WebSocketMessage_NameChange : WebSocketMessage public string name { get; set; } = String.Empty; } - public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage - { + public class WebSocketMessage_FullMeshConnectivityCheckRequest : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } + } + + public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } public List connectivity_map { get; set; } = new(); } @@ -3176,4 +3186,9 @@ public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage } + public class WebSocketMessage_MatchmakerSetupProgress : WebSocketMessage + { + public int timeout_ms { get; set; } + } + } diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index 8d93efb..2997cbe 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -1020,25 +1020,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession lobbyInfo.StartFullMeshConnectivityCheck(); // start full mesh connectivity checks - WebSocketMessage_Simple startCommand = new WebSocketMessage_Simple(); - startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; - - // Serialize once before broadcasting - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember lobbyMember in lobbyInfo.Members) - { - if (lobbyMember != null) - { - if (lobbyMember.GetSession().TryGetTarget(out UserSession? sess)) - { - if (sess != null) - { - sess.QueueWebsocketSend(bytesJSON); - } - } - } - } + lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); } else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) { @@ -1052,7 +1034,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession Lobby? lobby = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); if (lobby != null) { - await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg.connectivity_map); + await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg); } } } diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index bc0df6d..8e2c5d7 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -56,9 +56,33 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - private const int MSToWaitForFullMeshChecks = 5000; // really shouldnt take more than 5 seconds... this might even be too much + private const int MSToWaitForFullMeshChecks = 5000; + private const int MaxFullMeshCheckAttempts = 2; + private const int MSBeforeFullMeshCheckRetry = 3000; + public const int MaxFullMeshConnectivityCheckDurationMS = + (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) + + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); + + private readonly object m_FullMeshCheckLock = new(); + + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; [JsonIgnore] + public int FullMeshCheckAttempt { get; private set; } = 0; + + [JsonIgnore] + public Int64 FullMeshCheckID { get; private set; } = 0; + + [JsonIgnore] + private Int64 m_TimeToRetryFullMeshChecks = -1; + + [JsonIgnore] + private bool m_bCurrentAttemptHasLegacyResponse = false; + + private static Int64 s_NextFullMeshCheckID = 0; + + [JsonIgnore] public ConcurrentDictionary> FullMeshConnectivityChecks { get; set; } = new(); @@ -142,24 +166,143 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) // End AC Probes public void StartFullMeshConnectivityCheck() + { + lock (m_FullMeshCheckLock) + { + FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); + FullMeshCheckAttempt = 1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + BeginFullMeshConnectivityCheckAttempt(); + } + } + + private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; - FullMeshConnectivityChecks = new(); + FullMeshConnectivityChecks = new(); + m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; } - public async Task StoreFullMeshConnectivityResponse(Int64 sourceUser, List connectivityMap) + public void SendFullMeshConnectivityCheckRequestToMembers() + { + WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); + startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; + startCommand.mesh_check_id = FullMeshCheckID; + startCommand.attempt = FullMeshCheckAttempt; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player + // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. + private void RestartSignallingForMissingConnections(List lstMissingConnections) { - FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(connectivityMap); + HashSet<(Int64, Int64)> alreadyResignalled = new(); - // check again for being done - await ProcessPendingFullMeshConnectivityChecks(); + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } } - public async Task ProcessPendingFullMeshConnectivityChecks() + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) { - // TODO: Add a timeout to this - if (PendingFullMeshConnectivityChecks) + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } + } + + public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + lock (m_FullMeshCheckLock) + { + bool bLegacyResponse = response.mesh_check_id == 0 && response.attempt == 0; + bool bMatchesCurrentAttempt = bLegacyResponse + ? FullMeshCheckAttempt == 1 + : response.mesh_check_id == FullMeshCheckID && response.attempt == FullMeshCheckAttempt; + + LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); + if (PendingFullMeshConnectivityChecks + && m_TimeToRetryFullMeshChecks == -1 + && sourceMember?.IsHuman() == true + && bMatchesCurrentAttempt) + { + m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; + FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + } + + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + public Task ProcessPendingFullMeshConnectivityChecks() + { + lock (m_FullMeshCheckLock) + { + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + + // Give re-signalled connections time to establish before starting the retry. + if (m_TimeToRetryFullMeshChecks != -1) + { + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) + { + return; + } + + m_TimeToRetryFullMeshChecks = -1; + BeginFullMeshConnectivityCheckAttempt(); + SendFullMeshConnectivityCheckRequestToMembers(); + return; + } + { bool bDoneChecks = false; int totalMapEntriesExpected = GetNumberOfHumans(); @@ -209,6 +352,24 @@ public async Task ProcessPendingFullMeshConnectivityChecks() } } + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -222,6 +383,18 @@ public async Task ProcessPendingFullMeshConnectivityChecks() + bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + + bool bAllMembersReportedCurrentAttempt = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; + bool bCanSafelyRetry = bAllMembersReportedCurrentAttempt && !m_bCurrentAttemptHasLegacyResponse; + if (!bMeshComplete && bCanSafelyRetry && FullMeshCheckAttempt < MaxFullMeshCheckAttempts) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(lstMissingConnections); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + return; + } + // inform host that we are done // start full mesh connectivity checks WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -238,6 +411,8 @@ public async Task ProcessPendingFullMeshConnectivityChecks() outcome.missing_connections = lstMissingConnections; } + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID // send to host @@ -251,6 +426,7 @@ public async Task ProcessPendingFullMeshConnectivityChecks() // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } } } @@ -664,6 +840,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -944,6 +1122,37 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa } } + public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) + { + if (!departingMember.GetSession().TryGetTarget(out UserSession? departingSession) || departingSession == null) + { + return; + } + + foreach (LobbyMember remoteMember in Members) + { + if (remoteMember.SlotState != EPlayerType.SLOT_PLAYER || remoteMember.UserID == departingMember.UserID) + { + continue; + } + + WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); + remotePlayerAcMsg.msg_id = (int)EWebSocketMessageID.AC_DEREGISTER_PLAYER; + remotePlayerAcMsg.user_id = remoteMember.UserID; + remotePlayerAcMsg.mwid = remoteMember.MiddlewareUserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); + + if (State != ELobbyState.INGAME) + { + WebSocketMessage_NetworkDisconnectPlayer remotePlayerMsg = new WebSocketMessage_NetworkDisconnectPlayer(); + remotePlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_DISCONNECT_PLAYER; + remotePlayerMsg.lobby_id = LobbyID; + remotePlayerMsg.user_id = remoteMember.UserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); + } + } + } + public async Task RemoveMember(LobbyMember member) { // TODO_LOBBY: Optimize this @@ -1791,4 +2000,4 @@ public bool IsUserInLobby(Lobby lobby, Int64 user_id) return member != null; } } -} +} diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index d311dd9..eba46fd 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -309,14 +309,17 @@ public class MatchmakingBucket private Int64 m_timeReachedMinPlayers = -1; private bool m_bReachedMinPlayers = false; - private bool m_bWaitingOnLobbyJoins = false; - private bool m_bHasStartedCountdown = false; - private bool m_bMergedAway = false; + private bool m_bWaitingOnLobbyJoins = false; + private bool m_bHasStartedCountdown = false; + private bool m_bWaitingOnMeshConnectivityChecks = false; + private volatile bool m_bAutoStartInvalidated = false; + private bool m_bPendingDeletion = false; + private bool m_bMergedAway = false; // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() { return m_bMergedAway; } - public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins; } + public bool IsMergedAway() { return m_bMergedAway; } + public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; } public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } @@ -486,6 +489,11 @@ public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) { + if (m_bPendingDeletion || bucketToMerge.m_bPendingDeletion) + { + return false; + } + // playlist must match if (bucketToMerge.PlaylistID != this.PlaylistID) { @@ -599,6 +607,11 @@ public bool RemovePlayer(UserSession playerSession) { if (member.Is(playerSession)) { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + m_lstMembers.Remove(member); return true; } @@ -612,7 +625,7 @@ public int CurrentMemberCount() return m_lstMembers.Count; } - // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, + // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, // which both blocks the "everyone joined the lobby" check and skews the average elo public void PruneDeadMembers() { @@ -621,9 +634,14 @@ public void PruneDeadMembers() if (member.GetAssociatedSession() == null) { m_lstMembers.Remove(member); - } - } - } + } + } + } + + internal void MarkPendingDeletion() + { + m_bPendingDeletion = true; + } // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) @@ -657,9 +675,9 @@ public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joini } public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // stale bucket that was merged into another one - if (m_bMergedAway) + { + // stale buckets must never accept new members + if (m_bMergedAway || m_bPendingDeletion) { return false; } @@ -784,16 +802,140 @@ public Int64 GetLobbyID() return m_LobbyID; } - Int64 m_LobbyID = -1; - Int64 m_StartTime = -1; - Int64 m_timeStartedWaitingOnLobbyJoins = -1; + Int64 m_LobbyID = -1; + Int64 m_StartTime = -1; + Int64 m_timeStartedWaitingOnLobbyJoins = -1; + + // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers + private const Int64 c_LobbyJoinTimeoutMSec = 45000; + + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) + { + Console.WriteLine("START GAME"); + + WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); + startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(bytesJSON); + } + } + + await lobby.UpdateState(ELobbyState.INGAME); + MatchmakingManager.DestroyBucket(this); + } - // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers - private const Int64 c_LobbyJoinTimeoutMSec = 45000; - public async Task Tick() + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) { - // merged into another bucket - our members live there now, ticking would create a second lobby for them - if (m_bMergedAway) + lobby.StartFullMeshConnectivityCheck(); + + const int MeshCheckClientTimeoutMarginMS = 2000; + WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); + setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; + setupProgress.timeout_ms = Lobby.MaxFullMeshConnectivityCheckDurationMS + MeshCheckClientTimeoutMarginMS; + byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(setupProgressJSON); + } + } + + lobby.SendFullMeshConnectivityCheckRequestToMembers(); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); + } + } + + m_bWaitingOnMeshConnectivityChecks = true; + } + + private async Task AbortQuickMatchAutoStart(string reason) + { + List sessionsToRequeue = new(); + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } + } + + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + m_bAutoStartInvalidated = false; + + LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); + Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); + if (quickMatchLobby != null) + { + foreach (UserSession memberSession in sessionsToRequeue) + { + LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); + if (lobbyMember != null) + { + // Legacy clients do not understand the requeue action, but they can still tear down + // peer and anti-cheat connections before joining the next temporary lobby. + quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); + await quickMatchLobby.RemoveMember(lobbyMember); + } + } + } + + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); + requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; + byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); + + foreach (UserSession memberSession in sessionsToRequeue) + { + memberSession.UpdateSessionLobbyID(-1); + memberSession.QueueWebsocketSend(requeueActionJSON); + + bool bAlreadyQueued = false; + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && pendingSession == memberSession) + { + bAlreadyQueued = true; + break; + } + } + + if (!bAlreadyQueued) + { + lstSessions.Add(new WeakReference(memberSession)); + } + + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); + } + + m_lstMembers.Clear(); + m_LobbyID = -1; + + MatchmakingManager.DestroyBucket(this); + } + + public async Task Tick() + { + // merged/deleted buckets must not create another lobby or continue a committed setup + if (m_bMergedAway || m_bPendingDeletion) { return; } @@ -804,14 +946,48 @@ public async Task Tick() // never reach the "everyone is in the lobby" condition PruneDeadMembers(); - // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket - if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket + if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) + { + // nobody left? clean ourselves up rather than lingering as a ghost bucket + if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + { + MatchmakingManager.DestroyBucket(this); + return; + } + + if (m_bAutoStartInvalidated) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + return; + } + + if (m_bWaitingOnMeshConnectivityChecks) { - MatchmakingManager.DestroyBucket(this); - return; + Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); + if (lobbyDuringMeshCheck == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); + + if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) + { + m_bWaitingOnMeshConnectivityChecks = false; + + if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true) + { + await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); + } + else + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because not all players were fully mesh-connected."); + } + } + + return; } // do we need to start? @@ -1090,39 +1266,21 @@ await SendMatchmakingMessage(memberSession, } - // TODO_QUICKMATCH: Do full mesh connectivity check + handle not being connected + // trigger full mesh check before issuing the final quickmatch start command // do we have a countdown? if (m_StartTime != -1) { if (Environment.TickCount64 >= m_StartTime) { m_StartTime = -1; - - Console.WriteLine("START GAME"); - - // send start - WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); - startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(bytesJSON); - } - } - - // start match + create placeholder match Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby != null) + if (lobby == null) { - await lobby.UpdateState(ELobbyState.INGAME); + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; } - // destroy the bucket - MatchmakingManager.DestroyBucket(this); + await TriggerFullMeshConnectivityChecks(lobby); } } } @@ -1453,6 +1611,7 @@ public static async Task Tick() private static ConcurrentList m_lstBucketsPendingDeletion = new(); public static void DestroyBucket(MatchmakingBucket bucket) { + bucket.MarkPendingDeletion(); m_lstBucketsPendingDeletion.Add(bucket); } @@ -1504,10 +1663,10 @@ private static void RemoveSessionFromPendingList(UserSession plr) } } - private static void RemovePlayerFromAllBuckets(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - + private static void RemovePlayerFromAllBuckets(UserSession plr) + { + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + // also remove from any bucket we are in to avoid ghost buckets foreach (var kvPair in m_dictMatchmakingBuckets) { @@ -1553,4 +1712,4 @@ public static void DeregisterPlayer(UserSession plr) Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); lobbyManager.LeaveAnyLobby(plr.m_UserID); } -} \ No newline at end of file +} From 98462a6b9cd567105b21b5430682c538b3f60f8a Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:15:30 +0200 Subject: [PATCH 2/5] fix(matchmaking): harden setup concurrency and recovery Serialize registration, cancellation, assignment, and requeue operations. Make lobby removal and bucket cleanup safe under concurrent disconnect and teardown paths. --- GenOnlineService/Constants.cs | 18 +- .../Matchmaking/MatchmakingController.cs | 14 +- .../WebSocket/WebSocketController.cs | 2 +- GenOnlineService/LobbyManager.cs | 377 ++++++----- GenOnlineService/MatchmakingManager.cs | 612 ++++++++++++------ 5 files changed, 631 insertions(+), 392 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index e04f7f6..4f8fc6a 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -1178,9 +1178,11 @@ public class UserSession private string ACExeCRC = String.Empty; - // Matchmaking data - public UInt16 MatchmakingPlaylistID = 0; - public ConcurrentList MatchmakingMapIndicies = new(); + // Matchmaking data + public UInt16 MatchmakingPlaylistID = 0; + public ConcurrentList MatchmakingMapIndicies = new(); + internal bool IsRegisteredForMatchmaking { get; set; } = false; + internal SemaphoreSlim MatchmakingStateLock { get; } = new(1, 1); // NOTE: These are not set on login, only when in quickmatch! public UInt32 ExeCRC = 0; @@ -1657,11 +1659,11 @@ public static async Task FullyDestroyPlayerSession(Int64 user_id, UserSession? u await lobbyManager.CleanupUserLobbiesNotStarted(user_id); - // remove from any matchmaking - if (userData != null) - { - MatchmakingManager.DeregisterPlayer(userData); - } + // remove from any matchmaking + if (userData != null) + { + await MatchmakingManager.DeregisterPlayer(userData); + } // TODO: Client needs to handle this... itll start returning 404 } diff --git a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs index 3a9794d..16c131c 100644 --- a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs +++ b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs @@ -118,9 +118,9 @@ public void Put_Widen() } } - [HttpDelete] - [Authorize(Roles = "GameClient")] - public void Delete() + [HttpDelete] + [Authorize(Roles = "GameClient")] + public async Task Delete() { Int64 user_id = TokenHelper.GetUserID(this); EUserSessionType sessionType = TokenHelper.GetSessionType(this); @@ -128,10 +128,10 @@ public void Delete() { UserSession? playerSession = WebSocketManager.GetSessionFromUser(user_id, sessionType); - if (playerSession != null) - { - MatchmakingManager.DeregisterPlayer(playerSession); - } + if (playerSession != null) + { + await MatchmakingManager.DeregisterPlayer(playerSession); + } } } diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index 2997cbe..238dc35 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -1020,7 +1020,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession lobbyInfo.StartFullMeshConnectivityCheck(); // start full mesh connectivity checks - lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); + lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); } else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) { diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 8e2c5d7..87be4c4 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -35,9 +35,35 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace GenOnlineService -{ - public class Lobby +namespace GenOnlineService +{ + internal static class FullMeshCheckProtocol + { + // TODO: Remove the zero-valued response compatibility path when legacy + // clients are no longer supported. + internal static bool IsLegacyResponse(WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + return response.mesh_check_id == 0 && response.attempt == 0; + } + + internal static bool MatchesCurrentAttempt( + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response, + Int64 currentCheckID, + int currentAttempt) + { + return IsLegacyResponse(response) + ? currentAttempt == 1 + : response.mesh_check_id == currentCheckID && response.attempt == currentAttempt; + } + + internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int currentAttempt, int maxAttempts) + { + // TODO: Remove legacy retry suppression together with the legacy response path. + return !meshComplete && !hasLegacyResponse && currentAttempt < maxAttempts; + } + } + + public class Lobby { public Int64 LobbyID { get; private set; } = -1; public Int64 Owner { get; private set; } = -1; @@ -62,21 +88,21 @@ public class Lobby public const int MaxFullMeshConnectivityCheckDurationMS = (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); - - private readonly object m_FullMeshCheckLock = new(); - - [JsonIgnore] - public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; + + private readonly object m_FullMeshCheckLock = new(); + + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; [JsonIgnore] - public int FullMeshCheckAttempt { get; private set; } = 0; - + public int FullMeshCheckAttempt { get; private set; } = 0; + [JsonIgnore] public Int64 FullMeshCheckID { get; private set; } = 0; [JsonIgnore] private Int64 m_TimeToRetryFullMeshChecks = -1; - + [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; @@ -167,94 +193,95 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) // End AC Probes public void StartFullMeshConnectivityCheck() { - lock (m_FullMeshCheckLock) + lock (m_FullMeshCheckLock) { FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); - FullMeshCheckAttempt = 1; - m_TimeToRetryFullMeshChecks = -1; - LastFullMeshConnectivityCheckOutcome = null; - BeginFullMeshConnectivityCheckAttempt(); - } - } - - private void BeginFullMeshConnectivityCheckAttempt() - { + FullMeshCheckAttempt = 1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + BeginFullMeshConnectivityCheckAttempt(); + } + } + + private void BeginFullMeshConnectivityCheckAttempt() + { PendingFullMeshConnectivityChecks = true; FullMeshConnectivityChecks = new(); m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; } - public void SendFullMeshConnectivityCheckRequestToMembers() + public void SendFullMeshConnectivityCheckRequestToMembers() { WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; startCommand.mesh_check_id = FullMeshCheckID; startCommand.attempt = FullMeshCheckAttempt; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember member in Members) - { - if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) - { - session.QueueWebsocketSend(bytesJSON); - } - } - } - - // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player - // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. - private void RestartSignallingForMissingConnections(List lstMissingConnections) - { - HashSet<(Int64, Int64)> alreadyResignalled = new(); - - foreach (MissingConnectionEntry missingConnection in lstMissingConnections) - { - Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); - Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); - - if (!alreadyResignalled.Add((lowUserID, highUserID))) - { - continue; - } - - LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); - LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); - - if (sourceMember == null || targetMember == null) - { - continue; - } - - Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); - - SendStartSignallingToMember(sourceMember, targetMember); - SendStartSignallingToMember(targetMember, sourceMember); - } - } - - private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) - { - if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) - { - WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); - signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; - signallingMsg.lobby_id = LobbyID; - signallingMsg.user_id = peer.UserID; - signallingMsg.preferred_port = peer.Port; - signallingMsg.middleware_id = peer.MiddlewareUserID; - recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); - } - } - + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player + // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. + private void RestartSignallingForMissingConnections(List lstMissingConnections) + { + HashSet<(Int64, Int64)> alreadyResignalled = new(); + + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } + } + + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) + { + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } + } + public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) { lock (m_FullMeshCheckLock) { - bool bLegacyResponse = response.mesh_check_id == 0 && response.attempt == 0; - bool bMatchesCurrentAttempt = bLegacyResponse - ? FullMeshCheckAttempt == 1 - : response.mesh_check_id == FullMeshCheckID && response.attempt == FullMeshCheckAttempt; + bool bLegacyResponse = FullMeshCheckProtocol.IsLegacyResponse(response); + bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( + response, + FullMeshCheckID, + FullMeshCheckAttempt); LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); if (PendingFullMeshConnectivityChecks @@ -264,45 +291,45 @@ public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage { m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); - } - - ProcessPendingFullMeshConnectivityChecksInternal(); - } - - return Task.CompletedTask; - } - - public Task ProcessPendingFullMeshConnectivityChecks() - { - lock (m_FullMeshCheckLock) - { - ProcessPendingFullMeshConnectivityChecksInternal(); - } - - return Task.CompletedTask; - } - - private void ProcessPendingFullMeshConnectivityChecksInternal() - { - if (!PendingFullMeshConnectivityChecks) - { - return; - } - - // Give re-signalled connections time to establish before starting the retry. - if (m_TimeToRetryFullMeshChecks != -1) - { - if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) - { - return; - } - - m_TimeToRetryFullMeshChecks = -1; - BeginFullMeshConnectivityCheckAttempt(); - SendFullMeshConnectivityCheckRequestToMembers(); - return; - } - + } + + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + public Task ProcessPendingFullMeshConnectivityChecks() + { + lock (m_FullMeshCheckLock) + { + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + + // Give re-signalled connections time to establish before starting the retry. + if (m_TimeToRetryFullMeshChecks != -1) + { + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) + { + return; + } + + m_TimeToRetryFullMeshChecks = -1; + BeginFullMeshConnectivityCheckAttempt(); + SendFullMeshConnectivityCheckRequestToMembers(); + return; + } + { bool bDoneChecks = false; int totalMapEntriesExpected = GetNumberOfHumans(); @@ -352,24 +379,24 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } } - // A member that never reported cannot be assumed connected, so treat them as missing to everyone. - foreach (LobbyMember member in Members) - { - if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) - { - foreach (LobbyMember otherMember in Members) - { - if (otherMember.IsHuman() && otherMember.UserID != member.UserID) - { - MissingConnectionEntry missingConnectionEntry = new(); - missingConnectionEntry.source_user_id = member.UserID; - missingConnectionEntry.target_user_id = otherMember.UserID; - lstMissingConnections.Add(missingConnectionEntry); - } - } - } - } - + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -383,18 +410,20 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() - bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; - - bool bAllMembersReportedCurrentAttempt = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; - bool bCanSafelyRetry = bAllMembersReportedCurrentAttempt && !m_bCurrentAttemptHasLegacyResponse; - if (!bMeshComplete && bCanSafelyRetry && FullMeshCheckAttempt < MaxFullMeshCheckAttempts) - { - ++FullMeshCheckAttempt; - RestartSignallingForMissingConnections(lstMissingConnections); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; - return; - } - + bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + + if (FullMeshCheckProtocol.ShouldRetry( + bMeshComplete, + m_bCurrentAttemptHasLegacyResponse, + FullMeshCheckAttempt, + MaxFullMeshCheckAttempts)) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(lstMissingConnections); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + return; + } + // inform host that we are done // start full mesh connectivity checks WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -411,8 +440,8 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() outcome.missing_connections = lstMissingConnections; } - LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; - + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID // send to host @@ -426,7 +455,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } } } @@ -840,8 +869,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { - await ProcessPendingFullMeshConnectivityChecks(); - + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -1153,16 +1182,32 @@ public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) } } - public async Task RemoveMember(LobbyMember member) - { - // TODO_LOBBY: Optimize this - Int64 UserID = member.UserID; - - Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); - - LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); - Members[member.SlotIndex] = placeholderMember; - TimeMemberLeft[UserID] = DateTime.UtcNow; + public async Task RemoveMember(LobbyMember member) + { + // Matchmaking cancellation and the client's explicit lobby leave can arrive concurrently. + // Claim the slot once so teardown, host migration, and destruction callbacks stay idempotent. + await g_SlotLock.WaitAsync(); + try + { + if (member.SlotIndex < 0 + || member.SlotIndex >= Members.Length + || !ReferenceEquals(Members[member.SlotIndex], member)) + { + return; + } + + LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); + Members[member.SlotIndex] = placeholderMember; + TimeMemberLeft[member.UserID] = DateTime.UtcNow; + } + finally + { + g_SlotLock.Release(); + } + + // TODO_LOBBY: Optimize this + Int64 UserID = member.UserID; + Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); // AC dergister WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index eba46fd..43d29b0 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -309,17 +309,41 @@ public class MatchmakingBucket private Int64 m_timeReachedMinPlayers = -1; private bool m_bReachedMinPlayers = false; - private bool m_bWaitingOnLobbyJoins = false; - private bool m_bHasStartedCountdown = false; - private bool m_bWaitingOnMeshConnectivityChecks = false; - private volatile bool m_bAutoStartInvalidated = false; - private bool m_bPendingDeletion = false; - private bool m_bMergedAway = false; + private bool m_bWaitingOnLobbyJoins = false; + private bool m_bHasStartedCountdown = false; + private bool m_bWaitingOnMeshConnectivityChecks = false; + private bool m_bAutoStartInvalidated = false; + private bool m_bPendingDeletion = false; + private bool m_bMergedAway = false; + private bool m_bAbortInProgress = false; + private bool m_bStartCommitted = false; + private readonly object m_StateLock = new(); // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() { return m_bMergedAway; } - public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; } + public bool IsMergedAway() + { + lock (m_StateLock) + { + return m_bMergedAway; + } + } + + public bool IsLockedForMatchStart() + { + lock (m_StateLock) + { + return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; + } + } + + private bool IsPendingDeletion() + { + lock (m_StateLock) + { + return m_bPendingDeletion; + } + } public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } @@ -489,7 +513,7 @@ public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) { - if (m_bPendingDeletion || bucketToMerge.m_bPendingDeletion) + if (IsPendingDeletion() || bucketToMerge.IsPendingDeletion()) { return false; } @@ -501,7 +525,7 @@ public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // either bucket already merged away this tick? it is stale, never touch it again - if (m_bMergedAway || bucketToMerge.m_bMergedAway) + if (IsMergedAway() || bucketToMerge.IsMergedAway()) { return false; } @@ -572,7 +596,10 @@ public async Task MergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // the source bucket is now empty and flagged so it can never merge/accept players again - bucketToMerge.m_bMergedAway = true; + lock (bucketToMerge.m_StateLock) + { + bucketToMerge.m_bMergedAway = true; + } bucketToMerge.m_lstMembers.Clear(); // nothing else to copy... everything else should match since we were a merge candidate @@ -601,19 +628,29 @@ public bool HasPlayer(UserSession playerSession) return false; } - public bool RemovePlayer(UserSession playerSession) + public bool RemovePlayer(UserSession playerSession, out bool bCancellationRejected) { - foreach (MatchmakingBucketMember member in m_lstMembers) + bCancellationRejected = false; + lock (m_StateLock) { - if (member.Is(playerSession)) + foreach (MatchmakingBucketMember member in m_lstMembers) { - if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) - { - m_bAutoStartInvalidated = true; - } - - m_lstMembers.Remove(member); - return true; + if (member.Is(playerSession)) + { + if (m_bStartCommitted) + { + bCancellationRejected = true; + return false; + } + + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + + m_lstMembers.Remove(member); + return true; + } } } @@ -625,23 +662,38 @@ public int CurrentMemberCount() return m_lstMembers.Count; } - // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, + // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, // which both blocks the "everyone joined the lobby" check and skews the average elo - public void PruneDeadMembers() + public bool PruneDeadMembers() { - foreach (MatchmakingBucketMember member in m_lstMembers) + bool bInvalidatedSetup = false; + lock (m_StateLock) { - if (member.GetAssociatedSession() == null) + foreach (MatchmakingBucketMember member in m_lstMembers) { - m_lstMembers.Remove(member); - } - } - } - - internal void MarkPendingDeletion() - { - m_bPendingDeletion = true; - } + if (member.GetAssociatedSession() == null) + { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + bInvalidatedSetup = true; + } + + m_lstMembers.Remove(member); + } + } + } + + return bInvalidatedSetup; + } + + internal void MarkPendingDeletion() + { + lock (m_StateLock) + { + m_bPendingDeletion = true; + } + } // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) @@ -675,9 +727,9 @@ public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joini } public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // stale buckets must never accept new members - if (m_bMergedAway || m_bPendingDeletion) + { + // stale buckets must never accept new members + if (IsMergedAway() || IsPendingDeletion()) { return false; } @@ -732,7 +784,7 @@ public int GetAvgElo() { SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); - if (memberUserData != null) + if (memberUserData?.GameStats != null) { avgElo += MatchmakingManager.GetMatchmakingElo(memberUserData.GameStats); ++numContributingMembers; @@ -802,14 +854,14 @@ public Int64 GetLobbyID() return m_LobbyID; } - Int64 m_LobbyID = -1; - Int64 m_StartTime = -1; - Int64 m_timeStartedWaitingOnLobbyJoins = -1; - - // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers - private const Int64 c_LobbyJoinTimeoutMSec = 45000; - - private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) + Int64 m_LobbyID = -1; + Int64 m_StartTime = -1; + Int64 m_timeStartedWaitingOnLobbyJoins = -1; + + // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers + private const Int64 c_LobbyJoinTimeoutMSec = 45000; + + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) { Console.WriteLine("START GAME"); @@ -832,23 +884,29 @@ private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) { - lobby.StartFullMeshConnectivityCheck(); - - const int MeshCheckClientTimeoutMarginMS = 2000; - WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); - setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; - setupProgress.timeout_ms = Lobby.MaxFullMeshConnectivityCheckDurationMS + MeshCheckClientTimeoutMarginMS; - byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(setupProgressJSON); - } - } - + lobby.StartFullMeshConnectivityCheck(); + lock (m_StateLock) + { + // Publish the state transition before any notification work. If a send fails, the + // regular lobby timeout still completes or aborts the setup instead of stranding it. + m_bWaitingOnMeshConnectivityChecks = true; + } + + const int MeshCheckClientTimeoutMarginMS = 2000; + WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); + setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; + setupProgress.timeout_ms = Lobby.MaxFullMeshConnectivityCheckDurationMS + MeshCheckClientTimeoutMarginMS; + byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(setupProgressJSON); + } + } + lobby.SendFullMeshConnectivityCheckRequestToMembers(); foreach (MatchmakingBucketMember member in m_lstMembers) @@ -859,27 +917,35 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); } } - - m_bWaitingOnMeshConnectivityChecks = true; } private async Task AbortQuickMatchAutoStart(string reason) { List sessionsToRequeue = new(); - foreach (MatchmakingBucketMember member in m_lstMembers) + lock (m_StateLock) { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) + if (m_bAbortInProgress || m_bStartCommitted) { - sessionsToRequeue.Add(memberSession); + return; + } + + m_bAbortInProgress = true; + m_bPendingDeletion = true; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } } - } - m_StartTime = -1; - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = false; - m_bWaitingOnMeshConnectivityChecks = false; - m_bAutoStartInvalidated = false; + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + } LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); @@ -890,40 +956,26 @@ private async Task AbortQuickMatchAutoStart(string reason) LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); if (lobbyMember != null) { - // Legacy clients do not understand the requeue action, but they can still tear down - // peer and anti-cheat connections before joining the next temporary lobby. - quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); + // TODO: Remove this fallback once all supported clients handle MATCHMAKING_ACTION_REQUEUE. + // Legacy clients do not understand the requeue action, but they can still tear down + // peer and anti-cheat connections before joining the next temporary lobby. + quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); await quickMatchLobby.RemoveMember(lobbyMember); } } } - WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); - requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; - byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); - + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); + requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; + byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); + foreach (UserSession memberSession in sessionsToRequeue) { - memberSession.UpdateSessionLobbyID(-1); - memberSession.QueueWebsocketSend(requeueActionJSON); - - bool bAlreadyQueued = false; - foreach (WeakReference wrSession in lstSessions) - { - if (wrSession.TryGetTarget(out UserSession? pendingSession) && pendingSession == memberSession) - { - bAlreadyQueued = true; - break; - } - } - - if (!bAlreadyQueued) + if (await TryRequeueRegisteredPlayer(memberSession, requeueActionJSON)) { - lstSessions.Add(new WeakReference(memberSession)); + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); } - - await SendMatchmakingMessage(memberSession, reason); - await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); } m_lstMembers.Clear(); @@ -932,10 +984,24 @@ private async Task AbortQuickMatchAutoStart(string reason) MatchmakingManager.DestroyBucket(this); } - public async Task Tick() - { - // merged/deleted buckets must not create another lobby or continue a committed setup - if (m_bMergedAway || m_bPendingDeletion) + public async Task Tick() + { + bool bPendingDeletion; + bool bAutoStartInvalidated; + bool bWaitingOnMeshConnectivityChecks; + bool bWaitingOnLobbyJoins; + bool bHasStartedCountdown; + lock (m_StateLock) + { + bPendingDeletion = m_bPendingDeletion; + bAutoStartInvalidated = m_bAutoStartInvalidated; + bWaitingOnMeshConnectivityChecks = m_bWaitingOnMeshConnectivityChecks; + bWaitingOnLobbyJoins = m_bWaitingOnLobbyJoins; + bHasStartedCountdown = m_bHasStartedCountdown; + } + + // merged/deleted buckets must not create another lobby or continue a committed setup + if (IsMergedAway() || bPendingDeletion) { return; } @@ -944,25 +1010,28 @@ public async Task Tick() // drop any members whose session has gone away, otherwise they are counted forever and the bucket can // never reach the "everyone is in the lobby" condition - PruneDeadMembers(); - - // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket - if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) - { - MatchmakingManager.DestroyBucket(this); - return; - } - - if (m_bAutoStartInvalidated) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); - return; - } - - if (m_bWaitingOnMeshConnectivityChecks) + if (PruneDeadMembers()) + { + bAutoStartInvalidated = true; + } + + // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket + if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) + { + // nobody left? clean ourselves up rather than lingering as a ghost bucket + if (CurrentMemberCount() == 0 && !bWaitingOnLobbyJoins && !bHasStartedCountdown) + { + MatchmakingManager.DestroyBucket(this); + return; + } + + if (bAutoStartInvalidated) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + return; + } + + if (bWaitingOnMeshConnectivityChecks) { Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); if (lobbyDuringMeshCheck == null) @@ -975,19 +1044,55 @@ public async Task Tick() if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) { - m_bWaitingOnMeshConnectivityChecks = false; + // Start the mesh check alongside the existing five-second countdown. A successful + // check still waits for the countdown, while a failed check can abort immediately. + if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true + && m_StartTime != -1 + && Environment.TickCount64 < m_StartTime) + { + return; + } - if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true) + bool bStartGame; + bool bAbortStart; + bool bInvalidatedAtDecision; + lock (m_StateLock) + { + bInvalidatedAtDecision = m_bAutoStartInvalidated; + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartGame = false; + bAbortStart = false; + } + else + { + m_bWaitingOnMeshConnectivityChecks = false; + m_bHasStartedCountdown = false; + m_StartTime = -1; + bStartGame = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; + bAbortStart = !bStartGame; + if (bStartGame) + { + m_bStartCommitted = true; + m_bPendingDeletion = true; + } + } + } + + if (bStartGame) { await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); } - else + else if (bAbortStart) { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because not all players were fully mesh-connected."); + string reason = bInvalidatedAtDecision + ? "QuickMatch auto-start was aborted because a player left during match setup." + : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; + await AbortQuickMatchAutoStart(reason); } - } - - return; + } + + return; } // do we need to start? @@ -1056,8 +1161,11 @@ await SendMatchmakingMessage(memberSession, m_bReachedMinPlayers = false; m_timeReachedMinPlayers = -1; - m_bWaitingOnLobbyJoins = true; - m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + lock (m_StateLock) + { + m_bWaitingOnLobbyJoins = true; + m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + } // tell everyone UserSession? dummyHostUser = null; @@ -1224,7 +1332,12 @@ await SendMatchmakingMessage(memberSession, m_timeStartedWaitingOnLobbyJoins = -1; // wait 5 sec - m_StartTime = Environment.TickCount64 + 5000; + lock (m_StateLock) + { + m_StartTime = Environment.TickCount64 + 5000; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = true; + } foreach (MatchmakingBucketMember member in m_lstMembers) { UserSession? memberSession = member.GetAssociatedSession(); @@ -1234,9 +1347,6 @@ await SendMatchmakingMessage(memberSession, } } - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = true; - // finalize the teams const int playlistMaxPlayerPerTeam = 2; bool bIsFFA = true; @@ -1261,28 +1371,13 @@ await SendMatchmakingMessage(memberSession, } } } + + await TriggerFullMeshConnectivityChecks(lobby); } } } - // trigger full mesh check before issuing the final quickmatch start command - // do we have a countdown? - if (m_StartTime != -1) - { - if (Environment.TickCount64 >= m_StartTime) - { - m_StartTime = -1; - Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby == null) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); - return; - } - - await TriggerFullMeshConnectivityChecks(lobby); - } - } } } } @@ -1469,10 +1564,14 @@ public static async Task Tick() } // queue for deletion - m_lstBucketsPendingDeletion.AddRange(lstBucketsMergedNeedingDeleted); + foreach (MatchmakingBucket bucket in lstBucketsMergedNeedingDeleted) + { + m_bucketsPendingDeletion.Enqueue(bucket); + } - // cleanup any pending destruction (cannot do this in tick, collection will be modified) - foreach (MatchmakingBucket bucket in m_lstBucketsPendingDeletion) + // Drain the queue rather than enumerating and clearing a shared list. A concurrent cancellation can + // enqueue a bucket while cleanup is running, and clearing the list would otherwise lose that request. + while (m_bucketsPendingDeletion.TryDequeue(out MatchmakingBucket? bucket)) { if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) { @@ -1481,7 +1580,6 @@ public static async Task Tick() m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); } } - m_lstBucketsPendingDeletion.Clear(); List> lstDestroy = new(); foreach (WeakReference wrSession in lstSessions) @@ -1492,16 +1590,36 @@ public static async Task Tick() } else { - SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); - - if (thisSessionUserData == null) - { - lstDestroy.Add(wrSession); - } - else + await thisSession.MatchmakingStateLock.WaitAsync(); + try { - if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + // A cancellation can remove the weak reference while this tick is iterating a snapshot. + // Re-check registration while holding the per-session gate before assigning any bucket. + if (!thisSession.IsRegisteredForMatchmaking || !IsPendingSession(thisSession)) + { + lstDestroy.Add(wrSession); + continue; + } + + SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); + + if (thisSessionUserData == null) { + lstDestroy.Add(wrSession); + } + else + { + PlayerStats? thisSessionStats = thisSessionUserData.GameStats; + if (thisSessionStats == null) + { + thisSession.IsRegisteredForMatchmaking = false; + lstDestroy.Add(wrSession); + await SendMatchmakingMessage(thisSession, "Matchmaking could not start because your player statistics are unavailable. Please try again."); + continue; + } + + if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + { // TODO_MATCHAMAKING: Better way of tracking this, we need to know who is already in a bucket // Was the user in a bucket? if so theres nothing to do in terms of bucket management @@ -1585,13 +1703,18 @@ public static async Task Tick() lstDestroy.Add(wrSession); } } - } - else - { - // invalid playlist somehow - lstDestroy.Add(wrSession); + } + else + { + // invalid playlist somehow + lstDestroy.Add(wrSession); + } } } + finally + { + thisSession.MatchmakingStateLock.Release(); + } } } @@ -1608,11 +1731,60 @@ public static async Task Tick() // TODO_MATCHMAKING: Deregister player if they disconnect or leave quickmatch private static ConcurrentList> lstSessions = new(); - private static ConcurrentList m_lstBucketsPendingDeletion = new(); + private static bool IsPendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && ReferenceEquals(pendingSession, session)) + { + return true; + } + } + + return false; + } + + private static void RemovePendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions.ToList()) + { + if (!wrSession.TryGetTarget(out UserSession? pendingSession) || ReferenceEquals(pendingSession, session)) + { + lstSessions.Remove(wrSession); + } + } + } + + private static async Task TryRequeueRegisteredPlayer(UserSession session, byte[] requeueActionJSON) + { + await session.MatchmakingStateLock.WaitAsync(); + try + { + if (!session.IsRegisteredForMatchmaking) + { + return false; + } + + if (!IsPendingSession(session)) + { + lstSessions.Add(new WeakReference(session)); + } + + session.UpdateSessionLobbyID(-1); + session.QueueWebsocketSend(requeueActionJSON); + return true; + } + finally + { + session.MatchmakingStateLock.Release(); + } + } + + private static ConcurrentQueue m_bucketsPendingDeletion = new(); public static void DestroyBucket(MatchmakingBucket bucket) { bucket.MarkPendingDeletion(); - m_lstBucketsPendingDeletion.Add(bucket); + m_bucketsPendingDeletion.Enqueue(bucket); } public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) @@ -1636,43 +1808,54 @@ public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List return; } - // make sure a re-register (or a duplicate request) cannot leave the player queued twice, or queued while - // still sat in an existing bucket - either would matchmake them into two lobbies at once - RemoveSessionFromPendingList(plr); - RemovePlayerFromAllBuckets(plr); + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + // A duplicate registration must not leave the player queued twice or in two buckets. + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); - plr.MatchmakingPlaylistID = playlistID; - plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); - plr.ExeCRC = exe_crc; - plr.IniCRC = ini_crc; - plr.AnticheatID = anticheatID; - lstSessions.Add(new WeakReference(plr)); + if (!bCancellationRejected) + { + plr.MatchmakingPlaylistID = playlistID; + plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); + plr.ExeCRC = exe_crc; + plr.IniCRC = ini_crc; + plr.AnticheatID = anticheatID; + plr.IsRegisteredForMatchmaking = true; + lstSessions.Add(new WeakReference(plr)); + } + } + finally + { + plr.MatchmakingStateLock.Release(); + } + + if (bCancellationRejected) + { + await SendMatchmakingMessage(plr, "Matchmaking cannot be restarted because your game is already starting."); + return; + } await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); } - // NOTE: WeakReference does not implement value equality, so entries must be matched by their target session - private static void RemoveSessionFromPendingList(UserSession plr) + private static async Task RemovePlayerFromAllBuckets(UserSession plr) { - foreach (WeakReference wrSession in lstSessions) - { - if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null || thisSession == plr) - { - lstSessions.Remove(wrSession); - } - } - } + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + bool bCancellationRejected = false; - private static void RemovePlayerFromAllBuckets(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - // also remove from any bucket we are in to avoid ghost buckets foreach (var kvPair in m_dictMatchmakingBuckets) { foreach (MatchmakingBucket mmBucket in kvPair.Value) { - if (mmBucket.HasPlayer(plr)) + bool bRemoved = mmBucket.RemovePlayer(plr, out bool bBucketCancellationRejected); + bCancellationRejected |= bBucketCancellationRejected; + + if (bRemoved) { // remove from QM lobby too Lobby? lobby = lobbyManager.GetLobby(mmBucket.GetLobbyID()); @@ -1682,13 +1865,10 @@ private static void RemovePlayerFromAllBuckets(UserSession plr) if (lobbyMember != null) { Console.WriteLine("User {0} Leave MM Lobby", plr.m_UserID); - lobby.RemoveMember(lobbyMember); + await lobby.RemoveMember(lobbyMember); } } - // remove player - mmBucket.RemovePlayer(plr); - // if we're the last player, destroy the bucket if (mmBucket.CurrentMemberCount() == 0) { @@ -1697,19 +1877,31 @@ private static void RemovePlayerFromAllBuckets(UserSession plr) } } } + + return bCancellationRejected; } - public static void DeregisterPlayer(UserSession plr) + public static async Task DeregisterPlayer(UserSession plr) { var lobbyManager = ServiceLocator.Services.GetRequiredService(); - RemoveSessionFromPendingList(plr); - - // TODO_QUICKMATCH: What happens if the game is going to start? we should handle that, right now people probably goto game solo - - RemovePlayerFromAllBuckets(plr); + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); + } + finally + { + plr.MatchmakingStateLock.Release(); + } // leave QM lobby too - Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); - lobbyManager.LeaveAnyLobby(plr.m_UserID); + if (!bCancellationRejected) + { + Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); + await lobbyManager.LeaveAnyLobby(plr.m_UserID); + } } -} +} From 50eac8c67c186a1d66d5253d1c73e7e6f0161573 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:40:53 +0200 Subject: [PATCH 3/5] fix(matchmaking): delete aborted quickmatch lobbies --- GenOnlineService/MatchmakingManager.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 43d29b0..aa62319 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -963,6 +963,8 @@ private async Task AbortQuickMatchAutoStart(string reason) await quickMatchLobby.RemoveMember(lobbyMember); } } + + await lobbyManager.DeleteLobby(quickMatchLobby); } WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); From d37731e40e07c4dc324825ec32caf8ac20faa667 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:00:31 +0200 Subject: [PATCH 4/5] fix(lobby): prevent joins after deletion --- GenOnlineService/LobbyManager.cs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 87be4c4..0c28b0c 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -955,10 +955,15 @@ public async Task Tick() public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { // NOTE: AddMember is called async, so timing + slot determination could result in players being inserted in the same slot - await g_SlotLock.WaitAsync(); - try - { - // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass + await g_SlotLock.WaitAsync(); + try + { + if (State != ELobbyState.GAME_SETUP) + { + return false; + } + + // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass // the check and end up occupying two slots LobbyMember? existingMember = GetMemberFromUserID(playerSession.m_UserID); if (existingMember != null) // we're already in this lobby @@ -1448,7 +1453,15 @@ public bool HadAIAtStart() public async Task UpdateState(ELobbyState state) { - State = state; + await g_SlotLock.WaitAsync(); + try + { + State = state; + } + finally + { + g_SlotLock.Release(); + } // if start, init our AC probe if (state == ELobbyState.INGAME) From 7e5914600676870c66bc09d2dda95edd40af101b Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:22:02 +0200 Subject: [PATCH 5/5] fix(matchmaking): start countdown after mesh checks --- GenOnlineService/MatchmakingManager.cs | 127 ++++++++++++++++--------- 1 file changed, 84 insertions(+), 43 deletions(-) diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index aa62319..5ba2285 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -860,6 +860,8 @@ public Int64 GetLobbyID() // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers private const Int64 c_LobbyJoinTimeoutMSec = 45000; + private const int c_GameStartCountdownMSec = 5000; + private const int c_SetupClientTimeoutMarginMSec = 2000; private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) { @@ -882,20 +884,11 @@ private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) MatchmakingManager.DestroyBucket(this); } - private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) + private void QueueSetupProgress(int timeoutMSec) { - lobby.StartFullMeshConnectivityCheck(); - lock (m_StateLock) - { - // Publish the state transition before any notification work. If a send fails, the - // regular lobby timeout still completes or aborts the setup instead of stranding it. - m_bWaitingOnMeshConnectivityChecks = true; - } - - const int MeshCheckClientTimeoutMarginMS = 2000; WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; - setupProgress.timeout_ms = Lobby.MaxFullMeshConnectivityCheckDurationMS + MeshCheckClientTimeoutMarginMS; + setupProgress.timeout_ms = timeoutMSec; byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); foreach (MatchmakingBucketMember member in m_lstMembers) @@ -906,6 +899,19 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) memberSession.QueueWebsocketSend(setupProgressJSON); } } + } + + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) + { + lobby.StartFullMeshConnectivityCheck(); + lock (m_StateLock) + { + // Publish the state transition before any notification work. If a send fails, the + // regular lobby timeout still completes or aborts the setup instead of stranding it. + m_bWaitingOnMeshConnectivityChecks = true; + } + + QueueSetupProgress(Lobby.MaxFullMeshConnectivityCheckDurationMS + c_SetupClientTimeoutMarginMSec); lobby.SendFullMeshConnectivityCheckRequestToMembers(); @@ -914,7 +920,7 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) UserSession? memberSession = member.GetAssociatedSession(); if (memberSession != null) { - await SendMatchmakingMessage(memberSession, "Running full mesh connectivity checks before game start..."); + await SendMatchmakingMessage(memberSession, "Preparing match..."); } } } @@ -1046,16 +1052,7 @@ public async Task Tick() if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) { - // Start the mesh check alongside the existing five-second countdown. A successful - // check still waits for the countdown, while a failed check can abort immediately. - if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true - && m_StartTime != -1 - && Environment.TickCount64 < m_StartTime) - { - return; - } - - bool bStartGame; + bool bStartCountdown; bool bAbortStart; bool bInvalidatedAtDecision; lock (m_StateLock) @@ -1063,27 +1060,33 @@ public async Task Tick() bInvalidatedAtDecision = m_bAutoStartInvalidated; if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) { - bStartGame = false; + bStartCountdown = false; bAbortStart = false; } else { m_bWaitingOnMeshConnectivityChecks = false; - m_bHasStartedCountdown = false; - m_StartTime = -1; - bStartGame = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; - bAbortStart = !bStartGame; - if (bStartGame) + bStartCountdown = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; + bAbortStart = !bStartCountdown; + if (bStartCountdown) { - m_bStartCommitted = true; - m_bPendingDeletion = true; + m_StartTime = Environment.TickCount64 + c_GameStartCountdownMSec; + m_bHasStartedCountdown = true; } } } - if (bStartGame) + if (bStartCountdown) { - await StartGameAfterSuccessfulMeshCheck(lobbyDuringMeshCheck); + QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec); + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, $"Starting game in {c_GameStartCountdownMSec / 1000} seconds."); + } + } } else if (bAbortStart) { @@ -1097,6 +1100,55 @@ public async Task Tick() return; } + if (bHasStartedCountdown) + { + if (Environment.TickCount64 < m_StartTime) + { + return; + } + + Lobby? lobbyAfterCountdown = lobbyManager.GetLobby(m_LobbyID); + if (lobbyAfterCountdown == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + bool bStartGame; + bool bAbortStart; + lock (m_StateLock) + { + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartGame = false; + bAbortStart = false; + } + else + { + m_bHasStartedCountdown = false; + m_StartTime = -1; + bStartGame = !m_bAutoStartInvalidated; + bAbortStart = !bStartGame; + if (bStartGame) + { + m_bStartCommitted = true; + m_bPendingDeletion = true; + } + } + } + + if (bStartGame) + { + await StartGameAfterSuccessfulMeshCheck(lobbyAfterCountdown); + } + else if (bAbortStart) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + } + + return; + } + // do we need to start? // TODO_MATCHMAKING: Add a timeout at which > min players starts if (!m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) @@ -1333,20 +1385,9 @@ await SendMatchmakingMessage(memberSession, { m_timeStartedWaitingOnLobbyJoins = -1; - // wait 5 sec lock (m_StateLock) { - m_StartTime = Environment.TickCount64 + 5000; m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = true; - } - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, $"Starting Game in 5 seconds"); - } } // finalize the teams