diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index 55aac44..4f8fc6a 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -1178,9 +1178,11 @@ public class UserSession private string ACExeCRC = String.Empty; - // Matchmaking data - public UInt16 MatchmakingPlaylistID = 0; - public ConcurrentList MatchmakingMapIndicies = new(); + // Matchmaking data + public UInt16 MatchmakingPlaylistID = 0; + public ConcurrentList MatchmakingMapIndicies = new(); + internal bool IsRegisteredForMatchmaking { get; set; } = false; + internal SemaphoreSlim MatchmakingStateLock { get; } = new(1, 1); // NOTE: These are not set on login, only when in quickmatch! public UInt32 ExeCRC = 0; @@ -1657,11 +1659,11 @@ public static async Task FullyDestroyPlayerSession(Int64 user_id, UserSession? u await lobbyManager.CleanupUserLobbiesNotStarted(user_id); - // remove from any matchmaking - if (userData != null) - { - MatchmakingManager.DeregisterPlayer(userData); - } + // remove from any matchmaking + if (userData != null) + { + await MatchmakingManager.DeregisterPlayer(userData); + } // TODO: Client needs to handle this... itll start returning 404 } @@ -2843,10 +2845,12 @@ public enum EWebSocketMessageID SOCIAL_FRIENDS_LIST_DIRTY = 37, SOCIAL_CANT_ADD_FRIEND_LIST_FULL = 38, PROBE_RESP = 39, - AC_REGISTER_PLAYER = 40, - AC_DEREGISTER_PLAYER = 41, - WS_KEEPALIVE = 42, + AC_REGISTER_PLAYER = 40, + AC_DEREGISTER_PLAYER = 41, + WS_KEEPALIVE = 42, WS_KEEPALIVE_CLIENT = 43, + MATCHMAKING_ACTION_REQUEUE = 44, + MATCHMAKING_ACTION_SETUP_PROGRESS = 45, MODERATION_NOTICE = 46, MODERATION_COMMAND = 47, MODERATION_COMMAND_RESULT = 48 @@ -2985,8 +2989,16 @@ public class WebSocketMessage_NameChange : WebSocketMessage public string name { get; set; } = String.Empty; } - public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage - { + public class WebSocketMessage_FullMeshConnectivityCheckRequest : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } + } + + public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSocketMessage + { + public Int64 mesh_check_id { get; set; } + public int attempt { get; set; } public List connectivity_map { get; set; } = new(); } @@ -3176,4 +3188,9 @@ public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage } + public class WebSocketMessage_MatchmakerSetupProgress : WebSocketMessage + { + public int timeout_ms { get; set; } + } + } diff --git a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs index 3a9794d..16c131c 100644 --- a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs +++ b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs @@ -118,9 +118,9 @@ public void Put_Widen() } } - [HttpDelete] - [Authorize(Roles = "GameClient")] - public void Delete() + [HttpDelete] + [Authorize(Roles = "GameClient")] + public async Task Delete() { Int64 user_id = TokenHelper.GetUserID(this); EUserSessionType sessionType = TokenHelper.GetSessionType(this); @@ -128,10 +128,10 @@ public void Delete() { UserSession? playerSession = WebSocketManager.GetSessionFromUser(user_id, sessionType); - if (playerSession != null) - { - MatchmakingManager.DeregisterPlayer(playerSession); - } + if (playerSession != null) + { + await MatchmakingManager.DeregisterPlayer(playerSession); + } } } diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index 8d93efb..238dc35 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -1020,25 +1020,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession lobbyInfo.StartFullMeshConnectivityCheck(); // start full mesh connectivity checks - WebSocketMessage_Simple startCommand = new WebSocketMessage_Simple(); - startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; - - // Serialize once before broadcasting - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember lobbyMember in lobbyInfo.Members) - { - if (lobbyMember != null) - { - if (lobbyMember.GetSession().TryGetTarget(out UserSession? sess)) - { - if (sess != null) - { - sess.QueueWebsocketSend(bytesJSON); - } - } - } - } + lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); } else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) { @@ -1052,7 +1034,7 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession Lobby? lobby = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); if (lobby != null) { - await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg.connectivity_map); + await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg); } } } diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index bc0df6d..0c28b0c 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -35,9 +35,35 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace GenOnlineService -{ - public class Lobby +namespace GenOnlineService +{ + internal static class FullMeshCheckProtocol + { + // TODO: Remove the zero-valued response compatibility path when legacy + // clients are no longer supported. + internal static bool IsLegacyResponse(WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + return response.mesh_check_id == 0 && response.attempt == 0; + } + + internal static bool MatchesCurrentAttempt( + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response, + Int64 currentCheckID, + int currentAttempt) + { + return IsLegacyResponse(response) + ? currentAttempt == 1 + : response.mesh_check_id == currentCheckID && response.attempt == currentAttempt; + } + + internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int currentAttempt, int maxAttempts) + { + // TODO: Remove legacy retry suppression together with the legacy response path. + return !meshComplete && !hasLegacyResponse && currentAttempt < maxAttempts; + } + } + + public class Lobby { public Int64 LobbyID { get; private set; } = -1; public Int64 Owner { get; private set; } = -1; @@ -56,9 +82,33 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - private const int MSToWaitForFullMeshChecks = 5000; // really shouldnt take more than 5 seconds... this might even be too much + private const int MSToWaitForFullMeshChecks = 5000; + private const int MaxFullMeshCheckAttempts = 2; + private const int MSBeforeFullMeshCheckRetry = 3000; + public const int MaxFullMeshConnectivityCheckDurationMS = + (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) + + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); + + private readonly object m_FullMeshCheckLock = new(); + + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; [JsonIgnore] + public int FullMeshCheckAttempt { get; private set; } = 0; + + [JsonIgnore] + public Int64 FullMeshCheckID { get; private set; } = 0; + + [JsonIgnore] + private Int64 m_TimeToRetryFullMeshChecks = -1; + + [JsonIgnore] + private bool m_bCurrentAttemptHasLegacyResponse = false; + + private static Int64 s_NextFullMeshCheckID = 0; + + [JsonIgnore] public ConcurrentDictionary> FullMeshConnectivityChecks { get; set; } = new(); @@ -143,23 +193,143 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) // End AC Probes public void StartFullMeshConnectivityCheck() { + lock (m_FullMeshCheckLock) + { + FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); + FullMeshCheckAttempt = 1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + BeginFullMeshConnectivityCheckAttempt(); + } + } + + private void BeginFullMeshConnectivityCheckAttempt() + { PendingFullMeshConnectivityChecks = true; - FullMeshConnectivityChecks = new(); + FullMeshConnectivityChecks = new(); + m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; } - public async Task StoreFullMeshConnectivityResponse(Int64 sourceUser, List connectivityMap) - { - FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(connectivityMap); - - // check again for being done - await ProcessPendingFullMeshConnectivityChecks(); - } - - public async Task ProcessPendingFullMeshConnectivityChecks() - { - // TODO: Add a timeout to this - if (PendingFullMeshConnectivityChecks) + public void SendFullMeshConnectivityCheckRequestToMembers() + { + WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); + startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; + startCommand.mesh_check_id = FullMeshCheckID; + startCommand.attempt = FullMeshCheckAttempt; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player + // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. + private void RestartSignallingForMissingConnections(List lstMissingConnections) + { + HashSet<(Int64, Int64)> alreadyResignalled = new(); + + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } + } + + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) + { + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } + } + + public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + lock (m_FullMeshCheckLock) + { + bool bLegacyResponse = FullMeshCheckProtocol.IsLegacyResponse(response); + bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( + response, + FullMeshCheckID, + FullMeshCheckAttempt); + + LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); + if (PendingFullMeshConnectivityChecks + && m_TimeToRetryFullMeshChecks == -1 + && sourceMember?.IsHuman() == true + && bMatchesCurrentAttempt) + { + m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; + FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + } + + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + public Task ProcessPendingFullMeshConnectivityChecks() + { + lock (m_FullMeshCheckLock) + { + ProcessPendingFullMeshConnectivityChecksInternal(); + } + + return Task.CompletedTask; + } + + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + + // Give re-signalled connections time to establish before starting the retry. + if (m_TimeToRetryFullMeshChecks != -1) + { + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) + { + return; + } + + m_TimeToRetryFullMeshChecks = -1; + BeginFullMeshConnectivityCheckAttempt(); + SendFullMeshConnectivityCheckRequestToMembers(); + return; + } + { bool bDoneChecks = false; int totalMapEntriesExpected = GetNumberOfHumans(); @@ -209,6 +379,24 @@ public async Task ProcessPendingFullMeshConnectivityChecks() } } + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -222,6 +410,20 @@ public async Task ProcessPendingFullMeshConnectivityChecks() + bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + + if (FullMeshCheckProtocol.ShouldRetry( + bMeshComplete, + m_bCurrentAttemptHasLegacyResponse, + FullMeshCheckAttempt, + MaxFullMeshCheckAttempts)) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(lstMissingConnections); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + return; + } + // inform host that we are done // start full mesh connectivity checks WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -238,6 +440,8 @@ public async Task ProcessPendingFullMeshConnectivityChecks() outcome.missing_connections = lstMissingConnections; } + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID // send to host @@ -251,6 +455,7 @@ public async Task ProcessPendingFullMeshConnectivityChecks() // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } } } @@ -664,6 +869,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -748,10 +955,15 @@ public async Task Tick() public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { // NOTE: AddMember is called async, so timing + slot determination could result in players being inserted in the same slot - await g_SlotLock.WaitAsync(); - try - { - // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass + await g_SlotLock.WaitAsync(); + try + { + if (State != ELobbyState.GAME_SETUP) + { + return false; + } + + // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass // the check and end up occupying two slots LobbyMember? existingMember = GetMemberFromUserID(playerSession.m_UserID); if (existingMember != null) // we're already in this lobby @@ -944,16 +1156,63 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa } } - public async Task RemoveMember(LobbyMember member) - { - // TODO_LOBBY: Optimize this - Int64 UserID = member.UserID; - - Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); - - LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); - Members[member.SlotIndex] = placeholderMember; - TimeMemberLeft[UserID] = DateTime.UtcNow; + public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) + { + if (!departingMember.GetSession().TryGetTarget(out UserSession? departingSession) || departingSession == null) + { + return; + } + + foreach (LobbyMember remoteMember in Members) + { + if (remoteMember.SlotState != EPlayerType.SLOT_PLAYER || remoteMember.UserID == departingMember.UserID) + { + continue; + } + + WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); + remotePlayerAcMsg.msg_id = (int)EWebSocketMessageID.AC_DEREGISTER_PLAYER; + remotePlayerAcMsg.user_id = remoteMember.UserID; + remotePlayerAcMsg.mwid = remoteMember.MiddlewareUserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); + + if (State != ELobbyState.INGAME) + { + WebSocketMessage_NetworkDisconnectPlayer remotePlayerMsg = new WebSocketMessage_NetworkDisconnectPlayer(); + remotePlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_DISCONNECT_PLAYER; + remotePlayerMsg.lobby_id = LobbyID; + remotePlayerMsg.user_id = remoteMember.UserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); + } + } + } + + public async Task RemoveMember(LobbyMember member) + { + // Matchmaking cancellation and the client's explicit lobby leave can arrive concurrently. + // Claim the slot once so teardown, host migration, and destruction callbacks stay idempotent. + await g_SlotLock.WaitAsync(); + try + { + if (member.SlotIndex < 0 + || member.SlotIndex >= Members.Length + || !ReferenceEquals(Members[member.SlotIndex], member)) + { + return; + } + + LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); + Members[member.SlotIndex] = placeholderMember; + TimeMemberLeft[member.UserID] = DateTime.UtcNow; + } + finally + { + g_SlotLock.Release(); + } + + // TODO_LOBBY: Optimize this + Int64 UserID = member.UserID; + Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); // AC dergister WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); @@ -1194,7 +1453,15 @@ public bool HadAIAtStart() public async Task UpdateState(ELobbyState state) { - State = state; + await g_SlotLock.WaitAsync(); + try + { + State = state; + } + finally + { + g_SlotLock.Release(); + } // if start, init our AC probe if (state == ELobbyState.INGAME) @@ -1791,4 +2058,4 @@ public bool IsUserInLobby(Lobby lobby, Int64 user_id) return member != null; } } -} +} diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index d311dd9..5ba2285 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -311,12 +311,39 @@ public class MatchmakingBucket private bool m_bReachedMinPlayers = false; private bool m_bWaitingOnLobbyJoins = false; private bool m_bHasStartedCountdown = false; + private bool m_bWaitingOnMeshConnectivityChecks = false; + private bool m_bAutoStartInvalidated = false; + private bool m_bPendingDeletion = false; private bool m_bMergedAway = false; + private bool m_bAbortInProgress = false; + private bool m_bStartCommitted = false; + private readonly object m_StateLock = new(); // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() { return m_bMergedAway; } - public bool IsLockedForMatchStart() { return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins; } + public bool IsMergedAway() + { + lock (m_StateLock) + { + return m_bMergedAway; + } + } + + public bool IsLockedForMatchStart() + { + lock (m_StateLock) + { + return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; + } + } + + private bool IsPendingDeletion() + { + lock (m_StateLock) + { + return m_bPendingDeletion; + } + } public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } @@ -486,6 +513,11 @@ public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) { + if (IsPendingDeletion() || bucketToMerge.IsPendingDeletion()) + { + return false; + } + // playlist must match if (bucketToMerge.PlaylistID != this.PlaylistID) { @@ -493,7 +525,7 @@ public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // either bucket already merged away this tick? it is stale, never touch it again - if (m_bMergedAway || bucketToMerge.m_bMergedAway) + if (IsMergedAway() || bucketToMerge.IsMergedAway()) { return false; } @@ -564,7 +596,10 @@ public async Task MergeWithOtherBucket(MatchmakingBucket bucketToMerge) } // the source bucket is now empty and flagged so it can never merge/accept players again - bucketToMerge.m_bMergedAway = true; + lock (bucketToMerge.m_StateLock) + { + bucketToMerge.m_bMergedAway = true; + } bucketToMerge.m_lstMembers.Clear(); // nothing else to copy... everything else should match since we were a merge candidate @@ -593,14 +628,29 @@ public bool HasPlayer(UserSession playerSession) return false; } - public bool RemovePlayer(UserSession playerSession) + public bool RemovePlayer(UserSession playerSession, out bool bCancellationRejected) { - foreach (MatchmakingBucketMember member in m_lstMembers) + bCancellationRejected = false; + lock (m_StateLock) { - if (member.Is(playerSession)) + foreach (MatchmakingBucketMember member in m_lstMembers) { - m_lstMembers.Remove(member); - return true; + if (member.Is(playerSession)) + { + if (m_bStartCommitted) + { + bCancellationRejected = true; + return false; + } + + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + + m_lstMembers.Remove(member); + return true; + } } } @@ -614,15 +664,35 @@ public int CurrentMemberCount() // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, // which both blocks the "everyone joined the lobby" check and skews the average elo - public void PruneDeadMembers() + public bool PruneDeadMembers() { - foreach (MatchmakingBucketMember member in m_lstMembers) + bool bInvalidatedSetup = false; + lock (m_StateLock) { - if (member.GetAssociatedSession() == null) + foreach (MatchmakingBucketMember member in m_lstMembers) { - m_lstMembers.Remove(member); + if (member.GetAssociatedSession() == null) + { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + bInvalidatedSetup = true; + } + + m_lstMembers.Remove(member); + } } } + + return bInvalidatedSetup; + } + + internal void MarkPendingDeletion() + { + lock (m_StateLock) + { + m_bPendingDeletion = true; + } } // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs @@ -658,8 +728,8 @@ public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joini public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) { - // stale bucket that was merged into another one - if (m_bMergedAway) + // stale buckets must never accept new members + if (IsMergedAway() || IsPendingDeletion()) { return false; } @@ -714,7 +784,7 @@ public int GetAvgElo() { SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); - if (memberUserData != null) + if (memberUserData?.GameStats != null) { avgElo += MatchmakingManager.GetMatchmakingElo(memberUserData.GameStats); ++numContributingMembers; @@ -790,10 +860,156 @@ public Int64 GetLobbyID() // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers private const Int64 c_LobbyJoinTimeoutMSec = 45000; + private const int c_GameStartCountdownMSec = 5000; + private const int c_SetupClientTimeoutMarginMSec = 2000; + + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) + { + Console.WriteLine("START GAME"); + + WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); + startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(bytesJSON); + } + } + + await lobby.UpdateState(ELobbyState.INGAME); + MatchmakingManager.DestroyBucket(this); + } + + private void QueueSetupProgress(int timeoutMSec) + { + WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); + setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; + setupProgress.timeout_ms = timeoutMSec; + byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(setupProgressJSON); + } + } + } + + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) + { + lobby.StartFullMeshConnectivityCheck(); + lock (m_StateLock) + { + // Publish the state transition before any notification work. If a send fails, the + // regular lobby timeout still completes or aborts the setup instead of stranding it. + m_bWaitingOnMeshConnectivityChecks = true; + } + + QueueSetupProgress(Lobby.MaxFullMeshConnectivityCheckDurationMS + c_SetupClientTimeoutMarginMSec); + + lobby.SendFullMeshConnectivityCheckRequestToMembers(); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Preparing match..."); + } + } + } + + private async Task AbortQuickMatchAutoStart(string reason) + { + List sessionsToRequeue = new(); + lock (m_StateLock) + { + if (m_bAbortInProgress || m_bStartCommitted) + { + return; + } + + m_bAbortInProgress = true; + m_bPendingDeletion = true; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } + } + + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + } + + LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); + Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); + if (quickMatchLobby != null) + { + foreach (UserSession memberSession in sessionsToRequeue) + { + LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); + if (lobbyMember != null) + { + // TODO: Remove this fallback once all supported clients handle MATCHMAKING_ACTION_REQUEUE. + // Legacy clients do not understand the requeue action, but they can still tear down + // peer and anti-cheat connections before joining the next temporary lobby. + quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); + await quickMatchLobby.RemoveMember(lobbyMember); + } + } + + await lobbyManager.DeleteLobby(quickMatchLobby); + } + + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); + requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; + byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); + + foreach (UserSession memberSession in sessionsToRequeue) + { + if (await TryRequeueRegisteredPlayer(memberSession, requeueActionJSON)) + { + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); + } + } + + m_lstMembers.Clear(); + m_LobbyID = -1; + + MatchmakingManager.DestroyBucket(this); + } + public async Task Tick() { - // merged into another bucket - our members live there now, ticking would create a second lobby for them - if (m_bMergedAway) + bool bPendingDeletion; + bool bAutoStartInvalidated; + bool bWaitingOnMeshConnectivityChecks; + bool bWaitingOnLobbyJoins; + bool bHasStartedCountdown; + lock (m_StateLock) + { + bPendingDeletion = m_bPendingDeletion; + bAutoStartInvalidated = m_bAutoStartInvalidated; + bWaitingOnMeshConnectivityChecks = m_bWaitingOnMeshConnectivityChecks; + bWaitingOnLobbyJoins = m_bWaitingOnLobbyJoins; + bHasStartedCountdown = m_bHasStartedCountdown; + } + + // merged/deleted buckets must not create another lobby or continue a committed setup + if (IsMergedAway() || bPendingDeletion) { return; } @@ -802,18 +1018,137 @@ public async Task Tick() // drop any members whose session has gone away, otherwise they are counted forever and the bucket can // never reach the "everyone is in the lobby" condition - PruneDeadMembers(); + if (PruneDeadMembers()) + { + bAutoStartInvalidated = true; + } // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) { // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + if (CurrentMemberCount() == 0 && !bWaitingOnLobbyJoins && !bHasStartedCountdown) { MatchmakingManager.DestroyBucket(this); return; } + if (bAutoStartInvalidated) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + return; + } + + if (bWaitingOnMeshConnectivityChecks) + { + Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); + if (lobbyDuringMeshCheck == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); + + if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) + { + bool bStartCountdown; + bool bAbortStart; + bool bInvalidatedAtDecision; + lock (m_StateLock) + { + bInvalidatedAtDecision = m_bAutoStartInvalidated; + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartCountdown = false; + bAbortStart = false; + } + else + { + m_bWaitingOnMeshConnectivityChecks = false; + bStartCountdown = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; + bAbortStart = !bStartCountdown; + if (bStartCountdown) + { + m_StartTime = Environment.TickCount64 + c_GameStartCountdownMSec; + m_bHasStartedCountdown = true; + } + } + } + + if (bStartCountdown) + { + QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec); + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, $"Starting game in {c_GameStartCountdownMSec / 1000} seconds."); + } + } + } + else if (bAbortStart) + { + string reason = bInvalidatedAtDecision + ? "QuickMatch auto-start was aborted because a player left during match setup." + : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; + await AbortQuickMatchAutoStart(reason); + } + } + + return; + } + + if (bHasStartedCountdown) + { + if (Environment.TickCount64 < m_StartTime) + { + return; + } + + Lobby? lobbyAfterCountdown = lobbyManager.GetLobby(m_LobbyID); + if (lobbyAfterCountdown == null) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + bool bStartGame; + bool bAbortStart; + lock (m_StateLock) + { + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartGame = false; + bAbortStart = false; + } + else + { + m_bHasStartedCountdown = false; + m_StartTime = -1; + bStartGame = !m_bAutoStartInvalidated; + bAbortStart = !bStartGame; + if (bStartGame) + { + m_bStartCommitted = true; + m_bPendingDeletion = true; + } + } + } + + if (bStartGame) + { + await StartGameAfterSuccessfulMeshCheck(lobbyAfterCountdown); + } + else if (bAbortStart) + { + await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + } + + return; + } + // do we need to start? // TODO_MATCHMAKING: Add a timeout at which > min players starts if (!m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) @@ -880,8 +1215,11 @@ await SendMatchmakingMessage(memberSession, m_bReachedMinPlayers = false; m_timeReachedMinPlayers = -1; - m_bWaitingOnLobbyJoins = true; - m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + lock (m_StateLock) + { + m_bWaitingOnLobbyJoins = true; + m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + } // tell everyone UserSession? dummyHostUser = null; @@ -1047,20 +1385,11 @@ await SendMatchmakingMessage(memberSession, { m_timeStartedWaitingOnLobbyJoins = -1; - // wait 5 sec - m_StartTime = Environment.TickCount64 + 5000; - foreach (MatchmakingBucketMember member in m_lstMembers) + lock (m_StateLock) { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, $"Starting Game in 5 seconds"); - } + m_bWaitingOnLobbyJoins = false; } - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = true; - // finalize the teams const int playlistMaxPlayerPerTeam = 2; bool bIsFFA = true; @@ -1085,46 +1414,13 @@ await SendMatchmakingMessage(memberSession, } } } + + await TriggerFullMeshConnectivityChecks(lobby); } } } - // TODO_QUICKMATCH: Do full mesh connectivity check + handle not being connected - // do we have a countdown? - if (m_StartTime != -1) - { - if (Environment.TickCount64 >= m_StartTime) - { - m_StartTime = -1; - - Console.WriteLine("START GAME"); - - // send start - WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); - startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(bytesJSON); - } - } - - // start match + create placeholder match - Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby != null) - { - await lobby.UpdateState(ELobbyState.INGAME); - } - - // destroy the bucket - MatchmakingManager.DestroyBucket(this); - } - } } } } @@ -1311,10 +1607,14 @@ public static async Task Tick() } // queue for deletion - m_lstBucketsPendingDeletion.AddRange(lstBucketsMergedNeedingDeleted); + foreach (MatchmakingBucket bucket in lstBucketsMergedNeedingDeleted) + { + m_bucketsPendingDeletion.Enqueue(bucket); + } - // cleanup any pending destruction (cannot do this in tick, collection will be modified) - foreach (MatchmakingBucket bucket in m_lstBucketsPendingDeletion) + // Drain the queue rather than enumerating and clearing a shared list. A concurrent cancellation can + // enqueue a bucket while cleanup is running, and clearing the list would otherwise lose that request. + while (m_bucketsPendingDeletion.TryDequeue(out MatchmakingBucket? bucket)) { if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) { @@ -1323,7 +1623,6 @@ public static async Task Tick() m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); } } - m_lstBucketsPendingDeletion.Clear(); List> lstDestroy = new(); foreach (WeakReference wrSession in lstSessions) @@ -1334,16 +1633,36 @@ public static async Task Tick() } else { - SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); - - if (thisSessionUserData == null) + await thisSession.MatchmakingStateLock.WaitAsync(); + try { - lstDestroy.Add(wrSession); - } - else - { - if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + // A cancellation can remove the weak reference while this tick is iterating a snapshot. + // Re-check registration while holding the per-session gate before assigning any bucket. + if (!thisSession.IsRegisteredForMatchmaking || !IsPendingSession(thisSession)) { + lstDestroy.Add(wrSession); + continue; + } + + SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); + + if (thisSessionUserData == null) + { + lstDestroy.Add(wrSession); + } + else + { + PlayerStats? thisSessionStats = thisSessionUserData.GameStats; + if (thisSessionStats == null) + { + thisSession.IsRegisteredForMatchmaking = false; + lstDestroy.Add(wrSession); + await SendMatchmakingMessage(thisSession, "Matchmaking could not start because your player statistics are unavailable. Please try again."); + continue; + } + + if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + { // TODO_MATCHAMAKING: Better way of tracking this, we need to know who is already in a bucket // Was the user in a bucket? if so theres nothing to do in terms of bucket management @@ -1427,12 +1746,17 @@ public static async Task Tick() lstDestroy.Add(wrSession); } } + } + else + { + // invalid playlist somehow + lstDestroy.Add(wrSession); + } } - else - { - // invalid playlist somehow - lstDestroy.Add(wrSession); - } + } + finally + { + thisSession.MatchmakingStateLock.Release(); } } } @@ -1450,10 +1774,60 @@ public static async Task Tick() // TODO_MATCHMAKING: Deregister player if they disconnect or leave quickmatch private static ConcurrentList> lstSessions = new(); - private static ConcurrentList m_lstBucketsPendingDeletion = new(); + private static bool IsPendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && ReferenceEquals(pendingSession, session)) + { + return true; + } + } + + return false; + } + + private static void RemovePendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions.ToList()) + { + if (!wrSession.TryGetTarget(out UserSession? pendingSession) || ReferenceEquals(pendingSession, session)) + { + lstSessions.Remove(wrSession); + } + } + } + + private static async Task TryRequeueRegisteredPlayer(UserSession session, byte[] requeueActionJSON) + { + await session.MatchmakingStateLock.WaitAsync(); + try + { + if (!session.IsRegisteredForMatchmaking) + { + return false; + } + + if (!IsPendingSession(session)) + { + lstSessions.Add(new WeakReference(session)); + } + + session.UpdateSessionLobbyID(-1); + session.QueueWebsocketSend(requeueActionJSON); + return true; + } + finally + { + session.MatchmakingStateLock.Release(); + } + } + + private static ConcurrentQueue m_bucketsPendingDeletion = new(); public static void DestroyBucket(MatchmakingBucket bucket) { - m_lstBucketsPendingDeletion.Add(bucket); + bucket.MarkPendingDeletion(); + m_bucketsPendingDeletion.Enqueue(bucket); } public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) @@ -1477,43 +1851,54 @@ public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List return; } - // make sure a re-register (or a duplicate request) cannot leave the player queued twice, or queued while - // still sat in an existing bucket - either would matchmake them into two lobbies at once - RemoveSessionFromPendingList(plr); - RemovePlayerFromAllBuckets(plr); - - plr.MatchmakingPlaylistID = playlistID; - plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); - plr.ExeCRC = exe_crc; - plr.IniCRC = ini_crc; - plr.AnticheatID = anticheatID; - lstSessions.Add(new WeakReference(plr)); - - await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); - } - - // NOTE: WeakReference does not implement value equality, so entries must be matched by their target session - private static void RemoveSessionFromPendingList(UserSession plr) - { - foreach (WeakReference wrSession in lstSessions) + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try { - if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null || thisSession == plr) + // A duplicate registration must not leave the player queued twice or in two buckets. + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); + + if (!bCancellationRejected) { - lstSessions.Remove(wrSession); + plr.MatchmakingPlaylistID = playlistID; + plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); + plr.ExeCRC = exe_crc; + plr.IniCRC = ini_crc; + plr.AnticheatID = anticheatID; + plr.IsRegisteredForMatchmaking = true; + lstSessions.Add(new WeakReference(plr)); } } + finally + { + plr.MatchmakingStateLock.Release(); + } + + if (bCancellationRejected) + { + await SendMatchmakingMessage(plr, "Matchmaking cannot be restarted because your game is already starting."); + return; + } + + await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); } - private static void RemovePlayerFromAllBuckets(UserSession plr) + private static async Task RemovePlayerFromAllBuckets(UserSession plr) { var lobbyManager = ServiceLocator.Services.GetRequiredService(); + bool bCancellationRejected = false; // also remove from any bucket we are in to avoid ghost buckets foreach (var kvPair in m_dictMatchmakingBuckets) { foreach (MatchmakingBucket mmBucket in kvPair.Value) { - if (mmBucket.HasPlayer(plr)) + bool bRemoved = mmBucket.RemovePlayer(plr, out bool bBucketCancellationRejected); + bCancellationRejected |= bBucketCancellationRejected; + + if (bRemoved) { // remove from QM lobby too Lobby? lobby = lobbyManager.GetLobby(mmBucket.GetLobbyID()); @@ -1523,13 +1908,10 @@ private static void RemovePlayerFromAllBuckets(UserSession plr) if (lobbyMember != null) { Console.WriteLine("User {0} Leave MM Lobby", plr.m_UserID); - lobby.RemoveMember(lobbyMember); + await lobby.RemoveMember(lobbyMember); } } - // remove player - mmBucket.RemovePlayer(plr); - // if we're the last player, destroy the bucket if (mmBucket.CurrentMemberCount() == 0) { @@ -1538,19 +1920,31 @@ private static void RemovePlayerFromAllBuckets(UserSession plr) } } } + + return bCancellationRejected; } - public static void DeregisterPlayer(UserSession plr) + public static async Task DeregisterPlayer(UserSession plr) { var lobbyManager = ServiceLocator.Services.GetRequiredService(); - RemoveSessionFromPendingList(plr); - - // TODO_QUICKMATCH: What happens if the game is going to start? we should handle that, right now people probably goto game solo - - RemovePlayerFromAllBuckets(plr); + bool bCancellationRejected; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + bCancellationRejected = await RemovePlayerFromAllBuckets(plr); + } + finally + { + plr.MatchmakingStateLock.Release(); + } // leave QM lobby too - Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); - lobbyManager.LeaveAnyLobby(plr.m_UserID); + if (!bCancellationRejected) + { + Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); + await lobbyManager.LeaveAnyLobby(plr.m_UserID); + } } -} \ No newline at end of file +}