Orchestration relaunch resilience: persist dispatch state, resume workers, fix watchdog (#207)

PureWeen · Copilot · web-flow · commit 7088a9b411d5 · 2026-02-25T08:54:45.000-06:00
## Summary

Makes multi-agent orchestration dispatch resilient to app relaunches.
When the app is relaunched while workers are processing, the
orchestrator can now automatically resume and collect results.

### Root Cause
When the app relaunches mid-dispatch, the in-memory `Task.WhenAll`
awaiting worker TCS completions dies with the old process. Workers
continue on the backend but their results are never collected for
synthesis.

### Changes

**Relaunch Resilience** (`CopilotService.Organization.cs`)
- `PendingOrchestration` model persisted to
`~/.polypilot/pending-orchestration.json` before dispatching workers
- `ResumeOrchestrationIfPendingAsync` detects pending orchestrations on
restart and monitors workers
- `MonitorAndSynthesizeAsync` polls every 5s (15min timeout) until all
workers idle, then auto-synthesizes
- Dispatch state cleared in `finally` blocks to prevent leaked files on
cancellation/error
- Worker results filtered by dispatch timestamp to avoid picking up
stale pre-dispatch messages
- UTC→local time conversion for reliable timestamp comparison

**Watchdog Fix** (`CopilotService.cs`)
- `IsMultiAgentSession` carried forward on session reconnect — without
this, the watchdog used the 120s inactivity timeout instead of 600s,
killing long-running workers prematurely
- `[DISPATCH]` tag added to diagnostic log file filter for post-mortem
analysis

**Tests** (`MultiAgentRegressionTests.cs`)
- 7 new tests: save/load/clear lifecycle, no-file resume, missing-group
cleanup, dispatch tag guard, reconnect state guard, timestamp filter
guard, finally block guard

### Multi-Model Review
Reviewed by 3 models (Opus 4.6, Sonnet 4.6, GPT-5.3-Codex). All 3
consensus findings fixed:
1. ✅ Worker result collection picks up pre-dispatch history → timestamp
filter added
2. ✅ UTC/local time mismatch → converted on comparison
3. ✅ Pending file leaked on cancellation → try/finally added

### Testing
- 1268 tests total, 1267 pass (1 pre-existing flaky
`TestIsolationGuard`)
- Manually tested orchestration resume after app relaunch

---------

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/PolyPilot.Tests/BaseDirCollection.cs b/PolyPilot.Tests/BaseDirCollection.cs
@@ -0,0 +1,14 @@
+using Xunit;
+
+namespace PolyPilot.Tests;
+
+/// <summary>
+/// xUnit collection that serializes all test classes that read or mutate
+/// CopilotService.BaseDir (via SetBaseDirForTesting). Without this,
+/// parallel test classes race on the shared static _polyPilotBaseDir field,
+/// causing flaky failures in TestIsolationGuardTests.
+/// </summary>
+[CollectionDefinition("BaseDir")]
+public class BaseDirCollection : ICollectionFixture<BaseDirCollectionFixture> { }
+
+public class BaseDirCollectionFixture { }
diff --git a/PolyPilot.Tests/MultiAgentRegressionTests.cs b/PolyPilot.Tests/MultiAgentRegressionTests.cs
@@ -17,6 +17,7 @@ namespace PolyPilot.Tests;
 /// 5. Mode enum gaps: OrchestratorReflect missing from dropdowns and serialization
 /// 6. Reflection loop error handling: unhandled exceptions kill the async task silently
 /// </summary>
+[Collection("BaseDir")]
 public class MultiAgentRegressionTests
 {
     private readonly StubChatDatabase _chatDb = new();
@@ -1352,4 +1353,220 @@ public void ReconcileOrganization_PreInit_ZeroSessions_StillProtected()
     }
 
     #endregion
+
+    #region Orchestration Persistence (relaunch resilience)
+
+    [Fact]
+    public void PendingOrchestration_SaveLoadClear_FullLifecycle()
+    {
+        // Use a dedicated subdirectory to avoid races with parallel tests
+        var testDir = Path.Combine(TestSetup.TestBaseDir, "pending-orch-test-" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(testDir);
+        CopilotService.SetBaseDirForTesting(testDir);
+        try
+        {
+            var svc = CreateService();
+
+            // Save
+            var pending = new PendingOrchestration
+            {
+                GroupId = "test-group-id",
+                OrchestratorName = "test-orchestrator",
+                WorkerNames = new List<string> { "worker-1", "worker-2", "worker-3" },
+                OriginalPrompt = "Review the code",
+                StartedAt = new DateTime(2026, 2, 24, 15, 0, 0, DateTimeKind.Utc),
+                IsReflect = true,
+                ReflectIteration = 2
+            };
+            svc.SavePendingOrchestration(pending);
+
+            // Load and verify round-trip
+            var loaded = CopilotService.LoadPendingOrchestrationForTest();
+            Assert.NotNull(loaded);
+            Assert.Equal("test-group-id", loaded.GroupId);
+            Assert.Equal("test-orchestrator", loaded.OrchestratorName);
+            Assert.Equal(3, loaded.WorkerNames.Count);
+            Assert.Contains("worker-2", loaded.WorkerNames);
+            Assert.Equal("Review the code", loaded.OriginalPrompt);
+            Assert.True(loaded.IsReflect);
+            Assert.Equal(2, loaded.ReflectIteration);
+
+            // Clear and verify deletion
+            CopilotService.ClearPendingOrchestrationForTest();
+            Assert.Null(CopilotService.LoadPendingOrchestrationForTest());
+        }
+        finally
+        {
+            // Restore shared test dir
+            CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+        }
+    }
+
+    [Fact]
+    public async Task ResumeOrchestration_NoFile_DoesNothing()
+    {
+        CopilotService.ClearPendingOrchestrationForTest();
+
+        var svc = CreateService();
+        // Should complete without error and not add any messages
+        await svc.ResumeOrchestrationIfPendingAsync();
+    }
+
+    [Fact]
+    public async Task ResumeOrchestration_MissingGroup_ClearsState()
+    {
+        var testDir = Path.Combine(TestSetup.TestBaseDir, "pending-orch-resume-" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(testDir);
+        CopilotService.SetBaseDirForTesting(testDir);
+        try
+        {
+            var svc = CreateService();
+            svc.SavePendingOrchestration(new PendingOrchestration
+            {
+                GroupId = "nonexistent-group",
+                OrchestratorName = "orch",
+                WorkerNames = new() { "w1" },
+                OriginalPrompt = "test",
+                StartedAt = DateTime.UtcNow
+            });
+
+            await svc.ResumeOrchestrationIfPendingAsync();
+
+            // Should have cleared the pending file since group doesn't exist
+            Assert.Null(CopilotService.LoadPendingOrchestrationForTest());
+        }
+        finally
+        {
+            CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+        }
+    }
+
+    [Fact]
+    public void DiagnosticLogFilter_IncludesDispatchTag()
+    {
+        // The Debug() method's file filter must include [DISPATCH] so orchestration
+        // events are written to event-diagnostics.log for post-mortem analysis.
+        // This was a bug: [DISPATCH] was written to Console but not persisted.
+        var source = File.ReadAllText(Path.Combine(GetRepoRoot(), "PolyPilot", "Services", "CopilotService.cs"));
+        Assert.Contains("[DISPATCH", source.Substring(source.IndexOf("message.StartsWith(\"[EVT")));
+    }
+
+    [Fact]
+    public void ReconnectState_ShouldCarryIsMultiAgentSession()
+    {
+        // After reconnect in SendPromptAsync, the new SessionState must carry forward
+        // IsMultiAgentSession from the old state. Without this, the watchdog uses the
+        // 120s inactivity timeout instead of 600s, killing long-running worker tasks.
+        var source = File.ReadAllText(Path.Combine(GetRepoRoot(), "PolyPilot", "Services", "CopilotService.cs"));
+        
+        // Find the reconnect block where HasUsedToolsThisTurn is carried forward
+        var reconnectBlock = source.Substring(source.IndexOf("newState.HasUsedToolsThisTurn = state.HasUsedToolsThisTurn"));
+        // IsMultiAgentSession must be carried forward in the same block
+        Assert.Contains("newState.IsMultiAgentSession = state.IsMultiAgentSession", reconnectBlock.Substring(0, 200));
+    }
+
+    [Fact]
+    public void MonitorAndSynthesize_ShouldFilterByDispatchTimestamp()
+    {
+        // MonitorAndSynthesizeAsync must filter worker results by dispatch timestamp
+        // to avoid picking up stale pre-dispatch assistant messages from prior conversations.
+        // This was a 3/3 consensus finding from multi-model review.
+        var source = File.ReadAllText(Path.Combine(GetRepoRoot(), "PolyPilot", "Services", "CopilotService.Organization.cs"));
+
+        // Find the result collection section in MonitorAndSynthesizeAsync
+        var monitorSection = source.Substring(source.IndexOf("Collect worker results from their chat history"));
+        var sectionEnd = Math.Min(monitorSection.Length, 1500);
+        var block = monitorSection.Substring(0, sectionEnd);
+        // Must convert StartedAt to local time for comparison with ChatMessage.Timestamp
+        Assert.Contains("dispatchTimeLocal", block);
+        // Must filter by timestamp
+        Assert.Contains("Timestamp >= dispatchTimeLocal", block);
+    }
+
+    [Fact]
+    public void PendingOrchestration_ShouldClearInFinallyBlock()
+    {
+        // ClearPendingOrchestration must be in a finally block so it's cleaned up
+        // even on cancellation/error. Otherwise stale pending files cause spurious
+        // resume on next launch. Opus review finding.
+        var source = File.ReadAllText(Path.Combine(GetRepoRoot(), "PolyPilot", "Services", "CopilotService.Organization.cs"));
+
+        // Non-reflect path: must have finally { ClearPendingOrchestration }
+        var nonReflectDispatch = source.Substring(source.IndexOf("Phase 3: Dispatch tasks to workers"));
+        var nextMethod = nonReflectDispatch.IndexOf("private string Build");
+        var dispatchBlock = nonReflectDispatch.Substring(0, nextMethod);
+        Assert.Contains("finally", dispatchBlock);
+        Assert.Contains("ClearPendingOrchestration", dispatchBlock);
+    }
+
+    private static string GetRepoRoot()
+    {
+        var dir = AppContext.BaseDirectory;
+        while (dir != null && !File.Exists(Path.Combine(dir, "PolyPilot.slnx")))
+            dir = Path.GetDirectoryName(dir);
+        return dir ?? throw new InvalidOperationException("Could not find repo root");
+    }
+
+    #endregion
+
+    #region GetOrchestratorGroupId
+
+    [Fact]
+    public void GetOrchestratorGroupId_ReturnsGroupId_ForOrchestratorSession()
+    {
+        // This tests the fix for the queue-drain dispatch bypass bug:
+        // When the orchestrator session was processing and a user sent a message,
+        // it was queued. On dequeue, it bypassed the multi-agent routing and went
+        // directly to SendPromptAsync instead of SendToMultiAgentGroupAsync.
+        var svc = CreateService();
+        CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+
+        var group = svc.CreateMultiAgentGroup("DispatchTest", MultiAgentMode.Orchestrator);
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "orch", GroupId = group.Id, Role = MultiAgentRole.Orchestrator });
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "worker", GroupId = group.Id, Role = MultiAgentRole.Worker });
+
+        var result = svc.GetOrchestratorGroupId("orch");
+        Assert.Equal(group.Id, result);
+    }
+
+    [Fact]
+    public void GetOrchestratorGroupId_ReturnsNull_ForWorkerSession()
+    {
+        var svc = CreateService();
+        CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+
+        var group = svc.CreateMultiAgentGroup("DispatchTest2", MultiAgentMode.Orchestrator);
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "orch2", GroupId = group.Id, Role = MultiAgentRole.Orchestrator });
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "worker2", GroupId = group.Id, Role = MultiAgentRole.Worker });
+
+        var result = svc.GetOrchestratorGroupId("worker2");
+        Assert.Null(result);
+    }
+
+    [Fact]
+    public void GetOrchestratorGroupId_ReturnsNull_ForNonGroupSession()
+    {
+        var svc = CreateService();
+        CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+
+        var result = svc.GetOrchestratorGroupId("nonexistent-session");
+        Assert.Null(result);
+    }
+
+    [Fact]
+    public void GetOrchestratorGroupId_ReturnsNull_ForBroadcastMode()
+    {
+        var svc = CreateService();
+        CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
+
+        var group = svc.CreateMultiAgentGroup("BroadcastTest", MultiAgentMode.Broadcast);
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "b1", GroupId = group.Id });
+        svc.Organization.Sessions.Add(new SessionMeta { SessionName = "b2", GroupId = group.Id });
+
+        // Broadcast mode has no orchestrator — should return null for all members
+        Assert.Null(svc.GetOrchestratorGroupId("b1"));
+        Assert.Null(svc.GetOrchestratorGroupId("b2"));
+    }
+
+    #endregion
 }
diff --git a/PolyPilot.Tests/TestIsolationGuardTests.cs b/PolyPilot.Tests/TestIsolationGuardTests.cs
@@ -16,6 +16,7 @@ namespace PolyPilot.Tests;
 /// settings. This has caused production data loss (squad groups destroyed)
 /// multiple times before the guard was added.
 /// </summary>
+[Collection("BaseDir")]
 public class TestIsolationGuardTests
 {
     [Fact]
diff --git a/PolyPilot.Tests/UsageStatsTests.cs b/PolyPilot.Tests/UsageStatsTests.cs
@@ -4,6 +4,7 @@
 
 namespace PolyPilot.Tests;
 
+[Collection("BaseDir")]
 public class UsageStatsTests : IDisposable
 {
     private readonly string _testDir;
@@ -25,10 +26,8 @@ private void ResetStaticFields()
             BindingFlags.NonPublic | BindingFlags.Static);
         statsPathField?.SetValue(null, null);
         
-        // Override CopilotService.BaseDir
-        var baseDirField = typeof(CopilotService).GetField("_polyPilotBaseDir", 
-            BindingFlags.NonPublic | BindingFlags.Static);
-        baseDirField?.SetValue(null, _testDir);
+        // Override CopilotService.BaseDir via the proper API
+        CopilotService.SetBaseDirForTesting(_testDir);
     }
 
     private UsageStatsService CreateService()
@@ -58,9 +57,8 @@ public void Dispose()
             BindingFlags.NonPublic | BindingFlags.Static);
         statsPathField?.SetValue(null, null);
         
-        var baseDirField = typeof(CopilotService).GetField("_polyPilotBaseDir", 
-            BindingFlags.NonPublic | BindingFlags.Static);
-        baseDirField?.SetValue(null, null);
+        // Restore to the shared test base dir (never null it — causes races)
+        CopilotService.SetBaseDirForTesting(TestSetup.TestBaseDir);
     }
 
     [Fact]
diff --git a/PolyPilot/Components/DiffView.razor b/PolyPilot/Components/DiffView.razor
@@ -69,4 +69,5 @@ else
     {
         Files = DiffParser.Parse(RawDiff);
     }
+
 }
diff --git a/PolyPilot/Components/Pages/Dashboard.razor b/PolyPilot/Components/Pages/Dashboard.razor
@@ -1209,6 +1209,7 @@
         var session = sessions.FirstOrDefault(s => s.Name == sessionName);
         if (session?.IsProcessing == true)
         {
+            CopilotService.LogDispatchRoute(sessionName, true, "QUEUED", null, null, null, false);
             List<string>? queueImagePaths = null;
             if (hasImages)
             {
@@ -1250,19 +1251,24 @@
             var group = sessionMeta?.GroupId != null
                 ? CopilotService.Organization.Groups.FirstOrDefault(g => g.Id == sessionMeta.GroupId)
                 : null;
+            var orchSession = group != null ? CopilotService.GetOrchestratorSession(group.Id) : null;
             var isOrchestrator = group is { IsMultiAgent: true }
                 && (group.OrchestratorMode == MultiAgentMode.Orchestrator || group.OrchestratorMode == MultiAgentMode.OrchestratorReflect)
-                && CopilotService.GetOrchestratorSession(group.Id) == sessionName;
+                && orchSession == sessionName;
+
+            // Diagnostic: log every send attempt so we can trace routing failures
+            CopilotService.LogDispatchRoute(sessionName, sessionMeta != null, group?.Name, group?.IsMultiAgent, group?.OrchestratorMode, orchSession, isOrchestrator);
 
             if (isOrchestrator && (imagePaths == null || imagePaths.Count == 0))
             {
                 _ = CopilotService.SendToMultiAgentGroupAsync(group!.Id, finalPrompt).ContinueWith(t =>
                 {
                     if (t.IsFaulted)
                     {
+                        var msg = t.Exception?.InnerException?.Message ?? t.Exception?.Message ?? "unknown";
                         InvokeAsync(() =>
                         {
-                            Console.WriteLine($"Error sending to multi-agent group: {t.Exception?.InnerException?.Message}");
+                            Console.WriteLine($"[DISPATCH] Error sending to multi-agent group: {msg}");
                         });
                     }
                 });
diff --git a/PolyPilot/Services/CopilotService.Events.cs b/PolyPilot/Services/CopilotService.Events.cs
@@ -824,6 +824,10 @@ private void CompleteResponse(SessionState state, long? expectedGeneration = nul
             var skipHistory = state.Info.ReflectionCycle is { IsActive: true } &&
                               ReflectionCycle.IsReflectionFollowUpPrompt(nextPrompt);
 
+            // Check if the dequeued message is for an orchestrator session — if so,
+            // route through the multi-agent dispatch pipeline instead of direct send.
+            var orchGroupId = GetOrchestratorGroupId(state.Info.Name);
+
             // Use Task.Run to dispatch on a clean stack frame, avoiding reentrancy
             // issues where CompleteResponse hasn't fully unwound yet.
             _ = Task.Run(async () =>
@@ -839,7 +843,15 @@ private void CompleteResponse(SessionState state, long? expectedGeneration = nul
                         {
                             try
                             {
-                                await SendPromptAsync(state.Info.Name, nextPrompt, imagePaths: nextImagePaths, skipHistoryMessage: skipHistory, agentMode: nextAgentMode);
+                                if (orchGroupId != null && nextImagePaths is null or { Count: 0 })
+                                {
+                                    Debug($"[DISPATCH] Queue drain routing to multi-agent pipeline: session='{state.Info.Name}', group='{orchGroupId}'");
+                                    await SendToMultiAgentGroupAsync(orchGroupId, nextPrompt);
+                                }
+                                else
+                                {
+                                    await SendPromptAsync(state.Info.Name, nextPrompt, imagePaths: nextImagePaths, skipHistoryMessage: skipHistory, agentMode: nextAgentMode);
+                                }
                                 tcs.TrySetResult();
                             }
                             catch (Exception ex)
@@ -851,7 +863,15 @@ private void CompleteResponse(SessionState state, long? expectedGeneration = nul
                     }
                     else
                     {
-                        await SendPromptAsync(state.Info.Name, nextPrompt, imagePaths: nextImagePaths, skipHistoryMessage: skipHistory, agentMode: nextAgentMode);
+                        if (orchGroupId != null && nextImagePaths is null or { Count: 0 })
+                        {
+                            Debug($"[DISPATCH] Queue drain routing to multi-agent pipeline: session='{state.Info.Name}', group='{orchGroupId}'");
+                            await SendToMultiAgentGroupAsync(orchGroupId, nextPrompt);
+                        }
+                        else
+                        {
+                            await SendPromptAsync(state.Info.Name, nextPrompt, imagePaths: nextImagePaths, skipHistoryMessage: skipHistory, agentMode: nextAgentMode);
+                        }
                     }
                 }
                 catch (Exception ex)
diff --git a/PolyPilot/Services/CopilotService.Organization.cs b/PolyPilot/Services/CopilotService.Organization.cs
diff --git a/PolyPilot/Services/CopilotService.cs b/PolyPilot/Services/CopilotService.cs

Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,7 @@ namespace PolyPilot.Tests;`
`16`	`16`	`/// settings. This has caused production data loss (squad groups destroyed)`
`17`	`17`	`/// multiple times before the guard was added.`
`18`	`18`	`/// </summary>`
	`19`	`+[Collection("BaseDir")]`
`19`	`20`	`public class TestIsolationGuardTests`
`20`	`21`	`{`
`21`	`22`	`[Fact]`
Original file line number	Diff line number	Diff line change
`@@ -69,4 +69,5 @@ else`
`69`	`69`	`{`
`70`	`70`	`Files = DiffParser.Parse(RawDiff);`
`71`	`71`	`}`
	`72`	`+`
`72`	`73`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1209,6 +1209,7 @@`
`1209`	`1209`	`var session = sessions.FirstOrDefault(s => s.Name == sessionName);`
`1210`	`1210`	`if (session?.IsProcessing == true)`
`1211`	`1211`	`{`
	`1212`	`+ CopilotService.LogDispatchRoute(sessionName, true, "QUEUED", null, null, null, false);`
`1212`	`1213`	`List<string>? queueImagePaths = null;`
`1213`	`1214`	`if (hasImages)`
`1214`	`1215`	`{`
`@@ -1250,19 +1251,24 @@`
`1250`	`1251`	`var group = sessionMeta?.GroupId != null`
`1251`	`1252`	`? CopilotService.Organization.Groups.FirstOrDefault(g => g.Id == sessionMeta.GroupId)`
`1252`	`1253`	`: null;`
	`1254`	`+ var orchSession = group != null ? CopilotService.GetOrchestratorSession(group.Id) : null;`
`1253`	`1255`	`var isOrchestrator = group is { IsMultiAgent: true }`
`1254`	`1256`	`&& (group.OrchestratorMode == MultiAgentMode.Orchestrator \|\| group.OrchestratorMode == MultiAgentMode.OrchestratorReflect)`
`1255`		`- && CopilotService.GetOrchestratorSession(group.Id) == sessionName;`
	`1257`	`+ && orchSession == sessionName;`
	`1258`	`+`
	`1259`	`+ // Diagnostic: log every send attempt so we can trace routing failures`
	`1260`	`+ CopilotService.LogDispatchRoute(sessionName, sessionMeta != null, group?.Name, group?.IsMultiAgent, group?.OrchestratorMode, orchSession, isOrchestrator);`
`1256`	`1261`
`1257`	`1262`	`if (isOrchestrator && (imagePaths == null \|\| imagePaths.Count == 0))`
`1258`	`1263`	`{`
`1259`	`1264`	`_ = CopilotService.SendToMultiAgentGroupAsync(group!.Id, finalPrompt).ContinueWith(t =>`
`1260`	`1265`	`{`
`1261`	`1266`	`if (t.IsFaulted)`
`1262`	`1267`	`{`
	`1268`	`+ var msg = t.Exception?.InnerException?.Message ?? t.Exception?.Message ?? "unknown";`
`1263`	`1269`	`InvokeAsync(() =>`
`1264`	`1270`	`{`
`1265`		`- Console.WriteLine($"Error sending to multi-agent group: {t.Exception?.InnerException?.Message}");`
	`1271`	`+ Console.WriteLine($"[DISPATCH] Error sending to multi-agent group: {msg}");`
`1266`	`1272`	`});`
`1267`	`1273`	`}`
`1268`	`1274`	`});`