feat: add engine health watchdog to monitor engine status and implement automatic reconnection logic

Author: AsieduDevelopmentHub

dashboard/lib/main.dart

@@ -31,7 +31,7 @@ class _StartupGateAppState extends State<StartupGateApp> {
     _boot = _bootstrap();
   }
 
-  Future<_BootResult> _bootstrap({bool userRetry = false}) async {
+  Future<_BootResult> _bootstrap() async {
     final notifications = DesktopNotificationService();
     await notifications.init(
       onDidReceiveNotificationResponse: (response) {
@@ -43,13 +43,9 @@ class _StartupGateAppState extends State<StartupGateApp> {
     );
     final settings = SettingsProvider();
     await settings.load();
-
-    // READY GATE (strict): do not enter the app until engine is healthy.
-    final out = userRetry
-        ? await EngineBundledLauncher.ensureReadyUserRetry()
-        : await EngineBundledLauncher.ensureReady();
-    return _BootResult(
-        settings: settings, notifications: notifications, gate: out);
+    // Engine startup is handled by EngineProvider (single owner) to avoid
+    // duplicate launches on app boot.
+    return _BootResult(settings: settings, notifications: notifications);
   }
 
   @override
@@ -69,21 +65,10 @@ class _StartupGateAppState extends State<StartupGateApp> {
         }
 
         final data = snap.data!;
-        if (!data.gate.success) {
-          return MaterialApp(
-            debugShowCheckedModeBanner: false,
-            theme: theme,
-            darkTheme: dark,
-            home: _StartupError(
-              message: data.gate.message ?? 'Engine failed to start.',
-              onRetry: () =>
-                  setState(() => _boot = _bootstrap(userRetry: true)),
-            ),
-          );
-        }
-
         return SentraCoreApp(
-            settings: data.settings, notifications: data.notifications);
+          settings: data.settings,
+          notifications: data.notifications,
+        );
       },
     );
   }
@@ -92,11 +77,9 @@ class _StartupGateAppState extends State<StartupGateApp> {
 class _BootResult {
   final SettingsProvider settings;
   final DesktopNotificationService notifications;
-  final EngineBootstrapOutcome gate;
   const _BootResult({
     required this.settings,
     required this.notifications,
-    required this.gate,
   });
 }
 
@@ -140,51 +123,10 @@ class _StartupSplash extends StatelessWidget {
   }
 }
 
-class _StartupError extends StatelessWidget {
-  final String message;
-  final VoidCallback onRetry;
-  const _StartupError({required this.message, required this.onRetry});
-
-  @override
-  Widget build(BuildContext context) {
-    return Scaffold(
-      body: Center(
-        child: Padding(
-          padding: const EdgeInsets.all(24),
-          child: ConstrainedBox(
-            constraints: const BoxConstraints(maxWidth: 520),
-            child: Column(
-              mainAxisSize: MainAxisSize.min,
-              crossAxisAlignment: CrossAxisAlignment.stretch,
-              children: [
-                Text(
-                  'Engine failed to start',
-                  style: TextStyle(
-                    color: AppTheme.textPrimaryFor(context),
-                    fontSize: 18,
-                    fontWeight: FontWeight.w800,
-                  ),
-                ),
-                const SizedBox(height: 8),
-                Text(
-                  message,
-                  style: TextStyle(color: AppTheme.textMutedFor(context)),
-                ),
-                const SizedBox(height: 16),
-                FilledButton(
-                  onPressed: onRetry,
-                  child: const Text('Retry'),
-                ),
-              ],
-            ),
-          ),
-        ),
-      ),
-    );
-  }
-}
+// Startup errors are handled inside the app (EngineProvider), so we no longer
+// gate app entry on engine health here.
 
-class SentraCoreApp extends StatelessWidget {
+class SentraCoreApp extends StatefulWidget {
   const SentraCoreApp({
     super.key,
     required this.settings,
@@ -194,19 +136,32 @@ class SentraCoreApp extends StatelessWidget {
   final SettingsProvider settings;
   final DesktopNotificationService notifications;
 
+  @override
+  State<SentraCoreApp> createState() => _SentraCoreAppState();
+}
+
+class _SentraCoreAppState extends State<SentraCoreApp> {
+  @override
+  void dispose() {
+    // Allow the app to stop the engine on close (owned process only).
+    // Users can still kill the process externally; we auto-restart on next launch.
+    EngineBundledLauncher.stopOwnedEngine();
+    super.dispose();
+  }
+
   @override
   Widget build(BuildContext context) {
     return MultiProvider(
       providers: [
-        Provider<DesktopNotificationService>.value(value: notifications),
-        ChangeNotifierProvider<SettingsProvider>.value(value: settings),
+        Provider<DesktopNotificationService>.value(value: widget.notifications),
+        ChangeNotifierProvider<SettingsProvider>.value(value: widget.settings),
         ChangeNotifierProvider<HistoryProvider>(
           create: (_) => HistoryProvider()..load(),
         ),
         ChangeNotifierProvider(
           create: (ctx) => EngineProvider(
-            settings: settings,
-            notifications: notifications,
+            settings: widget.settings,
+            notifications: widget.notifications,
             history: Provider.of<HistoryProvider>(ctx, listen: false),
           )..connect(),
         ),

dashboard/lib/providers/engine_provider.dart

@@ -112,6 +112,7 @@ class EngineProvider extends ChangeNotifier {
   Timer? _reconnectTimer;
   Timer? _cooldownTicker;
   Timer? _liveDataWatchdog;
+  Timer? _engineHealthWatchdog;
 
   /// Last successful [/ws/live] payload, or null until the first frame after subscribe.
   DateTime? _lastLiveStateAt;
@@ -153,6 +154,7 @@ class EngineProvider extends ChangeNotifier {
     _eventFetchTimer?.cancel();
     _reconnectTimer?.cancel();
     _cooldownTicker?.cancel();
+    _engineHealthWatchdog?.cancel();
     _service.dispose();
     final cfg = await EngineConfigStore.readOrCreate();
     _service = EngineService(host: cfg.host, port: cfg.port);
@@ -238,6 +240,14 @@ class EngineProvider extends ChangeNotifier {
         (_) => _fetchEvents(),
       );
 
+      // Health watchdog: if the engine is killed externally (Task Manager) or
+      // stops responding, attempt a bounded restart/reconnect.
+      _engineHealthWatchdog?.cancel();
+      _engineHealthWatchdog = Timer.periodic(
+        const Duration(seconds: 6),
+        (_) => unawaited(_engineHealthTick()),
+      );
+
       // Stay "disconnected" in UI until first live frame arrives (avoids false
       // positive if the socket dies immediately after subscribe).
       _connected = false;
@@ -262,6 +272,26 @@ class EngineProvider extends ChangeNotifier {
     }
   }
 
+  Future<void> _engineHealthTick() async {
+    // Avoid piling reconnects; reuse the same bootstrap + connect path.
+    if (_recoveringLive) return;
+    if (_bootstrapErrorPending) return;
+    if (_reconnectTimer != null) return;
+
+    try {
+      final j = await _service.getHealth();
+      if (j != null && j['engine'] == true) return;
+    } catch (_) {
+      // treat as unhealthy
+    }
+
+    // Engine is down or unhealthy: restart/reconnect.
+    _connected = false;
+    _connectionError = 'Engine stopped; restarting…';
+    notifyListeners();
+    _scheduleReconnect();
+  }
+
   void _checkLiveDataStale() {
     if (_liveSub == null || _recoveringLive) return;
     final reference = _lastLiveStateAt ?? _wsSubscribeAt;
@@ -459,14 +489,16 @@ class EngineProvider extends ChangeNotifier {
   @override
   void dispose() {
     _liveDataWatchdog?.cancel();
+    _engineHealthWatchdog?.cancel();
     _liveSub?.cancel();
     _alertSub?.cancel();
     _processFetchTimer?.cancel();
     _eventFetchTimer?.cancel();
     _reconnectTimer?.cancel();
     _cooldownTicker?.cancel();
     _service.dispose();
-    unawaited(EngineBundledLauncher.stopOwnedEngine());
+    // Do NOT kill the engine on app close. The engine is designed to be a
+    // background component and should survive dashboard restarts.
     super.dispose();
   }
 }

dashboard/lib/services/engine_bundled_launcher.dart

@@ -30,7 +30,7 @@ class EngineBundledLauncher {
   static bool _engineStartedByApp = false;
   static bool _busy = false;
 
-  static const Duration _healthWindow = Duration(seconds: 25);
+  static const Duration _healthWindow = Duration(seconds: 45);
   static const Duration _healthTick = Duration(milliseconds: 500);
   static const int _maxRestartCycles = 3;
 
@@ -223,6 +223,15 @@ class EngineBundledLauncher {
     _ownedProcess = null;
     _engineStartedByApp = false;
     if (p == null) return;
+    if (Platform.isWindows) {
+      try {
+        await Process.run(
+          'taskkill',
+          <String>['/PID', '${p.pid}', '/T', '/F'],
+          runInShell: true,
+        );
+      } catch (_) {}
+    }
     try {
       p.kill();
       await Future<void>.delayed(const Duration(milliseconds: 400));

dashboard/lib/services/engine_config_store.dart

@@ -21,8 +21,7 @@ class EngineConfigStore {
     if (Platform.isWindows) {
       final base = Platform.environment['LOCALAPPDATA'];
       if (base != null && base.trim().isNotEmpty) {
-        return Directory(
-            '${base.trim()}${Platform.pathSeparator}SentraCore');
+        return Directory('${base.trim()}${Platform.pathSeparator}SentraCore');
       }
     }
     if (Platform.isMacOS) {
@@ -35,8 +34,7 @@ class EngineConfigStore {
     // Linux / fallback
     final xdg = Platform.environment['XDG_CONFIG_HOME'];
     if (xdg != null && xdg.trim().isNotEmpty) {
-      return Directory(
-          '${xdg.trim()}${Platform.pathSeparator}sentracore');
+      return Directory('${xdg.trim()}${Platform.pathSeparator}sentracore');
     }
     final home = Platform.environment['HOME'] ?? '';
     if (home.trim().isNotEmpty) {

engine/engine_config.py

@@ -58,7 +58,13 @@ def engine_config_path() -> Path:
             return Path(local) / "SentraCore" / "engine-config.json"
     if system().lower() == "darwin":
         home = Path.home()
-        return home / "Library" / "Application Support" / "SentraCore" / "engine-config.json"
+        return (
+            home
+            / "Library"
+            / "Application Support"
+            / "SentraCore"
+            / "engine-config.json"
+        )
     xdg = os.environ.get("XDG_CONFIG_HOME")
     if xdg:
         return Path(xdg) / "sentracore" / "engine-config.json"

installer/sentracore.iss

@@ -32,16 +32,14 @@ Name: "{group}\Uninstall SentraCore"; Filename: "{uninstallexe}"
 
 [Tasks]
 Name: "desktopicon"; Description: "Create a &desktop shortcut"; GroupDescription: "Additional icons:"
-Name: "postinstallengine"; Description: "Start SentraCore Engine once after installation"; GroupDescription: "After installation:"
-Name: "startup"; Description: "Run SentraCore Engine automatically when Windows starts"; GroupDescription: "Background Service:"
 
 [Registry]
-; Add the Engine to Windows Startup if the user selected the task
-Root: HKCU; Subkey: "Software\Microsoft\Windows\CurrentVersion\Run"; ValueType: string; ValueName: "SentraCoreEngine"; ValueData: """{app}\SentraCoreEngine.exe"""; Tasks: startup
+; Always run engine on user login (background)
+Root: HKCU; Subkey: "Software\Microsoft\Windows\CurrentVersion\Run"; ValueType: string; ValueName: "SentraCoreEngine"; ValueData: """{app}\SentraCoreEngine.exe"""
 
 [Run]
-; Optional: first engine start (dashboard can still start the engine later if unchecked)
-Filename: "{app}\SentraCoreEngine.exe"; Description: "Start SentraCore Engine now"; Flags: nowait postinstall runhidden; Tasks: postinstallengine
+; Always start engine after install (background)
+Filename: "{app}\SentraCoreEngine.exe"; Flags: nowait postinstall runhidden
 ; Optionally launch the dashboard
 Filename: "{app}\sentracore_dashboard.exe"; Description: "Launch SentraCore Dashboard"; Flags: nowait postinstall