From eec5c94a2192f9b95eb15c42b7a9d6001d72077f Mon Sep 17 00:00:00 2001 From: GuoHao <897690986@qq.com> Date: Sun, 26 Jul 2026 04:40:59 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dserver=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E5=BF=83=E8=B7=B3=E6=A3=80=E6=B5=8B=E7=9A=84=E4=B8=A4?= =?UTF-8?q?=E4=B8=AA=E5=85=B3=E9=94=AE=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. LastHeartBeat 在每次健康检查后都更新，不再仅在Healthy时更新。 修复前：健康检查失败时 LastHeartBeat 冻结在注册时间，管理后台看起来像 心跳检测从未运行过，且 removeServiceInterval 可能因此误删服务。 2. 健康检查后台循环增加 try-catch 异常保护。 修复前：while 循环中任何未捕获异常（如数据库查询失败）都会静默杀死 整个健康检查任务，导致所有 server 模式服务永久失去心跳检测。 3. 优化： - CheckInterval 取值提前到循环外，防止 getter 抛异常杀循环 - CheckAService 新增成功日志和失败时的 HTTP 状态码 - Task.Run 内的健康检查任务增加独立异常保护 - 启动时打印健康检查间隔，便于运维确认配置 Co-Authored-By: Claude --- .../ServiceHealthCheckService.cs | 104 +++++++++++------- .../ServiceInfoService.cs | 2 +- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/src/AgileConfig.Server.Service/ServiceHealthCheckService.cs b/src/AgileConfig.Server.Service/ServiceHealthCheckService.cs index 434d0984..a3fb1903 100644 --- a/src/AgileConfig.Server.Service/ServiceHealthCheckService.cs +++ b/src/AgileConfig.Server.Service/ServiceHealthCheckService.cs @@ -96,63 +96,83 @@ public Task StartCheckAsync() { _logger.LogInformation("start to service health check"); + // Cache the check interval upfront to avoid an exception during the loop + // (the getter throws if the config value is <= 0, which would crash the loop silently). + var interval = CheckInterval; + _logger.LogInformation("service health check interval: {0}s", interval); + Task.Factory.StartNew(async () => { while (true) { - // Skip services without a configured heartbeat mode. - var services = await _serviceInfoService - .QueryAsync(x => x.HeartBeatMode != null && x.HeartBeatMode != ""); - foreach (var service in services) + try { - if (service.HeartBeatMode == HeartBeatModes.none.ToString()) continue; - - var lstHeartBeat = service.LastHeartBeat; - if (!lstHeartBeat.HasValue) lstHeartBeat = service.RegisterTime ?? DateTime.MinValue; - - // A heartbeat mode is specified. - if (!string.IsNullOrWhiteSpace(service.HeartBeatMode)) + // Skip services without a configured heartbeat mode. + var services = await _serviceInfoService + .QueryAsync(x => x.HeartBeatMode != null && x.HeartBeatMode != ""); + foreach (var service in services) { - if (RemoveServiceInterval > 0 && - (DateTime.Now - lstHeartBeat.Value).TotalSeconds > RemoveServiceInterval) - { - // Remove the service if it has exceeded the configured lifetime. - await _serviceInfoService.RemoveAsync(service.Id); - continue; - } + if (service.HeartBeatMode == HeartBeatModes.none.ToString()) continue; - // Client-initiated heartbeats do not require an HTTP health check. - if (service.HeartBeatMode == HeartBeatModes.client.ToString()) - { - if ((DateTime.Now - lstHeartBeat.Value).TotalSeconds > UnhealthInterval) - // For client heartbeats, treat services as unavailable after UnhealthInterval without heartbeats. - if (service.Status == ServiceStatus.Healthy) - await _serviceInfoService.UpdateServiceStatus(service, ServiceStatus.Unhealthy); + var lstHeartBeat = service.LastHeartBeat; + if (!lstHeartBeat.HasValue) lstHeartBeat = service.RegisterTime ?? DateTime.MinValue; - continue; - } - - // Server-initiated HTTP health check. - if (service.HeartBeatMode == HeartBeatModes.server.ToString()) + // A heartbeat mode is specified. + if (!string.IsNullOrWhiteSpace(service.HeartBeatMode)) { - if (string.IsNullOrWhiteSpace(service.CheckUrl)) + if (RemoveServiceInterval > 0 && + (DateTime.Now - lstHeartBeat.Value).TotalSeconds > RemoveServiceInterval) { - // Without a CheckUrl, consider the service offline. - await _serviceInfoService.UpdateServiceStatus(service, ServiceStatus.Unhealthy); + // Remove the service if it has exceeded the configured lifetime. + await _serviceInfoService.RemoveAsync(service.Id); continue; } - _ = Task.Run(async () => + // Client-initiated heartbeats do not require an HTTP health check. + if (service.HeartBeatMode == HeartBeatModes.client.ToString()) { - var result = await CheckAService(service); - await _serviceInfoService.UpdateServiceStatus(service, - result ? ServiceStatus.Healthy : ServiceStatus.Unhealthy); - }); + if ((DateTime.Now - lstHeartBeat.Value).TotalSeconds > UnhealthInterval) + // For client heartbeats, treat services as unavailable after UnhealthInterval without heartbeats. + if (service.Status == ServiceStatus.Healthy) + await _serviceInfoService.UpdateServiceStatus(service, ServiceStatus.Unhealthy); + + continue; + } + + // Server-initiated HTTP health check. + if (service.HeartBeatMode == HeartBeatModes.server.ToString()) + { + if (string.IsNullOrWhiteSpace(service.CheckUrl)) + { + // Without a CheckUrl, consider the service offline. + await _serviceInfoService.UpdateServiceStatus(service, ServiceStatus.Unhealthy); + continue; + } + + _ = Task.Run(async () => + { + try + { + var result = await CheckAService(service); + await _serviceInfoService.UpdateServiceStatus(service, + result ? ServiceStatus.Healthy : ServiceStatus.Unhealthy); + } + catch (Exception ex) + { + _logger.LogError(ex, "check service health task error for {0} {1}", + service.ServiceId, service.ServiceName); + } + }); + } } } } + catch (Exception ex) + { + _logger.LogError(ex, "service health check loop error"); + } - await Task.Delay(CheckInterval * 1000); + await Task.Delay(interval * 1000); } }, TaskCreationOptions.LongRunning); @@ -170,9 +190,13 @@ private async Task CheckAService(ServiceInfo service) result = istatus >= 0 && istatus < 100; // Treat 2xx status codes as healthy responses. if (!result) + _logger.LogInformation("check service health {0} {1} {2} result：{3} status：{4}", service.CheckUrl, + service.ServiceId, + service.ServiceName, "down", (int)resp.StatusCode); + else _logger.LogInformation("check service health {0} {1} {2} result：{3}", service.CheckUrl, service.ServiceId, - service.ServiceName, "down"); + service.ServiceName, "up"); return result; } diff --git a/src/AgileConfig.Server.Service/ServiceInfoService.cs b/src/AgileConfig.Server.Service/ServiceInfoService.cs index 14b33590..4da37478 100644 --- a/src/AgileConfig.Server.Service/ServiceInfoService.cs +++ b/src/AgileConfig.Server.Service/ServiceInfoService.cs @@ -98,7 +98,7 @@ public async Task UpdateServiceStatus(ServiceInfo service, ServiceStatus status) if (service2 == null) return; service2.Status = status; - if (status != ServiceStatus.Unhealthy) service2.LastHeartBeat = DateTime.Now; + service2.LastHeartBeat = DateTime.Now; await _serviceInfoRepository.UpdateAsync(service2); if (oldStatus != status)