From bbbb45511f40fbaef8a45ddff9deb645f373cb76 Mon Sep 17 00:00:00 2001 From: Harish Senthilkumar Date: Wed, 2 Sep 2026 22:54:22 +0000 Subject: [PATCH] Add ACCELERATED_COMPUTE health check for the MPS control daemon --- agent/app/agent.go | 3 + agent/app/agent_capability_unix.go | 27 ++- agent/app/agent_mps_healthcheck_unix.go | 39 +++ agent/app/agent_mps_healthcheck_unix_test.go | 83 +++++++ .../app/agent_mps_healthcheck_unsupported.go | 27 +++ agent/doctor/mps_daemon_healthcheck.go | 122 ++++++++++ agent/doctor/mps_daemon_healthcheck_test.go | 228 ++++++++++++++++++ 7 files changed, 521 insertions(+), 8 deletions(-) create mode 100644 agent/app/agent_mps_healthcheck_unix.go create mode 100644 agent/app/agent_mps_healthcheck_unix_test.go create mode 100644 agent/app/agent_mps_healthcheck_unsupported.go create mode 100644 agent/doctor/mps_daemon_healthcheck.go create mode 100644 agent/doctor/mps_daemon_healthcheck_test.go diff --git a/agent/app/agent.go b/agent/app/agent.go index becc0115d84..0089e8bc154 100644 --- a/agent/app/agent.go +++ b/agent/app/agent.go @@ -725,6 +725,9 @@ func (agent *ecsAgent) newDoctorWithHealthchecks(cluster, containerInstanceARN s runtimeHealthCheck, } + // register the MPS control daemon health check on MPS-capable instances + healthcheckList = agent.appendMpsDaemonHealthcheck(healthcheckList) + // set up the doctor and return it return doctor.NewDoctor(healthcheckList, cluster, containerInstanceARN) } diff --git a/agent/app/agent_capability_unix.go b/agent/app/agent_capability_unix.go index a33930b5494..a18670fac79 100644 --- a/agent/app/agent_capability_unix.go +++ b/agent/app/agent_capability_unix.go @@ -126,22 +126,33 @@ func (agent *ecsAgent) appendNvidiaDriverVersionAttribute(capabilities []types.A return capabilities } -// appendGpuSharingMpsCapability advertises ecs.capability.gpu-sharing-mps when the -// instance can run MPS: a GPU is present, the MPS control binary and its systemd unit -// are installed and enabled, the GPU is not a vGPU slice, and every discovered GPU has -// a usable-memory value. The facts are gathered by ecs-init and read from the NvidiaGPUManager; -// the decision itself lives in the shared gpu package so the MI agent reaches the same verdict from the same inputs. -func (agent *ecsAgent) appendGpuSharingMpsCapability(capabilities []types.Attribute) []types.Attribute { +// mpsCapabilityInputs gathers the host facts the MPS gate decision is made from, +// reading them off the NvidiaGPUManager. ok is false when there is no GPU manager, in +// which case MPS cannot be evaluated on this instance. It is the single source of truth +// shared by the gpu-sharing-mps capability and the MPS daemon health check. +func (agent *ecsAgent) mpsCapabilityInputs() (gpu.MpsCapabilityInputs, bool) { if agent.resourceFields == nil || agent.resourceFields.NvidiaGPUManager == nil { - return capabilities + return gpu.MpsCapabilityInputs{}, false } mgr := agent.resourceFields.NvidiaGPUManager - inputs := gpu.MpsCapabilityInputs{ + return gpu.MpsCapabilityInputs{ GPUPresent: len(mgr.GetDevices()) > 0, MpsBinaryPresent: mgr.GetMpsControlBinaryPresent(), MpsServiceEnabled: mgr.GetMpsServiceEnabled(), IsVGPU: mgr.GetHasVGPU(), AllGPUsHaveMemory: gpu.AllGPUMemoryReported(mgr.GetGPUIDsUnsafe(), mgr.GetGPUMemoryMiBUnsafe()), + }, true +} + +// appendGpuSharingMpsCapability advertises ecs.capability.gpu-sharing-mps when the +// instance can run MPS: a GPU is present, the MPS control binary and its systemd unit +// are installed and enabled, the GPU is not a vGPU slice, and every discovered GPU has +// a usable-memory value. The facts are gathered by ecs-init and read from the NvidiaGPUManager; +// the decision itself lives in the shared gpu package so the MI agent reaches the same verdict from the same inputs. +func (agent *ecsAgent) appendGpuSharingMpsCapability(capabilities []types.Attribute) []types.Attribute { + inputs, ok := agent.mpsCapabilityInputs() + if !ok { + return capabilities } advertise, conditions := gpu.ShouldAdvertiseMpsCapability(inputs) if !advertise { diff --git a/agent/app/agent_mps_healthcheck_unix.go b/agent/app/agent_mps_healthcheck_unix.go new file mode 100644 index 00000000000..56fa48c75c3 --- /dev/null +++ b/agent/app/agent_mps_healthcheck_unix.go @@ -0,0 +1,39 @@ +//go:build linux +// +build linux + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package app + +import ( + agentdoctor "github.com/aws/amazon-ecs-agent/agent/doctor" + "github.com/aws/amazon-ecs-agent/ecs-agent/doctor" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/gpu" +) + +// appendMpsDaemonHealthcheck registers the MPS control-daemon health check on exactly +// the instances that advertise ecs.capability.gpu-sharing-mps, reusing the same +// predicate so the instance health signal and the capability agree on what +// MPS-capable means. A plain GPU box with no nvidia-mps.service would otherwise fail +// every probe and report a false ACCELERATED_COMPUTE=IMPAIRED. +func (agent *ecsAgent) appendMpsDaemonHealthcheck(list []doctor.Healthcheck) []doctor.Healthcheck { + inputs, ok := agent.mpsCapabilityInputs() + if !ok { + return list + } + if advertise, _ := gpu.ShouldAdvertiseMpsCapability(inputs); !advertise { + return list + } + return append(list, agentdoctor.NewMpsDaemonHealthcheck()) +} diff --git a/agent/app/agent_mps_healthcheck_unix_test.go b/agent/app/agent_mps_healthcheck_unix_test.go new file mode 100644 index 00000000000..ef7fbfe15c3 --- /dev/null +++ b/agent/app/agent_mps_healthcheck_unix_test.go @@ -0,0 +1,83 @@ +//go:build linux && unit +// +build linux,unit + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package app + +import ( + "testing" + + "github.com/aws/amazon-ecs-agent/agent/gpu" + "github.com/aws/amazon-ecs-agent/agent/taskresource" + "github.com/stretchr/testify/assert" +) + +// TestAppendMpsDaemonHealthcheck asserts that the MPS control-daemon health check is +// registered on exactly the instances that would advertise gpu-sharing-mps: it must be +// appended when ShouldAdvertiseMpsCapability is satisfied and withheld otherwise, so a +// plain GPU box never reports a false ACCELERATED_COMPUTE=IMPAIRED. It reuses the same +// NvidiaGPUManager setup as the capability test to keep the two decisions in lockstep. +func TestAppendMpsDaemonHealthcheck(t *testing.T) { + // newMpsManager returns a GPU manager carrying the given MPS facts and one device + // (with usable memory) unless gpuPresent is false. haveMemory controls whether that + // device reports a memory value, exercising the AllGPUsHaveMemory gate. + newMpsManager := func(gpuPresent, binary, service, vgpu, haveMemory bool) *gpu.NvidiaGPUManager { + m := &gpu.NvidiaGPUManager{ + MpsControlBinaryPresent: binary, + MpsServiceEnabled: service, + HasVGPU: vgpu, + } + if gpuPresent { + m.SetGPUIDs([]string{"gpu-0"}) + if haveMemory { + m.SetGPUMemoryMiB(map[string]uint64{"gpu-0": 22563}) + } + m.SetDevices() + } + return m + } + + cases := []struct { + name string + mgr *gpu.NvidiaGPUManager // nil means no GPU manager on the instance + register bool + }{ + {"all conditions met", newMpsManager(true, true, true, false, true), true}, + {"no gpu manager", nil, false}, + {"no gpu present", newMpsManager(false, true, true, false, true), false}, + {"mps binary absent", newMpsManager(true, false, true, false, true), false}, + {"mps service disabled", newMpsManager(true, true, false, false, true), false}, + {"is vgpu", newMpsManager(true, true, true, true, true), false}, + {"gpu present but no memory reported", newMpsManager(true, true, true, false, false), false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // NvidiaGPUManager is an interface, so a typed-nil pointer would read as a + // non-nil interface; leave the field unset to model "no GPU manager". + rf := &taskresource.ResourceFields{} + if tc.mgr != nil { + rf.NvidiaGPUManager = tc.mgr + } + agent := &ecsAgent{resourceFields: rf} + got := agent.appendMpsDaemonHealthcheck(nil) + if tc.register { + assert.Len(t, got, 1, "the MPS health check must be registered") + } else { + assert.Empty(t, got, "the MPS health check must not be registered") + } + }) + } +} diff --git a/agent/app/agent_mps_healthcheck_unsupported.go b/agent/app/agent_mps_healthcheck_unsupported.go new file mode 100644 index 00000000000..1d0811ccaa6 --- /dev/null +++ b/agent/app/agent_mps_healthcheck_unsupported.go @@ -0,0 +1,27 @@ +//go:build !linux +// +build !linux + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package app + +import ( + "github.com/aws/amazon-ecs-agent/ecs-agent/doctor" +) + +// appendMpsDaemonHealthcheck is a no-op off Linux, where the MPS control daemon and its +// pipe directory do not exist. +func (agent *ecsAgent) appendMpsDaemonHealthcheck(list []doctor.Healthcheck) []doctor.Healthcheck { + return list +} diff --git a/agent/doctor/mps_daemon_healthcheck.go b/agent/doctor/mps_daemon_healthcheck.go new file mode 100644 index 00000000000..27c436bff87 --- /dev/null +++ b/agent/doctor/mps_daemon_healthcheck.go @@ -0,0 +1,122 @@ +//go:build linux +// +build linux + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package doctor + +import ( + "fmt" + "os" + + "github.com/aws/amazon-ecs-agent/agent/doctor/statustracker" + "github.com/aws/amazon-ecs-agent/ecs-agent/doctor" + "github.com/aws/amazon-ecs-agent/ecs-agent/logger" + "github.com/aws/amazon-ecs-agent/ecs-agent/logger/field" + "github.com/aws/amazon-ecs-agent/ecs-agent/tcs/model/ecstcs" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/mps" +) + +// mpsDaemonImpairedThreshold is the number of consecutive probe failures before the +// instance is reported ACCELERATED_COMPUTE=IMPAIRED +const mpsDaemonImpairedThreshold = 3 + +type mpsDaemonHealthcheck struct { + *statustracker.HealthCheckStatusTracker + + exec execwrapper.Exec + statPipeDir func(string) (os.FileInfo, error) + probeCmd string + + consecutiveFailures int + threshold int +} + +// NewMpsDaemonHealthcheck is the constructor for the MPS control-daemon health check. +func NewMpsDaemonHealthcheck() doctor.Healthcheck { + return newMpsDaemonHealthcheck(execwrapper.NewExec(), os.Stat) +} + +func newMpsDaemonHealthcheck(exec execwrapper.Exec, + statPipeDir func(string) (os.FileInfo, error)) *mpsDaemonHealthcheck { + return &mpsDaemonHealthcheck{ + HealthCheckStatusTracker: statustracker.NewHealthCheckStatusTracker(), + exec: exec, + statPipeDir: statPipeDir, + probeCmd: mps.ProbeCommand, + threshold: mpsDaemonImpairedThreshold, + } +} + +// RunCheck runs one probe per tick and folds the result into the consecutive-failure +// counter. It reports Impaired only after threshold failures in a row; any success +// resets the counter, returning the status to Ok. +func (m *mpsDaemonHealthcheck) RunCheck() ecstcs.InstanceHealthCheckStatus { + res := m.probe() + serving := res.Err == nil + + if serving { + m.consecutiveFailures = 0 + } else { + m.consecutiveFailures++ + } + + status := ecstcs.InstanceHealthCheckStatusOk + if m.consecutiveFailures >= m.threshold { + status = ecstcs.InstanceHealthCheckStatusImpaired + } + + switch { + case status == ecstcs.InstanceHealthCheckStatusImpaired: + logger.Error("MPS control daemon health check impaired", logger.Fields{ + "consecutiveFailures": m.consecutiveFailures, + "timedOut": res.TimedOut, + field.Error: res.Err, + }) + case !serving: + logger.Warn("MPS control daemon probe failed, below impairment threshold", logger.Fields{ + "consecutiveFailures": m.consecutiveFailures, + "timedOut": res.TimedOut, + field.Error: res.Err, + }) + default: + logger.Debug("MPS control daemon is serving") + } + + m.SetHealthcheckStatus(status) + return m.GetHealthcheckStatus() +} + +// probe runs the pipe-directory pre-check and, if it passes, a single control-daemon +// probe. The daemon is serving when the returned result has a nil Err. An unusable +// pipe directory counts as this tick's failure and skips the exec; it can recover on a +// later tick. +func (m *mpsDaemonHealthcheck) probe() mps.ProbeResult { + fi, err := m.statPipeDir(mps.PipeDirectory) + if err != nil { + return mps.ProbeResult{Err: err} + } + if !fi.IsDir() { + return mps.ProbeResult{ + Err: fmt.Errorf("mps pipe directory %s is not a directory", mps.PipeDirectory), + } + } + return mps.ProbeControlDaemon(m.exec, m.probeCmd) +} + +// GetHealthcheckType returns the type of this health check. +func (m *mpsDaemonHealthcheck) GetHealthcheckType() string { + return ecstcs.InstanceHealthCheckTypeAcceleratedCompute +} diff --git a/agent/doctor/mps_daemon_healthcheck_test.go b/agent/doctor/mps_daemon_healthcheck_test.go new file mode 100644 index 00000000000..35839ad89a2 --- /dev/null +++ b/agent/doctor/mps_daemon_healthcheck_test.go @@ -0,0 +1,228 @@ +//go:build unit && linux +// +build unit,linux + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package doctor + +import ( + "context" + "errors" + "os" + "os/exec" + "testing" + "time" + + "github.com/aws/amazon-ecs-agent/ecs-agent/tcs/model/ecstcs" + mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/mps" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +// dirInfo is a stub os.FileInfo that reports itself as a directory. +type dirInfo struct{ os.FileInfo } + +func (dirInfo) IsDir() bool { return true } + +// dirStat reports the pipe directory as present, or statErr if non-nil. +func dirStat(statErr error) func(string) (os.FileInfo, error) { + return func(string) (os.FileInfo, error) { + if statErr != nil { + return nil, statErr + } + return dirInfo{}, nil + } +} + +// expectProbe queues the exec call sequence ProbeControlDaemon makes for one probe. A +// non-nil err makes that probe a failure with exit code 1 (daemon not serving). +func expectProbe(mockExec *mock_execwrapper.MockExec, mockCmd *mock_execwrapper.MockCmd, + out []byte, err error) { + mockExec.EXPECT().NewExecContextWithTimeout(gomock.Any(), mps.ProbeTimeout). + DoAndReturn(func(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, d) + }) + mockExec.EXPECT().CommandContext(gomock.Any(), mps.ControlBinary).Return(mockCmd) + mockCmd.EXPECT().SetEnv(gomock.Any()) + mockCmd.EXPECT().SetIOStreams(gomock.Any(), gomock.Any(), gomock.Any()) + mockCmd.EXPECT().CombinedOutput().Return(out, err) + if err != nil { + mockExec.EXPECT().ConvertToExitError(err).Return(&exec.ExitError{}, true) + mockExec.EXPECT().GetExitCode(gomock.Any()).Return(1) + } +} + +// expectTimeoutProbe queues one probe that hangs past the deadline: the context is +// already expired, so ctx.Err() reports DeadlineExceeded and the result is TimedOut. +func expectTimeoutProbe(mockExec *mock_execwrapper.MockExec, mockCmd *mock_execwrapper.MockCmd) { + mockExec.EXPECT().NewExecContextWithTimeout(gomock.Any(), mps.ProbeTimeout). + DoAndReturn(func(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, 0) + }) + mockExec.EXPECT().CommandContext(gomock.Any(), mps.ControlBinary).Return(mockCmd) + mockCmd.EXPECT().SetEnv(gomock.Any()) + mockCmd.EXPECT().SetIOStreams(gomock.Any(), gomock.Any(), gomock.Any()) + killErr := errors.New("signal: killed") + mockCmd.EXPECT().CombinedOutput().Return([]byte{}, killErr) + mockExec.EXPECT().ConvertToExitError(killErr).Return(nil, false) +} + +var probeFailErr = errors.New("Cannot find MPS control daemon process") + +func TestMpsGetHealthcheckType(t *testing.T) { + hc := NewMpsDaemonHealthcheck() + assert.Equal(t, ecstcs.InstanceHealthCheckTypeAcceleratedCompute, hc.GetHealthcheckType()) +} + +func TestMpsInitialHealth(t *testing.T) { + hc := NewMpsDaemonHealthcheck() + assert.Equal(t, ecstcs.InstanceHealthCheckStatusInitializing, hc.GetHealthcheckStatus()) +} + +// tickKind is the outcome one health-check tick observes when it probes the daemon. +type tickKind int + +const ( + tickServing tickKind = iota // daemon responds; the probe succeeds + tickFailure // probe runs but the daemon is not serving (exit 1) + tickTimeout // probe hangs past the deadline + tickPipeMissing // pipe directory absent; the exec is skipped +) + +// tick pairs a probe outcome with the instance status the check must report after it. +type tick struct { + kind tickKind + want ecstcs.InstanceHealthCheckStatus +} + +// TestMpsRunCheckSequences drives RunCheck through sequences of probe outcomes and +// asserts the reported status after every tick. Because a serving tick zeroes the +// counter, a status-only assertion still proves the reset semantics: a failure that +// follows a success reports Ok where an unbroken streak of the same length would be +// Impaired. +func TestMpsRunCheckSequences(t *testing.T) { + const ( + ok = ecstcs.InstanceHealthCheckStatusOk + impaired = ecstcs.InstanceHealthCheckStatusImpaired + ) + cases := []struct { + name string + ticks []tick + }{ + { + // Anti-flap: below the threshold a failing probe must not report IMPAIRED, + // so a short restart that lands on a tick is absorbed. + name: "below threshold stays ok", + ticks: []tick{{tickFailure, ok}, {tickFailure, ok}}, + }, + { + name: "third consecutive failure impaired", + ticks: []tick{{tickFailure, ok}, {tickFailure, ok}, {tickFailure, impaired}}, + }, + { + // The fourth tick is Ok only because the success zeroed the counter; an + // unbroken streak of four failures would have been Impaired by tick three. + name: "success resets counter", + ticks: []tick{{tickFailure, ok}, {tickFailure, ok}, {tickServing, ok}, {tickFailure, ok}}, + }, + { + // A single success mid-streak keeps the count from ever reaching the + // threshold, so the instance never reports Impaired. + name: "single success mid streak resets", + ticks: []tick{{tickFailure, ok}, {tickFailure, ok}, {tickServing, ok}, + {tickFailure, ok}, {tickFailure, ok}}, + }, + { + // A timeout is a failure like any other, and three in a row cross the threshold. + name: "timeout counts as failure", + ticks: []tick{{tickTimeout, ok}, {tickTimeout, ok}, {tickTimeout, impaired}}, + }, + { + // A missing pipe directory skips the exec and counts as one failure. Unlike + // the task gate this is not fail-closed, so three such ticks report Impaired + // and a later serving probe resets to Ok. + name: "pipe directory missing counts as failure then recovers", + ticks: []tick{{tickPipeMissing, ok}, {tickPipeMissing, ok}, + {tickPipeMissing, impaired}, {tickServing, ok}}, + }, + { + // Recovery: once past the threshold a serving probe returns the instance to Ok. + name: "recovery returns to ok", + ticks: []tick{{tickFailure, ok}, {tickFailure, ok}, {tickFailure, impaired}, + {tickServing, ok}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockExec := mock_execwrapper.NewMockExec(ctrl) + mockCmd := mock_execwrapper.NewMockCmd(ctrl) + + pipePresent := true + statErr := errors.New("no such file or directory") + hc := newMpsDaemonHealthcheck(mockExec, func(string) (os.FileInfo, error) { + if pipePresent { + return dirInfo{}, nil + } + return nil, statErr + }) + + for i, tk := range tc.ticks { + pipePresent = tk.kind != tickPipeMissing + switch tk.kind { + case tickServing: + expectProbe(mockExec, mockCmd, []byte("100.0\n"), nil) + case tickFailure: + expectProbe(mockExec, mockCmd, []byte(""), probeFailErr) + case tickTimeout: + expectTimeoutProbe(mockExec, mockCmd) + case tickPipeMissing: + // probe short-circuits on the stat error; no exec is expected. + } + assert.Equalf(t, tk.want, hc.RunCheck(), "tick %d", i) + } + }) + } +} + +// The transition back to Ok must be observable to the publisher, which sends only on +// change: GetStatusChangeTime advances on recovery. +func TestMpsRecoveryTransitionAdvancesStatusChangeTime(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockExec := mock_execwrapper.NewMockExec(ctrl) + mockCmd := mock_execwrapper.NewMockCmd(ctrl) + for i := 0; i < mpsDaemonImpairedThreshold; i++ { + expectProbe(mockExec, mockCmd, []byte(""), probeFailErr) + } + expectProbe(mockExec, mockCmd, []byte("100.0\n"), nil) + + hc := newMpsDaemonHealthcheck(mockExec, dirStat(nil)) + + hc.RunCheck() + hc.RunCheck() + assert.Equal(t, ecstcs.InstanceHealthCheckStatusImpaired, hc.RunCheck()) + impairedAt := hc.GetStatusChangeTime() + + // A distinct clock reading, so the recovery transition timestamp is provably newer. + time.Sleep(time.Millisecond) + + assert.Equal(t, ecstcs.InstanceHealthCheckStatusOk, hc.RunCheck()) + assert.True(t, hc.GetStatusChangeTime().After(impairedAt), + "recovery is a status change and must advance GetStatusChangeTime") + assert.Equal(t, ecstcs.InstanceHealthCheckStatusImpaired, hc.GetLastHealthcheckStatus()) +}