From 42d53f0105f6e158bd1919d7466fef75293254d5 Mon Sep 17 00:00:00 2001 From: Evgeniy Kozev Date: Mon, 20 Jul 2026 13:20:13 +0000 Subject: [PATCH 1/2] Report endpoint restore failures --- .../libs/endpoints/endpoint_manager.cpp | 78 +++++++++++++------ .../libs/endpoints/endpoint_manager_ut.cpp | 8 ++ 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp index 82b70fb7441..e2221879710 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp @@ -510,12 +510,14 @@ class TEndpointManager final NClient::IMetricClientPtr RestoringClient; TSet RestoringEndpoints; + TAtomic RestoringErrors = 0; enum { WaitingForRestoring = 0, ReadingStorage = 1, StartingEndpoints = 2, Completed = 3, + Failed = 4, }; TAtomic RestoringStage = WaitingForRestoring; @@ -628,13 +630,18 @@ class TEndpointManager final TFuture RestoreEndpoints() override { + AtomicSet(RestoringErrors, 0); AtomicSet(RestoringStage, ReadingStorage); return Executor->Execute([weakSelf = weak_from_this()] () mutable { if (auto self = weakSelf.lock()) { auto future = self->DoRestoreEndpoints(); AtomicSet(self->RestoringStage, self->StartingEndpoints); self->Executor->WaitFor(future); - AtomicSet(self->RestoringStage, self->Completed); + AtomicSet( + self->RestoringStage, + AtomicGet(self->RestoringErrors) + ? self->Failed + : self->Completed); } }); } @@ -660,6 +667,7 @@ class TEndpointManager final case ReadingStorage: return true; case StartingEndpoints: return RestoringEndpoints.contains(socket); case Completed: return false; + case Failed: return false; } return false; } @@ -1243,6 +1251,14 @@ NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( Y_UNUSED(ctx); Y_UNUSED(request); + if (AtomicGet(RestoringStage) == Failed) { + return TErrorResponse( + E_FAIL, + TStringBuilder() + << "failed to restore " << AtomicGet(RestoringErrors) + << " endpoint(s)"); + } + NProto::TListEndpointsResponse response; auto& responseEndpoints = *response.MutableEndpoints(); responseEndpoints.Reserve(Endpoints.size()); @@ -1826,10 +1842,14 @@ TFuture TEndpointManager::SwitchEndpointIfNeeded( TFuture TEndpointManager::DoRestoreEndpoints() { auto [endpointIds, error] = EndpointStorage->GetEndpointIds(); - if (HasError(error) && !HasProtoFlag(error.GetFlags(), NProto::EF_SILENT)) { - ReportEndpointRestoringError( - TStringBuilder() - << "Failed to get endpoints from storage: " << FormatError(error)); + if (HasError(error)) { + AtomicIncrement(RestoringErrors); + if (!HasProtoFlag(error.GetFlags(), NProto::EF_SILENT)) { + ReportEndpointRestoringError( + TStringBuilder() + << "Failed to get endpoints from storage: " + << FormatError(error)); + } return MakeFuture(); } @@ -1844,15 +1864,22 @@ TFuture TEndpointManager::DoRestoreEndpoints() if (HasError(error) && !HasProtoFlag(error.GetFlags(), NProto::EF_SILENT)) { + AtomicIncrement(RestoringErrors); // NBS-3678 STORAGE_WARN("Failed to restore endpoint. ID: " << endpointId << ", error: " << FormatError(error)); continue; } + if (HasError(error)) { + AtomicIncrement(RestoringErrors); + continue; + } + auto request = DeserializeEndpoint(str); if (!request) { + AtomicIncrement(RestoringErrors); ReportEndpointRestoringError( TStringBuilder() << "Failed to deserialize request, error: " << FormatError(error), @@ -1867,6 +1894,7 @@ TFuture TEndpointManager::DoRestoreEndpoints() auto error = NbdDeviceManager.AcquireDevice(nbdDevice); if (HasError(error)) { + AtomicIncrement(RestoringErrors); ReportEndpointRestoringError( TStringBuilder() << "Failed to acquire nbd device, error: " @@ -1897,25 +1925,31 @@ TFuture TEndpointManager::DoRestoreEndpoints() std::move(request)); auto weakPtr = weak_from_this(); - future.Subscribe([weakPtr, socketPath, endpointId] (const auto& f) { - const auto& response = f.GetValue(); - if (HasError(response)) { - ReportEndpointRestoringError( - TStringBuilder() << "Endpoint restoring error occurred for " - "endpoint, error: " - << FormatError(response.GetError()), - {{"endpointId", endpointId}, {"socketPath", socketPath}}); - } + auto handledFuture = future.Apply( + [weakPtr, socketPath, endpointId] (const auto& f) { + const auto& response = f.GetValue(); + if (HasError(response)) { + ReportEndpointRestoringError( + TStringBuilder() + << "Endpoint restoring error occurred for " + "endpoint, error: " + << FormatError(response.GetError()), + {{"endpointId", endpointId}, + {"socketPath", socketPath}}); + } - if (auto ptr = weakPtr.lock()) { - ptr->HandleRestoredEndpoint( - socketPath, - endpointId, - response.GetError()); - } - }); + if (auto ptr = weakPtr.lock()) { + if (HasError(response)) { + AtomicIncrement(ptr->RestoringErrors); + } + ptr->HandleRestoredEndpoint( + socketPath, + endpointId, + response.GetError()); + } + }); - futures.push_back(future.IgnoreResult()); + futures.push_back(std::move(handledFuture)); } return WaitAll(futures); diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp index e6f942aa70f..8e2ce04f637 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp @@ -1758,6 +1758,14 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) UNIT_ASSERT(wrongCount != correctCount); UNIT_ASSERT_VALUES_EQUAL(wrongCount, static_cast(*configCounter)); + + auto response = + ListEndpoints(*manager).GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_VALUES_EQUAL(E_FAIL, response.GetError().GetCode()); + UNIT_ASSERT(!response.GetEndpointsWereRestored()); + UNIT_ASSERT_STRING_CONTAINS( + response.GetError().GetMessage(), + "failed to restore 3 endpoint(s)"); } Y_UNIT_TEST(ShouldRemoveEndpointForNotFoundVolume) From 3031e32fcbf1a73ef8018cb0ed1bdd41bcd03d14 Mon Sep 17 00:00:00 2001 From: Evgeniy Kozev Date: Mon, 20 Jul 2026 18:01:27 +0000 Subject: [PATCH 2/2] Expose endpoint restore state --- .../libs/endpoints/endpoint_manager.cpp | 31 ++++++++++++++----- .../libs/endpoints/endpoint_manager_ut.cpp | 13 +++++--- .../public/api/protos/endpoints.proto | 15 +++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp index e2221879710..c05342dad16 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp @@ -672,6 +672,24 @@ class TEndpointManager final return false; } + NProto::EEndpointRestoringState GetEndpointRestoringState() + { + switch (AtomicGet(RestoringStage)) { + case WaitingForRestoring: + return NProto::ENDPOINT_RESTORING_STATE_WAITING_FOR_RESTORING; + case ReadingStorage: + return NProto::ENDPOINT_RESTORING_STATE_READING_STORAGE; + case StartingEndpoints: + return NProto::ENDPOINT_RESTORING_STATE_STARTING_ENDPOINTS; + case Completed: + return NProto::ENDPOINT_RESTORING_STATE_COMPLETED; + case Failed: + return NProto::ENDPOINT_RESTORING_STATE_FAILED; + } + + return NProto::ENDPOINT_RESTORING_STATE_FAILED; + } + NProto::TStartEndpointResponse StartEndpointImpl( TCallContextPtr ctx, std::shared_ptr request, @@ -1251,13 +1269,6 @@ NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( Y_UNUSED(ctx); Y_UNUSED(request); - if (AtomicGet(RestoringStage) == Failed) { - return TErrorResponse( - E_FAIL, - TStringBuilder() - << "failed to restore " << AtomicGet(RestoringErrors) - << " endpoint(s)"); - } NProto::TListEndpointsResponse response; auto& responseEndpoints = *response.MutableEndpoints(); @@ -1267,8 +1278,12 @@ NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( responseEndpoints.Add()->CopyFrom(*endpoint->Request); } + const auto restoringStage = AtomicGet(RestoringStage); response.SetEndpointsWereRestored( - AtomicGet(RestoringStage) == Completed); + restoringStage == Completed || restoringStage == Failed); + response.SetEndpointRestoringState(GetEndpointRestoringState()); + response.SetEndpointRestoringErrorCount(AtomicGet(RestoringErrors)); + return response; } diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp index 8e2ce04f637..9cddedff380 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp @@ -1761,11 +1761,14 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) auto response = ListEndpoints(*manager).GetValue(TDuration::Seconds(5)); - UNIT_ASSERT_VALUES_EQUAL(E_FAIL, response.GetError().GetCode()); - UNIT_ASSERT(!response.GetEndpointsWereRestored()); - UNIT_ASSERT_STRING_CONTAINS( - response.GetError().GetMessage(), - "failed to restore 3 endpoint(s)"); + UNIT_ASSERT(!HasError(response)); + UNIT_ASSERT(response.GetEndpointsWereRestored()); + UNIT_ASSERT_VALUES_EQUAL( + static_cast(NProto::ENDPOINT_RESTORING_STATE_FAILED), + static_cast(response.GetEndpointRestoringState())); + UNIT_ASSERT_VALUES_EQUAL( + wrongCount, + response.GetEndpointRestoringErrorCount()); } Y_UNIT_TEST(ShouldRemoveEndpointForNotFoundVolume) diff --git a/cloud/blockstore/public/api/protos/endpoints.proto b/cloud/blockstore/public/api/protos/endpoints.proto index 8b1417876ac..a3f5ab417a4 100644 --- a/cloud/blockstore/public/api/protos/endpoints.proto +++ b/cloud/blockstore/public/api/protos/endpoints.proto @@ -135,6 +135,15 @@ message TListEndpointsRequest THeaders Headers = 1; } +enum EEndpointRestoringState +{ + ENDPOINT_RESTORING_STATE_WAITING_FOR_RESTORING = 0; + ENDPOINT_RESTORING_STATE_READING_STORAGE = 1; + ENDPOINT_RESTORING_STATE_STARTING_ENDPOINTS = 2; + ENDPOINT_RESTORING_STATE_COMPLETED = 3; + ENDPOINT_RESTORING_STATE_FAILED = 4; +} + message TListEndpointsResponse { // Optional error, set only if error happened. @@ -145,6 +154,12 @@ message TListEndpointsResponse // Endpoints restoring status. bool EndpointsWereRestored = 3; + + // Endpoint restoring stage. + EEndpointRestoringState EndpointRestoringState = 4; + + // Number of endpoint restoring errors. + uint32 EndpointRestoringErrorCount = 5; } ////////////////////////////////////////////////////////////////////////////////