Skip to content

allow filestore to automatically update certificates #6598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yegorskii wants to merge 2 commits into
base: main
Choose a base branch
from
Open

allow filestore to automatically update certificates #6598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6807f7
allow filestore to automatically update certificates
yegorskii Jul 24, 2026
0ea29bb
update
yegorskii Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cloud/filestore/config/server.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ message TServerConfig
// Regions that have not been pinged within this timeout will be
// invalidated and unmapped.
optional uint32 SharedMemoryRegionTimeout = 23;

// Certificate refresh period (in milliseconds) for automatic TLS certificate
// reloading. Zero disables periodic certificate reloading.
optional uint32 RefreshCertsPeriod = 24;
}

////////////////////////////////////////////////////////////////////////////////
Expand Down
1 change: 1 addition & 0 deletions cloud/filestore/libs/daemon/common/bootstrap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class TBootstrapCommon
IActorSystemPtr ActorSystem;
NStorage::NFastShard::IServerPtr FastShardServer;
NCloud::NStorage::IStatsFetcherPtr StatsFetcher;
ITaskQueuePtr LongRunningTaskExecutor;
ICertificateProviderPtr CertificateProvider;

public:
Expand Down
25 changes: 22 additions & 3 deletions cloud/filestore/libs/daemon/server/bootstrap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <cloud/filestore/libs/service_local/config.h>
#include <cloud/filestore/libs/service_local/service.h>
#include <cloud/filestore/libs/service_null/service.h>
#include <cloud/filestore/libs/storage/api/components.h>
#include <cloud/filestore/libs/storage/core/config.h>
#include <cloud/filestore/libs/storage/core/probes.h>
#include <cloud/filestore/libs/storage/fastshard/bootstrap/core.h>
Expand Down Expand Up @@ -139,10 +140,28 @@ void TBootstrapServer::InitComponents()
});
}

if (!certPathList.empty()) {
CertificateProvider = CreateStaticCertificateProvider(
if (Configs->ServerConfig->GetRefreshCertsPeriod()) {
LongRunningTaskExecutor = CreateLongRunningTaskExecutor("CertRefresh");
}

if (certPathList.empty()) {
if (Configs->ServerConfig->GetSecurePort()) {
ythrow yexception()
<< "Secure port is configured without certificates";
}

CertificateProvider = CreateCertificateProviderStub();
} else {
CertificateProvider = CreateCertificateProvider(
Logging,
GetComponentName(
NStorage::TFileStoreComponents::TLS_CERTIFICATE_PROVIDER),
Scheduler,
LongRunningTaskExecutor,
serverCounters,
Configs->ServerConfig->GetRootCertsFile(),
std::move(certPathList));
std::move(certPathList),
Configs->ServerConfig->GetRefreshCertsPeriod());
}

Server = NServer::CreateServer(
Expand Down
118 changes: 110 additions & 8 deletions cloud/filestore/libs/daemon/vhost/bootstrap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include <cloud/filestore/libs/service_local/config.h>
#include <cloud/filestore/libs/service_local/service.h>
#include <cloud/filestore/libs/service_null/service.h>
#include <cloud/filestore/libs/storage/api/components.h>
#include <cloud/filestore/libs/storage/core/probes.h>
#include <cloud/filestore/libs/storage/fastshard/bootstrap/core.h>
#include <cloud/filestore/libs/storage/fastshard/client/async_client.h>
Expand Down Expand Up @@ -85,8 +86,51 @@ const TString ServerMetricsComponent = "server";

////////////////////////////////////////////////////////////////////////////////

struct TClientCertificateProviderFactory
{
ILoggingServicePtr Logging;
TString LogComponent;
ISchedulerPtr Scheduler;
ITaskQueuePtr LongRunningTaskExecutor;
NMonitoring::TDynamicCountersPtr ServerGroup;
TDuration RefreshInterval;

TClientCertificateProviderFactory(
ILoggingServicePtr logging,
TString logComponent,
ISchedulerPtr scheduler,
ITaskQueuePtr longRunningTaskExecutor,
NMonitoring::TDynamicCountersPtr serverGroup,
TDuration refreshInterval)
: Logging(std::move(logging))
, LogComponent(std::move(logComponent))
, Scheduler(std::move(scheduler))
, LongRunningTaskExecutor(std::move(longRunningTaskExecutor))
, ServerGroup(std::move(serverGroup))
, RefreshInterval(refreshInterval)
{}

ICertificateProviderPtr Build(
TString rootCertPath,
TVector<TCertificateFiles> certificates) const
{
return CreateCertificateProvider(
Logging,
LogComponent,
Scheduler,
LongRunningTaskExecutor,
Comment on lines +117 to +121

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Start vhost client certificate providers

When vhost is configured with RefreshCertsPeriod and a remote service endpoint uses SecurePort, this helper now returns a periodic certificate provider owned only by the filestore client. TFileStoreEndpoints::Start() only starts the durable endpoint, and the client start path never calls CertificateProvider->Start() (unlike the top-level daemon CertificateProvider started by TBootstrapCommon), so the provider never publishes its initial materials or schedules reloads; TLS clients created from it can miss certificates and automatic refresh does not work. Store/start/stop these client providers with the endpoint lifecycle before using them for secure channels.

Useful? React with 👍 / 👎.

ServerGroup,
std::move(rootCertPath),
std::move(certificates),
RefreshInterval);
}
};

////////////////////////////////////////////////////////////////////////////////

ICertificateProviderPtr CreateClientCertificateProvider(
const NClient::TClientConfigPtr& config)
const NClient::TClientConfigPtr& config,
const TClientCertificateProviderFactory& factory)
{
TVector<NCloud::TCertificateFiles> certPathList {
{
Expand All @@ -95,7 +139,12 @@ ICertificateProviderPtr CreateClientCertificateProvider(
}
};

return CreateStaticCertificateProvider(
if (config->GetSecurePort() && certPathList.empty()) {
ythrow yexception()
<< "Secure client port is configured without certificates";
}

return factory.Build(
config->GetRootCertsFile(),
std::move(certPathList));
}
Expand All @@ -107,24 +156,29 @@ class TFileStoreEndpoints final
{
private:
using TEndpointsMap = TMap<TString, IFileStoreServicePtr>;
using TCertificateProvidersMap = TMap<TString, ICertificateProviderPtr>;

private:
TClientCertificateProviderFactory CertificateProviderFactory;
ITimerPtr Timer;
ISchedulerPtr Scheduler;
ILoggingServicePtr Logging;
IFileStoreServicePtr LocalService;
IActorSystemPtr ActorSystem;

TEndpointsMap Endpoints;
TCertificateProvidersMap CertificateProviders;

public:
TFileStoreEndpoints(
TClientCertificateProviderFactory certificateProviderFactory,
ITimerPtr timer,
ISchedulerPtr scheduler,
ILoggingServicePtr logging,
IFileStoreServicePtr localService,
IActorSystemPtr actorSystem)
: Timer(std::move(timer))
: CertificateProviderFactory(std::move(certificateProviderFactory))
, Timer(std::move(timer))
, Scheduler(std::move(scheduler))
, Logging(std::move(logging))
, LocalService(std::move(localService))
Expand All @@ -133,6 +187,11 @@ class TFileStoreEndpoints final

void Start() override
{
for (const auto& [name, certificateProvider]: CertificateProviders) {
Y_UNUSED(name);
certificateProvider->Start();
}

for (const auto& [name, endpoint]: Endpoints) {
endpoint->Start();
}
Expand All @@ -143,6 +202,11 @@ class TFileStoreEndpoints final
for (const auto& [name, endpoint]: Endpoints) {
endpoint->Stop();
}

for (const auto& [name, certificateProvider]: CertificateProviders) {
Y_UNUSED(name);
certificateProvider->Stop();
}
}

IFileStoreServicePtr GetEndpoint(const TString& name) override
Expand All @@ -159,6 +223,7 @@ class TFileStoreEndpoints final
ui32 permanentActorCount)
{
auto clientConfig = std::make_shared<NClient::TClientConfig>(config);
ICertificateProviderPtr certificateProvider;
IFileStoreServicePtr fileStore;
switch (kind) {
case NDaemon::EServiceKind::Null: {
Expand All @@ -178,19 +243,28 @@ class TFileStoreEndpoints final
if (LocalService) {
fileStore = LocalService;
} else {
certificateProvider = CreateClientCertificateProvider(
clientConfig,
CertificateProviderFactory);
fileStore = NClient::CreateFileStoreClient(
clientConfig,
Logging,
CreateClientCertificateProvider(clientConfig));
certificateProvider);
}
break;
}
}

return AddEndpoint(
const bool inserted = AddEndpoint(
name,
std::move(clientConfig),
std::move(fileStore));

if (inserted && certificateProvider) {
CertificateProviders.emplace(name, std::move(certificateProvider));
}

return inserted;
}

bool Empty() const
Expand Down Expand Up @@ -367,6 +441,10 @@ void TBootstrapVhost::InitComponents()
Configs->VhostServiceConfig->GetEndpointStorageType());
}

if (Configs->ServerConfig->GetRefreshCertsPeriod()) {
LongRunningTaskExecutor = CreateLongRunningTaskExecutor("CertRefresh");
}

switch (Configs->Options->Service) {
case NDaemon::EServiceKind::Local:
case NDaemon::EServiceKind::Kikimr:
Expand Down Expand Up @@ -394,10 +472,24 @@ void TBootstrapVhost::InitComponents()
});
}

if (!certPathList.empty()) {
CertificateProvider = CreateStaticCertificateProvider(
if (certPathList.empty()) {
if (Configs->ServerConfig->GetSecurePort()) {
ythrow yexception()
<< "Secure port is configured without certificates";
}

CertificateProvider = CreateCertificateProviderStub();
} else {
CertificateProvider = CreateCertificateProvider(
Logging,
GetComponentName(
NStorage::TFileStoreComponents::TLS_CERTIFICATE_PROVIDER),
Scheduler,
LongRunningTaskExecutor,
serverCounters,
Configs->ServerConfig->GetRootCertsFile(),
std::move(certPathList));
std::move(certPathList),
Configs->ServerConfig->GetRefreshCertsPeriod());
}

Server = CreateServer(
Expand Down Expand Up @@ -459,7 +551,17 @@ void TBootstrapVhost::InitEndpoints()
localServiceConfig->Utf8DebugString().Quote().c_str());
}

TClientCertificateProviderFactory certFactory(
Logging,
GetComponentName(
NStorage::TFileStoreComponents::TLS_CERTIFICATE_PROVIDER),
Scheduler,
LongRunningTaskExecutor,

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Initialize client cert executor before endpoints

For vhost remote endpoints with RefreshCertsPeriod enabled, InitEndpoints() builds this factory before InitComponents() creates LongRunningTaskExecutor later in the method, so the factory captures a null task queue. Any periodic client certificate provider created from it will dereference that null queue from ScheduleUpdateAt() once its lifecycle is started or an update is requested, preventing client-side certificate refresh from working reliably; create the executor before initializing endpoints or pass a live queue into the factory.

Useful? React with 👍 / 👎.

FilestoreCounters->GetSubgroup("component", VhostMetricsComponent),
Configs->ServerConfig->GetRefreshCertsPeriod());

auto endpoints = std::make_shared<TFileStoreEndpoints>(
std::move(certFactory),
Timer,
Scheduler,
Logging,
Expand Down
1 change: 1 addition & 0 deletions cloud/filestore/libs/server/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ constexpr TDuration Seconds(int s)
xxx(SharedMemoryRegionTimeout, \
TDuration, \
TDuration::Max()) \
xxx(RefreshCertsPeriod, TDuration, Seconds(0) )\
// FILESTORE_SERVER_CONFIG

#define FILESTORE_SERVER_DECLARE_CONFIG(name, type, value) \
Expand Down
2 changes: 2 additions & 0 deletions cloud/filestore/libs/server/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class TServerConfig
TString GetSharedMemoryBasePath() const;
TDuration GetSharedMemoryRegionTimeout() const;

TDuration GetRefreshCertsPeriod() const;

const NProto::TServerConfig& GetProto() const
{
return ProtoConfig;
Expand Down
1 change: 1 addition & 0 deletions cloud/filestore/libs/storage/api/components.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ namespace NCloud::NFileStore::NStorage {
xxx(CLIENT) \
xxx(AUTH) \
xxx(USER_STATS) \
xxx(TLS_CERTIFICATE_PROVIDER) \
// FILESTORE_COMPONENTS

////////////////////////////////////////////////////////////////////////////////
Expand Down