From 948a645d822bd903c8bf725ba85d69413e0ad819 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 Jun 2026 14:19:14 +0000 Subject: [PATCH 1/5] Move expensive info stats to standalone commands --- include/redis_command.h | 38 ++++++++-- include/redis_handler.h | 18 +++++ src/redis_command.cpp | 151 ++++++++++++++++++++++++---------------- src/redis_handler.cpp | 38 +++++++++- src/redis_service.cpp | 4 ++ 5 files changed, 181 insertions(+), 68 deletions(-) diff --git a/include/redis_command.h b/include/redis_command.h index 03dbc175..c44253e0 100644 --- a/include/redis_command.h +++ b/include/redis_command.h @@ -352,6 +352,9 @@ enum struct RedisCommandType ELOQVEC_DELETE = 184, ELOQVEC_SEARCH = 185, #endif +#ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD + KEYSPACE = 186, +#endif }; // Anchors: fail the build if any value above is accidentally shifted @@ -381,6 +384,9 @@ static_assert(static_cast(RedisCommandType::FAULT_INJECT) == 177); static_assert(static_cast(RedisCommandType::ELOQVEC_CREATE) == 178); static_assert(static_cast(RedisCommandType::ELOQVEC_SEARCH) == 185); #endif +#ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +static_assert(static_cast(RedisCommandType::KEYSPACE) == 186); +#endif enum RedisResultType { @@ -883,9 +889,6 @@ struct InfoCommand : public DirectCommand std::string enable_wal_; uint32_t node_memory_limit_mb_{0}; - int64_t data_memory_allocated_{0}; - int64_t data_memory_committed_{0}; - uint64_t last_ckpt_ts_{0}; int event_dispatcher_num_{0}; std::string os_info_; std::string executable_path_; @@ -902,13 +905,33 @@ struct InfoCommand : public DirectCommand int64_t multi_cmd_count_{0}; int64_t cmds_per_sec_{0}; double cmd_latency_{0}; - std::vector dbsizes_; +}; + +struct MemoryStatsCommand : public DirectCommand +{ + void Execute(RedisServiceImpl *redis_impl, + RedisConnectionContext *ctx) override; + + void OutputResult(OutputHandler *reply) const override; + + uint64_t data_memory_allocated_{0}; + uint64_t data_memory_committed_{0}; + uint64_t last_ckpt_ts_{0}; +}; + #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +struct KeyspaceCommand : public DirectCommand +{ + KeyspaceCommand() = default; + + void Execute(RedisServiceImpl *redis_impl, + RedisConnectionContext *ctx) override; + void OutputResult(OutputHandler *reply) const override; + + std::vector dbsizes_; uint64_t store_disk_keys_{0}; -#endif }; -#ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD struct CompactCommand : public DirectCommand { CompactCommand() = default; @@ -7945,6 +7968,9 @@ std::tuple ParseInfoCommand( const std::vector &args, OutputHandler *output); #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +std::tuple ParseKeyspaceCommand( + const std::vector &args, OutputHandler *output); + std::tuple ParseCompactCommand( const std::vector &args, OutputHandler *output); #endif diff --git a/include/redis_handler.h b/include/redis_handler.h index c0251c84..357644fe 100644 --- a/include/redis_handler.h +++ b/include/redis_handler.h @@ -259,6 +259,24 @@ class InfoCommandHandler : public RedisCommandHandler }; #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +class KeyspaceCommandHandler : public RedisCommandHandler +{ +public: + explicit KeyspaceCommandHandler(RedisServiceImpl *redis_impl) + : redis_impl_(redis_impl) + { + } + + brpc::RedisCommandHandlerResult Run( + RedisConnectionContext *ctx, + const std::vector &args, + brpc::RedisReply *output, + bool /*flush_batched*/) override; + +private: + RedisServiceImpl *redis_impl_; +}; + class CompactCommandHandler : public RedisCommandHandler { public: diff --git a/src/redis_command.cpp b/src/redis_command.cpp index 72f1e7c9..bf9b4997 100644 --- a/src/redis_command.cpp +++ b/src/redis_command.cpp @@ -111,6 +111,7 @@ const std::vector> command_types{{ {"dbsize", RedisCommandType::DBSIZE}, #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD {"compact", RedisCommandType::COMPACT}, + {"keyspace", RedisCommandType::KEYSPACE}, #endif {"time", RedisCommandType::TIME}, {"slowlog", RedisCommandType::SLOWLOG}, @@ -1779,28 +1780,6 @@ void InfoCommand::Execute(RedisServiceImpl *redis_impl, enable_wal_ = redis_impl->GetEnableWal(); node_memory_limit_mb_ = redis_impl->GetNodeMemoryLimitMB(); - if (set_section_.size() == 0 || - set_section_.find("memory") != set_section_.end()) - { - auto &local_shards = redis_impl->GetTxService()->CcShards(); - size_t cnt = local_shards.Count(); - CkptTsCc ckpt_req(cnt, node_id_); - for (size_t i = 0; i < cnt; i++) - { - local_shards.EnqueueCcRequest(i, &ckpt_req); - } - ckpt_req.Wait(); - data_memory_allocated_ = ckpt_req.GetMemUsage(); - data_memory_committed_ = ckpt_req.GetMemCommited(); - last_ckpt_ts_ = ckpt_req.GetCkptTs(); - } - else - { - data_memory_allocated_ = -1; - data_memory_committed_ = -1; - last_ckpt_ts_ = -1; - } - event_dispatcher_num_ = redis_impl->GetEventDispatcherNum(); version_ = redis_impl->GetVersion(); os_info_ = redis_impl->GetOsInfo(); @@ -1821,28 +1800,43 @@ void InfoCommand::Execute(RedisServiceImpl *redis_impl, // cmds_per_sec_ = RedisStats::GetCommandsPerSecond(); } -#ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD - if (set_section_.size() == 0 || - set_section_.find("keyspace") != set_section_.end()) - { - std::vector table_names; - size_t total_redis_table_cnt = redis_impl->GetRedisTableCount(); - for (size_t db_id = 0; db_id < total_redis_table_cnt; ++db_id) - { - const TableName *tbn = redis_impl->RedisTableName(db_id); - table_names.emplace_back( - tbn->StringView(), tbn->Type(), TableEngine::EloqKv); - } +} - dbsizes_ = DBSizeCommand::FetchDBSize(std::move(table_names)); - auto *store_hd = redis_impl->GetStoreHandler(); - store_disk_keys_ = - store_hd == nullptr ? 0 : store_hd->ApproxStoreKeyCount(); +void MemoryStatsCommand::Execute(RedisServiceImpl *redis_impl, + RedisConnectionContext *ctx) +{ + uint32_t node_id = Sharder::Instance().NodeId(); + auto &local_shards = redis_impl->GetTxService()->CcShards(); + size_t cnt = local_shards.Count(); + CkptTsCc ckpt_req(cnt, node_id); + for (size_t i = 0; i < cnt; i++) + { + local_shards.EnqueueCcRequest(i, &ckpt_req); } -#endif + ckpt_req.Wait(); + data_memory_allocated_ = ckpt_req.GetMemUsage(); + data_memory_committed_ = ckpt_req.GetMemCommited(); + last_ckpt_ts_ = ckpt_req.GetCkptTs(); } #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +void KeyspaceCommand::Execute(RedisServiceImpl *redis_impl, + RedisConnectionContext *ctx) +{ + std::vector table_names; + size_t total_redis_table_cnt = redis_impl->GetRedisTableCount(); + for (size_t db_id = 0; db_id < total_redis_table_cnt; ++db_id) + { + const TableName *tbn = redis_impl->RedisTableName(db_id); + table_names.emplace_back( + tbn->StringView(), tbn->Type(), TableEngine::EloqKv); + } + + dbsizes_ = DBSizeCommand::FetchDBSize(std::move(table_names)); + auto *store_hd = redis_impl->GetStoreHandler(); + store_disk_keys_ = store_hd == nullptr ? 0 : store_hd->ApproxStoreKeyCount(); +} + void CompactCommand::Execute(RedisServiceImpl *redis_impl, RedisConnectionContext *ctx) { @@ -2585,11 +2579,6 @@ void InfoCommand::OutputResult(OutputHandler *reply) const std::to_string(GetProcessVmRSSKb(getpid())) + " kb"; result += "\r\ntotal_system_memory:" + std::to_string(total_system_memory_kb_) + " kb"; - result += "\r\ndata_memory_allocated:" + - std::to_string(data_memory_allocated_) + " kb"; - result += "\r\ndata_memory_committed:" + - std::to_string(data_memory_committed_) + " kb"; - // result += "data_memory_frag_ratio: " + ; // TODO result += "\r\nnode_memory_limit:" + std::to_string(node_memory_limit_mb_) + " mb "; @@ -2605,7 +2594,6 @@ void InfoCommand::OutputResult(OutputHandler *reply) const result += "# Persistence"; - result += "\r\nlast_success_ckpt_ts:" + std::to_string(last_ckpt_ts_); result += "\r\nenable_data_store:" + enable_data_store_; result += "\r\nenable_wal:" + enable_wal_; } @@ -2680,31 +2668,48 @@ void InfoCommand::OutputResult(OutputHandler *reply) const } } + result += "\r\n"; + reply->OnString(result); +} + +void MemoryStatsCommand::OutputResult(OutputHandler *reply) const +{ + reply->OnArrayStart(6); + reply->OnString("data_memory_allocated"); + reply->OnInt(data_memory_allocated_); + reply->OnString("data_memory_committed"); + reply->OnInt(data_memory_committed_); + reply->OnString("last_success_ckpt_ts"); + reply->OnInt(last_ckpt_ts_); + reply->OnArrayEnd(); +} + #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD - if (set_section_.size() == 0 || - set_section_.find("keyspace") != set_section_.end()) +void KeyspaceCommand::OutputResult(OutputHandler *reply) const +{ + size_t db_count = 0; + for (int64_t dbsize : dbsizes_) { - if (result.size() > 0) + if (dbsize > 0) { - result += "\r\n"; + ++db_count; } + } - result += "# Keyspace"; - for (size_t idx = 0; idx < dbsizes_.size(); ++idx) + reply->OnArrayStart(db_count * 2 + 2); + for (size_t idx = 0; idx < dbsizes_.size(); ++idx) + { + if (dbsizes_[idx] > 0) { - if (dbsizes_[idx] > 0) - { - result += "\r\ndb" + std::to_string(idx) + - ":keys=" + std::to_string(dbsizes_[idx]); - } + reply->OnString("db" + std::to_string(idx) + ":keys"); + reply->OnInt(dbsizes_[idx]); } - result += "\r\nstore:disk_keys=" + std::to_string(store_disk_keys_); } -#endif - - result += "\r\n"; - reply->OnString(result); + reply->OnString("store:disk_keys"); + reply->OnInt(store_disk_keys_); + reply->OnArrayEnd(); } +#endif void ClusterCommand::OutputResult(OutputHandler *reply) const { @@ -8452,6 +8457,17 @@ ParseMultiCommand(RedisServiceImpl *redis_impl, DirectRequest(ctx, std::make_unique(std::move(cmd)))}; } #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD + case RedisCommandType::KEYSPACE: + { + auto [success, cmd] = ParseKeyspaceCommand(args, output); + if (!success) + { + return {false, DirectRequest{}}; + } + return {success, + DirectRequest( + ctx, std::make_unique(std::move(cmd)))}; + } case RedisCommandType::COMPACT: { auto [success, cmd] = ParseCompactCommand(args, output); @@ -10764,6 +10780,19 @@ std::tuple ParseInfoCommand( } #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +std::tuple ParseKeyspaceCommand( + const std::vector &args, OutputHandler *output) +{ + if (args.size() != 1) + { + output->OnError( + "ERR wrong number of arguments for 'keyspace' command"); + return {false, KeyspaceCommand()}; + } + + return {true, KeyspaceCommand()}; +} + std::tuple ParseCompactCommand( const std::vector &args, OutputHandler *output) { diff --git a/src/redis_handler.cpp b/src/redis_handler.cpp index ee161616..02d944ec 100644 --- a/src/redis_handler.cpp +++ b/src/redis_handler.cpp @@ -650,6 +650,26 @@ brpc::RedisCommandHandlerResult InfoCommandHandler::Run( } #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD +brpc::RedisCommandHandlerResult KeyspaceCommandHandler::Run( + RedisConnectionContext *ctx, + const std::vector &args, + brpc::RedisReply *output, + bool /*flush_batched*/) +{ + assert(args[0] == "keyspace"); + + RedisReplier reply(output); + std::vector cmd_arg_list = Transform(args); + + auto [success, cmd] = ParseKeyspaceCommand(cmd_arg_list, &reply); + if (success) + { + redis_impl_->ExecuteCommand(ctx, &cmd, &reply); + } + + return brpc::REDIS_CMD_HANDLED; +} + brpc::RedisCommandHandlerResult CompactCommandHandler::Run( RedisConnectionContext *ctx, const std::vector &args, @@ -4962,7 +4982,11 @@ brpc::RedisCommandHandlerResult MemoryHandler::Run( RedisReplier reply(output); std::vector cmd_arg_list = Transform(args); - assert(cmd_arg_list.size() > 1); + if (cmd_arg_list.size() < 2) + { + reply.OnError("wrong number of arguments for 'memory' command"); + return brpc::REDIS_CMD_HANDLED; + } if (stringcomp(cmd_arg_list[1], "usage", 1)) { @@ -4990,6 +5014,18 @@ brpc::RedisCommandHandlerResult MemoryHandler::Run( in_tx); } } + else if (stringcomp(cmd_arg_list[1], "stats", 1)) + { + if (cmd_arg_list.size() != 2) + { + reply.OnError("wrong number of arguments for 'memory stats' command"); + } + else + { + MemoryStatsCommand cmd; + redis_impl_->ExecuteCommand(ctx, &cmd, &reply); + } + } else { std::string err_msg = "ERR unknown subcommand '"; diff --git a/src/redis_service.cpp b/src/redis_service.cpp index c5dbed8b..5bda56f7 100644 --- a/src/redis_service.cpp +++ b/src/redis_service.cpp @@ -1418,6 +1418,10 @@ void RedisServiceImpl::AddHandlers() AddCommandHandler("info", info_hd.get()); #ifdef ELOQKV_WITH_DSS_ROCKSDB_CLOUD + auto &keyspace_hd = + hd_vec_.emplace_back(std::make_unique(this)); + AddCommandHandler("keyspace", keyspace_hd.get()); + auto &compact_hd = hd_vec_.emplace_back(std::make_unique(this)); AddCommandHandler("compact", compact_hd.get()); From a57285c60d14e3baf27cc4bf3c5a1f35519de86e Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 Jun 2026 14:24:25 +0000 Subject: [PATCH 2/5] fmt --- src/redis_command.cpp | 7 +++---- src/redis_handler.cpp | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/redis_command.cpp b/src/redis_command.cpp index bf9b4997..900774f1 100644 --- a/src/redis_command.cpp +++ b/src/redis_command.cpp @@ -1799,7 +1799,6 @@ void InfoCommand::Execute(RedisServiceImpl *redis_impl, multi_cmd_count_ = RedisStats::GetMultiObjectCommandsCount(); // cmds_per_sec_ = RedisStats::GetCommandsPerSecond(); } - } void MemoryStatsCommand::Execute(RedisServiceImpl *redis_impl, @@ -1834,7 +1833,8 @@ void KeyspaceCommand::Execute(RedisServiceImpl *redis_impl, dbsizes_ = DBSizeCommand::FetchDBSize(std::move(table_names)); auto *store_hd = redis_impl->GetStoreHandler(); - store_disk_keys_ = store_hd == nullptr ? 0 : store_hd->ApproxStoreKeyCount(); + store_disk_keys_ = + store_hd == nullptr ? 0 : store_hd->ApproxStoreKeyCount(); } void CompactCommand::Execute(RedisServiceImpl *redis_impl, @@ -10785,8 +10785,7 @@ std::tuple ParseKeyspaceCommand( { if (args.size() != 1) { - output->OnError( - "ERR wrong number of arguments for 'keyspace' command"); + output->OnError("ERR wrong number of arguments for 'keyspace' command"); return {false, KeyspaceCommand()}; } diff --git a/src/redis_handler.cpp b/src/redis_handler.cpp index 02d944ec..3979d29d 100644 --- a/src/redis_handler.cpp +++ b/src/redis_handler.cpp @@ -5018,7 +5018,8 @@ brpc::RedisCommandHandlerResult MemoryHandler::Run( { if (cmd_arg_list.size() != 2) { - reply.OnError("wrong number of arguments for 'memory stats' command"); + reply.OnError( + "wrong number of arguments for 'memory stats' command"); } else { From 9b51e412a2b12d4feafacd9e46936addd12f80d6 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 Jun 2026 01:29:20 +0000 Subject: [PATCH 3/5] update subm --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index eea6b87d..2b4516e2 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit eea6b87d0e323a9d8d937846008cd40cbb52b35f +Subproject commit 2b4516e2ae4bf33fe6d19f7eda47ec4d6e081de8 From 5a2a0d2f744ed9395d77e64ccd9022d8e122d602 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 Jun 2026 01:31:28 +0000 Subject: [PATCH 4/5] fmt --- src/redis_handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis_handler.cpp b/src/redis_handler.cpp index 3979d29d..b62aea2d 100644 --- a/src/redis_handler.cpp +++ b/src/redis_handler.cpp @@ -4984,7 +4984,7 @@ brpc::RedisCommandHandlerResult MemoryHandler::Run( std::vector cmd_arg_list = Transform(args); if (cmd_arg_list.size() < 2) { - reply.OnError("wrong number of arguments for 'memory' command"); + reply.OnError("ERR wrong number of arguments for 'memory' command"); return brpc::REDIS_CMD_HANDLED; } From e1fbf0b6ce6d9e2144fc0d44cdf31a57fa609c62 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 Jun 2026 01:46:27 +0000 Subject: [PATCH 5/5] update subm --- data_substrate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_substrate b/data_substrate index 2b4516e2..614bb47a 160000 --- a/data_substrate +++ b/data_substrate @@ -1 +1 @@ -Subproject commit 2b4516e2ae4bf33fe6d19f7eda47ec4d6e081de8 +Subproject commit 614bb47a55f3be4f037ced28e0b139510ff643f7