feat: Upgrade rocksdb to v10.9.1 and add tuning profiles. (#1977)

* use rocksdb 10.9.1

* tune-up rocksdb

* provides rocksdb tune-up options for the different node types

* update changelog

* nit

* fix rocksdb build issue in nix

* update changelog

* lint

* deploy cache when loadLatestOptions failed

* rollback rocksdb to v10.9.1

* fix tune-up issues

* update CHANGELOG

* fix mem leak

* update cronos-store revision

* revert nix/rocksdb postPatch change

* add DirectIOForCompaction flag

* update toml comment

* add proper destroys

* add comment

---------

Signed-off-by: JayT106 <JayT106@users.noreply.github.com>

Author: JayT106

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 
+* [#1977](https://github.com/crypto-org-chain/cronos/pull/1977) Upgrade rocksdb to v10.9.1 and add tuning profiles.
 * [#1970](https://github.com/crypto-org-chain/cronos/pull/1970) feat(testground): refine testground test process in local
 * [#1971](https://github.com/crypto-org-chain/cronos/pull/1971) Upgrade cosmos-sdk to v0.53.4.
 * [#1976](https://github.com/crypto-org-chain/cronos/pull/1976) Minor improvements for cosmos-sdk v0.53.4 upgrade.

cmd/cronosd/cmd/root.go

@@ -276,6 +276,7 @@ func initAppConfig() (string, interface{}) {
 		MemIAVL   memiavlcfg.MemIAVLConfig `mapstructure:"memiavl"`
 		VersionDB VersionDBConfig          `mapstructure:"versiondb"`
 		Cronos    config2.CronosConfig     `mapstructure:"cronos"`
+		RocksDB   config2.RocksDBConfig    `mapstructure:"rocksdb"`
 	}
 
 	tpl, cfg := servercfg.AppConfig("")
@@ -285,9 +286,10 @@ func initAppConfig() (string, interface{}) {
 		MemIAVL:   memiavlcfg.DefaultMemIAVLConfig(),
 		VersionDB: DefaultVersionDBConfig(),
 		Cronos:    config2.DefaultCronosConfig(),
+		RocksDB:   config2.DefaultRocksDBConfig(),
 	}
 
-	return tpl + memiavlcfg.DefaultConfigTemplate + DefaultVersionDBTemplate + config2.DefaultCronosConfigTemplate, customAppConfig
+	return tpl + memiavlcfg.DefaultConfigTemplate + DefaultVersionDBTemplate + config2.DefaultCronosConfigTemplate + config2.DefaultRocksDBConfigTemplate, customAppConfig
 }
 
 // newApp creates the application

cmd/cronosd/cmd/versiondb.go

@@ -24,7 +24,7 @@ func ChangeSetCmd() *cobra.Command {
 		DefaultStores:  storeNames,
 		OpenReadOnlyDB: opendb.OpenReadOnlyDB,
 		AppRocksDBOptions: func(sstFileWriter bool) *grocksdb.Options {
-			return opendb.NewRocksdbOptions(nil, sstFileWriter)
+			return opendb.NewRocksdbOptions(nil, sstFileWriter, opendb.RocksDBTuneUpOptions{})
 		},
 	})
 }

cmd/cronosd/config/config.go

@@ -1,6 +1,11 @@
 package config
 
-import sdk "github.com/cosmos/cosmos-sdk/types"
+import (
+	"fmt"
+	"strings"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
 
 // SetBech32Prefixes sets the global prefixes to be used when serializing addresses and public keys to Bech32 strings.
 func SetBech32Prefixes(config *sdk.Config) {
@@ -16,9 +21,40 @@ type CronosConfig struct {
 	DisableOptimisticExecution bool `mapstructure:"disable-optimistic-execution"`
 }
 
+const (
+	NodeTypeDefault   = ""
+	NodeTypeValidator = "validator"
+	NodeTypeRPC       = "rpc"
+	NodeTypeArchive   = "archive"
+)
+
+type RocksDBConfig struct {
+	// Defines the tuning profile for RocksDB based on the node's primary workload.
+	// Valid values: "", "validator", "rpc", "archive"
+	NodeType string `mapstructure:"node_type"`
+}
+
+func (c *RocksDBConfig) Validate() error {
+	normalized := strings.ToLower(strings.TrimSpace(c.NodeType))
+	switch normalized {
+	case NodeTypeDefault, NodeTypeValidator, NodeTypeRPC, NodeTypeArchive:
+		c.NodeType = normalized
+		return nil
+	default:
+		return fmt.Errorf("invalid rocksdb.node_type %q: allowed values are %q, %q, %q, or %q (empty)",
+			c.NodeType, NodeTypeValidator, NodeTypeRPC, NodeTypeArchive, NodeTypeDefault)
+	}
+}
+
 func DefaultCronosConfig() CronosConfig {
 	return CronosConfig{
 		DisableTxReplacement:       false,
 		DisableOptimisticExecution: false,
 	}
 }
+
+func DefaultRocksDBConfig() RocksDBConfig {
+	return RocksDBConfig{
+		NodeType: "",
+	}
+}

cmd/cronosd/config/toml.go

@@ -14,3 +14,21 @@ disable-tx-replacement = {{ .Cronos.DisableTxReplacement }}
 # Set to true to disable optimistic execution (not recommended on validator nodes).
 disable-optimistic-execution = {{ .Cronos.DisableOptimisticExecution }}
 `
+
+// DefaultRocksDBConfigTemplate defines the configuration template for rocksdb configuration
+const DefaultRocksDBConfigTemplate = `
+###############################################################################
+###                             RocksDB Configuration                       ###
+###############################################################################
+
+[rocksdb]
+
+# Defines the tuning profile for RocksDB based on the node's primary workload.
+# This is an experimental feature for performance optimization.
+# Valid values:
+# - ""          (default): standard configuration, safe for all nodes.
+# - "validator" : optimizes for lowest latency point-lookups (state reads) during block execution.
+# - "rpc"       : optimizes for highly concurrent read workloads (eth_calls, state queries) with lock-free caches.
+# - "archive"   : optimizes for massive historical data scanning and sequential reads (eth_getLogs).
+node_type = "{{ .RocksDB.NodeType }}"
+`

cmd/cronosd/dbmigrate/migrate_rocksdb.go

@@ -11,7 +11,7 @@ import (
 
 // PrepareRocksDBOptions returns RocksDB options for migration
 func PrepareRocksDBOptions() interface{} {
-	return opendb.NewRocksdbOptions(nil, false)
+	return opendb.NewRocksdbOptions(nil, false, opendb.RocksDBTuneUpOptions{})
 }
 
 // openRocksDBForMigration opens a RocksDB database for migration (write mode)

cmd/cronosd/opendb/opendb_rocksdb.go

@@ -3,23 +3,59 @@
 package opendb
 
 import (
+	"fmt"
 	"path/filepath"
 	"runtime"
 	"strings"
 
 	dbm "github.com/cosmos/cosmos-db"
+	cronosconfig "github.com/crypto-org-chain/cronos/cmd/cronosd/config"
 	"github.com/linxGnu/grocksdb"
+	"github.com/spf13/cast"
 
 	"github.com/cosmos/cosmos-sdk/server/types"
 )
 
 // BlockCacheSize 3G block cache
 const BlockCacheSize = 3 << 30
 
-func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+type RocksDBTuneUpOptions struct {
+	EnableAsyncIo                bool
+	EnableAutoReadaheadSize      bool
+	EnableOptimizeForPointLookup bool
+	EnableHyperClockCache        bool
+	EnableDirectIOForCompaction  bool
+}
+
+func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
+
 	if backendType == dbm.RocksDBBackend {
-		return openRocksdb(filepath.Join(dataDir, "application.db"), false)
+		tuneUpOpts := RocksDBTuneUpOptions{}
+		if appOpts != nil {
+			if v := appOpts.Get("rocksdb.node_type"); v != nil {
+				cfg := cronosconfig.RocksDBConfig{NodeType: cast.ToString(v)}
+				if err := cfg.Validate(); err != nil {
+					return nil, fmt.Errorf("invalid rocksdb configuration: %w", err)
+				}
+				switch cfg.NodeType {
+				case cronosconfig.NodeTypeValidator:
+					tuneUpOpts.EnableOptimizeForPointLookup = true
+				case cronosconfig.NodeTypeRPC:
+					tuneUpOpts.EnableAutoReadaheadSize = true
+					tuneUpOpts.EnableOptimizeForPointLookup = true
+					tuneUpOpts.EnableHyperClockCache = true
+					tuneUpOpts.EnableDirectIOForCompaction = true
+				case cronosconfig.NodeTypeArchive:
+					tuneUpOpts.EnableAsyncIo = true
+					tuneUpOpts.EnableAutoReadaheadSize = true
+					tuneUpOpts.EnableHyperClockCache = true
+					tuneUpOpts.EnableOptimizeForPointLookup = true
+					tuneUpOpts.EnableDirectIOForCompaction = true
+				}
+			}
+		}
+		return openRocksdb(filepath.Join(dataDir, "application.db"), false, tuneUpOpts)
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
@@ -29,19 +65,31 @@ func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.D
 func OpenReadOnlyDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
 	if backendType == dbm.RocksDBBackend {
-		return openRocksdb(filepath.Join(dataDir, "application.db"), true)
+		return openRocksdb(filepath.Join(dataDir, "application.db"), true, RocksDBTuneUpOptions{})
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
 }
 
-func openRocksdb(dir string, readonly bool) (dbm.DB, error) {
-	opts, err := loadLatestOptions(dir)
+func openRocksdb(dir string, readonly bool, tuneUpOpts RocksDBTuneUpOptions) (dbm.DB, error) {
+	var cache *grocksdb.Cache
+	if tuneUpOpts.EnableHyperClockCache {
+		cache = grocksdb.NewHyperClockCache(BlockCacheSize, 0)
+	} else {
+		cache = grocksdb.NewLRUCache(BlockCacheSize)
+	}
+
+	// explicitly destroy cache to avoid memory leak. RocksDB holds a shared_ptr to the cache
+	// after table factory setup. Destroy() just decrements the refcount.
+	defer cache.Destroy()
+
+	opts, err := loadLatestOptions(dir, cache)
 	if err != nil {
 		return nil, err
 	}
 	// customize rocksdb options
-	opts = NewRocksdbOptions(opts, false)
+	opts = newRocksdbOptions(opts, false, tuneUpOpts, cache)
+	defer opts.Destroy()
 
 	var db *grocksdb.DB
 	if readonly {
@@ -54,38 +102,52 @@ func openRocksdb(dir string, readonly bool) (dbm.DB, error) {
 	}
 
 	ro := grocksdb.NewDefaultReadOptions()
+	if tuneUpOpts.EnableAsyncIo {
+		ro.SetAsyncIO(true)
+	}
+	if tuneUpOpts.EnableAutoReadaheadSize {
+		ro.SetAutoReadaheadSize(true)
+	}
 	wo := grocksdb.NewDefaultWriteOptions()
 	woSync := grocksdb.NewDefaultWriteOptions()
 	woSync.SetSync(true)
 	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
 }
 
 // loadLatestOptions try to load options from existing db, returns nil if not exists.
-func loadLatestOptions(dir string) (*grocksdb.Options, error) {
-	opts, err := grocksdb.LoadLatestOptions(dir, grocksdb.NewDefaultEnv(), true, grocksdb.NewLRUCache(BlockCacheSize))
+func loadLatestOptions(dir string, cache *grocksdb.Cache) (*grocksdb.Options, error) {
+	env := grocksdb.NewDefaultEnv()
+	defer env.Destroy()
+
+	opts, err := grocksdb.LoadLatestOptions(dir, env, true, cache)
 	if err != nil {
 		// not found is not an error
 		if strings.HasPrefix(err.Error(), "NotFound: ") {
 			return nil, nil
 		}
 		return nil, err
 	}
+	defer opts.Destroy()
 
 	cfNames := opts.ColumnFamilyNames()
 	cfOpts := opts.ColumnFamilyOpts()
 
 	for i := 0; i < len(cfNames); i++ {
 		if cfNames[i] == "default" {
-			return &cfOpts[i], nil
+			return cfOpts[i].Clone(), nil
 		}
 	}
 
-	return opts.Options(), nil
+	return opts.Options().Clone(), nil
 }
 
 // NewRocksdbOptions build options for `application.db`,
 // it overrides existing options if provided, otherwise create new one assuming it's a new database.
-func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Options {
+func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool, tuneUpOpts RocksDBTuneUpOptions) *grocksdb.Options {
+	return newRocksdbOptions(opts, sstFileWriter, tuneUpOpts, nil)
+}
+
+func newRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool, tuneUpOpts RocksDBTuneUpOptions, cache *grocksdb.Cache) *grocksdb.Options {
 	if opts == nil {
 		opts = grocksdb.NewDefaultOptions()
 		// only enable dynamic-level-bytes on new db, don't override for existing db
@@ -95,11 +157,29 @@ func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Opt
 	opts.IncreaseParallelism(runtime.NumCPU())
 	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
 	opts.SetTargetFileSizeMultiplier(2)
+	if tuneUpOpts.EnableOptimizeForPointLookup {
+		opts.SetMemTablePrefixBloomSizeRatio(0.02)
+		opts.SetMemtableWholeKeyFiltering(true)
+	}
+	if tuneUpOpts.EnableDirectIOForCompaction {
+		opts.SetUseDirectIOForFlushAndCompaction(true)
+	}
 
 	// block based table options
 	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
-
-	bbto.SetBlockCache(grocksdb.NewLRUCache(BlockCacheSize))
+	defer bbto.Destroy()
+
+	if cache != nil {
+		bbto.SetBlockCache(cache)
+	} else if tuneUpOpts.EnableHyperClockCache {
+		c := grocksdb.NewHyperClockCache(BlockCacheSize, 0)
+		defer c.Destroy()
+		bbto.SetBlockCache(c)
+	} else {
+		c := grocksdb.NewLRUCache(BlockCacheSize)
+		defer c.Destroy()
+		bbto.SetBlockCache(c)
+	}
 
 	// http://rocksdb.org/blog/2021/12/29/ribbon-filter.html
 	bbto.SetFilterPolicy(grocksdb.NewRibbonHybridFilterPolicy(9.9, 1))
@@ -117,6 +197,9 @@ func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Opt
 
 	// hash index is better for iavl tree which mostly do point lookup.
 	bbto.SetDataBlockIndexType(grocksdb.KDataBlockIndexTypeBinarySearchAndHash)
+	if tuneUpOpts.EnableOptimizeForPointLookup {
+		bbto.SetDataBlockHashRatio(0.75)
+	}
 
 	opts.SetBlockBasedTableFactory(bbto)