From ac3c46d0f44368ef1836f1a42fd98f8d73fc0630 Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:04:21 +0300 Subject: [PATCH 1/8] feat(parse): add default values constants --- internal/config/parse.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/config/parse.go b/internal/config/parse.go index 19dada4..8c64c46 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -9,6 +9,12 @@ import ( "gopkg.in/yaml.v3" ) +const ( + DefaultTimeout = 30 * time.Second + DefaultCacheTTL = 3 * time.Second + DefaultMaxConcurrent = 10 +) + var configPathFlag string // init sets flags with package initialization @@ -31,13 +37,13 @@ func GetPath() string { func (c *Config) applyDefaults() { if c.Global.Timeout == 0 { - c.Global.Timeout = 30 * time.Second + c.Global.Timeout = DefaultTimeout } if c.Global.CacheTTL == 0 { - c.Global.CacheTTL = 3 * time.Second + c.Global.CacheTTL = DefaultCacheTTL } if c.Global.MaxConcurrent == 0 { - c.Global.MaxConcurrent = 10 + c.Global.MaxConcurrent = DefaultMaxConcurrent } } From 036c78d6a68097a94e2e655c5b16b095f2992a9b Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:07:50 +0300 Subject: [PATCH 2/8] feat(cache): add metric output caching --- internal/collector/collector.go | 7 +++++++ internal/collector/processing.go | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 78f0ebb..173902c 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -12,10 +12,16 @@ type Executor interface { ExecuteCommand(ctx context.Context, command string, timeout time.Duration) (string, error) } +type cache struct { + output string + timestamp time.Time +} + type Collector struct { config *config.Config logger *slog.Logger executor Executor + cache map[string]cache } func NewCollector(cfg *config.Config, logger *slog.Logger, exec Executor) *Collector { @@ -23,6 +29,7 @@ func NewCollector(cfg *config.Config, logger *slog.Logger, exec Executor) *Colle config: cfg, logger: logger, executor: exec, + cache: make(map[string]cache), } } diff --git a/internal/collector/processing.go b/internal/collector/processing.go index 6e44746..4623e8c 100644 --- a/internal/collector/processing.go +++ b/internal/collector/processing.go @@ -6,12 +6,25 @@ import ( "pg-bash-exporter/internal/config" "strconv" "strings" + "time" ) // getCommandOutput executes command from metric config. // returns command output split into lines. // returns error if command fails to execute. func (c *Collector) getCommandOutput(metricConfig config.Metric) ([]string, error) { + chc, ok := c.cache[metricConfig.Command] + + ttl := c.config.Global.CacheTTL + if metricConfig.CacheTTL != config.DefaultCacheTTL { + ttl = metricConfig.CacheTTL + } + + if ok && time.Since(chc.timestamp) < ttl { + c.logger.Debug("cache taken", "command", metricConfig.Command) + return strings.Split(strings.TrimSpace(chc.output), "\n"), nil + } + timeout := c.config.Global.Timeout if metricConfig.Timeout > 0 { From 8e499b6b2962ea2af8e9d1a0544b4b98ed5a844f Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:47:12 +0300 Subject: [PATCH 3/8] fix: matching sub-metric with pattern condition --- internal/collector/processing.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/collector/processing.go b/internal/collector/processing.go index 4623e8c..7a767dd 100644 --- a/internal/collector/processing.go +++ b/internal/collector/processing.go @@ -102,8 +102,10 @@ func (c *Collector) collectComplicatedMetric(ch chan<- prometheus.Metric, metric } for _, subMetric := range metricConfig.SubMetrics { - if matched, err := c.matchPattern(line, subMetric.Match); !matched { - c.logger.Error("invalid regex patterin in sub-metric", "sub-metric", subMetric.Name, "pattern", subMetric.Match, "error", err) + if matched, err := c.matchPattern(line, subMetric.Match); !matched || err != nil { + if err != nil { + c.logger.Error("invalid regex patterin in sub-metric", "sub-metric", subMetric.Name, "pattern", subMetric.Match, "error", err) + } continue } From 1f17454ccd68533bcda028ca84ca6f4763c38fe9 Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:48:38 +0300 Subject: [PATCH 4/8] feat(cache): add concurrent safe metric output caching --- internal/cache/cache.go | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 internal/cache/cache.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..491076b --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,56 @@ +package cache + +import ( + "sync" + "time" +) + +type Cache struct { + mu sync.Mutex + items map[string]Item +} + +type Item struct { + Value string + Err error + Expiration time.Time +} + +func New() *Cache { + return &Cache{ + items: make(map[string]Item), + } +} + +func (c *Cache) Set(key, value string, err error, ttl time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + + var expiration time.Time + if ttl > 0 { + expiration = time.Now().Add(ttl) + } + + c.items[key] = Item{ + Value: value, + Err: err, + Expiration: expiration, + } +} + +func (c *Cache) Get(key string) (string, error, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + item, found := c.items[key] + if !found { + return "", nil, false + } + + if !item.Expiration.IsZero() && time.Now().After(item.Expiration) { + delete(c.items, key) + return "", nil, false + } + + return item.Value, item.Err, true +} From 70b30bab1a7721e4876b1cc991c9a6ef4a3dc154 Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:49:17 +0300 Subject: [PATCH 5/8] feat(collector): add cache usage in collector --- cmd/exporter/main.go | 5 ++++- internal/collector/collector.go | 12 ++++-------- internal/collector/processing.go | 12 +++++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cmd/exporter/main.go b/cmd/exporter/main.go index 2d00d5f..c0a4623 100644 --- a/cmd/exporter/main.go +++ b/cmd/exporter/main.go @@ -10,6 +10,7 @@ import ( "log/slog" "net/http" "os" + "pg-bash-exporter/internal/cache" "pg-bash-exporter/internal/collector" "pg-bash-exporter/internal/config" "pg-bash-exporter/internal/executor" @@ -50,9 +51,11 @@ func main() { slog.Info("Configuration loaded and logger initialized successfully") + cache := cache.New() + exec := &executor.BashExecutor{} - collector := collector.NewCollector(&cfg, slog.Default(), exec) + collector := collector.NewCollector(&cfg, slog.Default(), exec, cache) registry := prometheus.NewRegistry() registry.MustRegister(collector) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 173902c..4d4f31a 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -4,6 +4,7 @@ import ( "context" "github.com/prometheus/client_golang/prometheus" "log/slog" + "pg-bash-exporter/internal/cache" "pg-bash-exporter/internal/config" "time" ) @@ -12,24 +13,19 @@ type Executor interface { ExecuteCommand(ctx context.Context, command string, timeout time.Duration) (string, error) } -type cache struct { - output string - timestamp time.Time -} - type Collector struct { config *config.Config logger *slog.Logger executor Executor - cache map[string]cache + cache *cache.Cache } -func NewCollector(cfg *config.Config, logger *slog.Logger, exec Executor) *Collector { +func NewCollector(cfg *config.Config, logger *slog.Logger, exec Executor, cache *cache.Cache) *Collector { return &Collector{ config: cfg, logger: logger, executor: exec, - cache: make(map[string]cache), + cache: cache, } } diff --git a/internal/collector/processing.go b/internal/collector/processing.go index 7a767dd..86f5be1 100644 --- a/internal/collector/processing.go +++ b/internal/collector/processing.go @@ -6,23 +6,22 @@ import ( "pg-bash-exporter/internal/config" "strconv" "strings" - "time" ) // getCommandOutput executes command from metric config. // returns command output split into lines. // returns error if command fails to execute. func (c *Collector) getCommandOutput(metricConfig config.Metric) ([]string, error) { - chc, ok := c.cache[metricConfig.Command] + val, err, ok := c.cache.Get(metricConfig.Command) ttl := c.config.Global.CacheTTL - if metricConfig.CacheTTL != config.DefaultCacheTTL { + if metricConfig.CacheTTL > config.DefaultCacheTTL { ttl = metricConfig.CacheTTL } - if ok && time.Since(chc.timestamp) < ttl { + if ok { c.logger.Debug("cache taken", "command", metricConfig.Command) - return strings.Split(strings.TrimSpace(chc.output), "\n"), nil + return strings.Split(strings.TrimSpace(val), "\n"), err } timeout := c.config.Global.Timeout @@ -32,6 +31,9 @@ func (c *Collector) getCommandOutput(metricConfig config.Metric) ([]string, erro } out, err := c.executor.ExecuteCommand(context.Background(), metricConfig.Command, timeout) + + c.cache.Set(metricConfig.Command, out, err, ttl) + if err != nil { return nil, err } From 7b871e60218b8e42dd528771f4217f6a8fe9a70a Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 19:59:14 +0300 Subject: [PATCH 6/8] fix(collector_test): fixed for cache usage --- internal/collector/collector_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/collector/collector_test.go b/internal/collector/collector_test.go index b6eb993..30a3e6e 100644 --- a/internal/collector/collector_test.go +++ b/internal/collector/collector_test.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "io" "log/slog" + "pg-bash-exporter/internal/cache" "pg-bash-exporter/internal/config" "strings" "testing" @@ -328,7 +329,7 @@ filter_metric_filtered_sub 100 for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := slog.New(slog.NewJSONHandler(io.Discard, nil)) - collector := NewCollector(tc.config, logger, tc.executor) + collector := NewCollector(tc.config, logger, tc.executor, cache.New()) reg := prometheus.NewRegistry() reg.MustRegister(collector) From 0db73471a4c35bd822e788b75a2bdafc45992ff4 Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 20:00:14 +0300 Subject: [PATCH 7/8] fix: add generating cache key to prevent key collision in case of same commands in different metrics --- internal/collector/helpers.go | 5 +++++ internal/collector/processing.go | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/collector/helpers.go b/internal/collector/helpers.go index 72eca9a..b29a038 100644 --- a/internal/collector/helpers.go +++ b/internal/collector/helpers.go @@ -87,3 +87,8 @@ func (c *Collector) matchPattern(line, match string) (bool, error) { return matched, nil } + +// generateCacheKey creates a unique key for caching. +func generateCacheKey(metricName, command string) string { + return fmt.Sprintf("%s::%s", metricName, command) +} diff --git a/internal/collector/processing.go b/internal/collector/processing.go index 86f5be1..62d46b0 100644 --- a/internal/collector/processing.go +++ b/internal/collector/processing.go @@ -12,7 +12,8 @@ import ( // returns command output split into lines. // returns error if command fails to execute. func (c *Collector) getCommandOutput(metricConfig config.Metric) ([]string, error) { - val, err, ok := c.cache.Get(metricConfig.Command) + cacheKey := generateCacheKey(metricConfig.Name, metricConfig.Command) + val, err, ok := c.cache.Get(cacheKey) ttl := c.config.Global.CacheTTL if metricConfig.CacheTTL > config.DefaultCacheTTL { @@ -32,7 +33,7 @@ func (c *Collector) getCommandOutput(metricConfig config.Metric) ([]string, erro out, err := c.executor.ExecuteCommand(context.Background(), metricConfig.Command, timeout) - c.cache.Set(metricConfig.Command, out, err, ttl) + c.cache.Set(cacheKey, out, err, ttl) if err != nil { return nil, err From d265b271db754b5e2516ca5da1ad6363c31a2edd Mon Sep 17 00:00:00 2001 From: kvisidisi Date: Wed, 16 Jul 2025 20:15:34 +0300 Subject: [PATCH 8/8] test: cache testing --- internal/cache/cache_test.go | 104 +++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 internal/cache/cache_test.go diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..7a0c2f0 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,104 @@ +package cache + +import ( + "errors" + "testing" + "time" +) + +func TestCache(t *testing.T) { + testCases := []struct { + name string + key string + value string + err error + ttl time.Duration + expectFound bool + expectValue string + expectErr error + }{ + { + name: "set get", + key: "key1", + value: "value1", + err: nil, + ttl: 1 * time.Minute, + expectFound: true, + expectValue: "value1", + expectErr: nil, + }, + { + name: "get no existing value", + key: "nonexistent", + expectFound: false, + }, + { + name: "errors caching", + key: "error key", + value: "", + err: errors.New("some error"), + ttl: 1 * time.Minute, + expectFound: true, + expectValue: "", + expectErr: errors.New("some error"), + }, + { + name: "expired value", + key: "expired key", + value: "some value", + ttl: 1 * time.Millisecond, + expectFound: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cache := New() + + if tc.value != "" || tc.err != nil { + cache.Set(tc.key, tc.value, tc.err, tc.ttl) + } + + if tc.name == "expired value" { + time.Sleep(10 * time.Millisecond) + } + + val, err, found := cache.Get(tc.key) + + if found != tc.expectFound { + t.Errorf("expected found to be %v, but got %v", tc.expectFound, found) + } + + if val != tc.expectValue { + t.Errorf("expected value to be '%s', but got '%s'", tc.expectValue, val) + } + + if (err != nil && tc.expectErr == nil) || (err == nil && tc.expectErr != nil) || (err != nil && tc.expectErr != nil && err.Error() != tc.expectErr.Error()) { + t.Errorf("expected error to be '%v', but got '%v'", tc.expectErr, err) + } + }) + } +} + +func TestCache_UniqueKeys(t *testing.T) { + cache := New() + + key1 := "metric1::echo hello" + value1 := "output1" + + key2 := "metric2::echo hello" + value2 := "output2" + + cache.Set(key1, value1, nil, 1*time.Minute) + cache.Set(key2, value2, nil, 1*time.Minute) + + val, _, found := cache.Get(key1) + if !found || val != value1 { + t.Errorf("expected to find key '%s' with value '%s', but got '%s'", key1, value1, val) + } + + val, _, found = cache.Get(key2) + if !found || val != value2 { + t.Errorf("expected to find key '%s' with value '%s', but got '%s'", key2, value2, val) + } +}