From a477a6eb9caa6e57db90c131497980966c5d7ee1 Mon Sep 17 00:00:00 2001 From: Clint Sharp Date: Mon, 2 Mar 2026 20:22:10 -0800 Subject: [PATCH] Improve test coverage from 52% to 75% with simplified test helpers Add comprehensive tests across generator, internal, outputter, rater, run, and timer packages. Extract shared test helpers (setupGenTest, resetRunState) to reduce boilerplate. Consolidate token validation tests into table-driven subtests. Remove duplicate test and fix no-op assertions. Add CLAUDE.md with project conventions. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 102 +++ generator/generator_test.go | 196 +++++- generator/lua_test.go | 92 +++ internal/config_test.go | 1294 +++++++++++++++++++++++++++++++++++ internal/sample_test.go | 130 ++++ internal/share_test.go | 248 +++++++ outputter/send_test.go | 682 ++++++++++++++++++ outputter/write_test.go | 451 ++++++++++++ rater/rater_test.go | 144 ++++ run/run_test.go | 200 ++++++ run/runonce_test.go | 7 +- tests/generator/luaapi2.yml | 43 ++ timer/timer_test.go | 34 + 13 files changed, 3598 insertions(+), 25 deletions(-) create mode 100644 CLAUDE.md create mode 100644 outputter/send_test.go create mode 100644 outputter/write_test.go create mode 100644 run/run_test.go create mode 100644 tests/generator/luaapi2.yml diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4b48d2a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,102 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Gogen is an open source data generator for generating demo and test data, especially time series log and metric data. It's a Go CLI tool with an embedded Lua scripting engine, a Python AWS Lambda API backend, and a React/TypeScript UI. + +## Common Commands + +### Go (core CLI) + +```bash +make install # Build and install to $GOPATH/bin (default target) +make build # Cross-compile for linux, darwin, windows, wasm +make test # Run all Go tests: go test -v ./... +go test -v ./internal # Run tests for a single package +go test -v -run TestName ./internal # Run a single test +``` + +Version, git summary, build date, and GitHub OAuth credentials are injected via `-ldflags` in the Makefile. Always use `make install` rather than bare `go install`. + +Dependencies are vendored in `vendor/`. After adding deps, run `go mod vendor`. + +### Python API (`gogen-api/`) + +```bash +cd gogen-api +./start_dev.sh # Starts DynamoDB Local + MinIO via docker-compose, then SAM local API on port 4000 +./setup_local_db.sh # Seeds local DynamoDB schema +sam build && sam local start-api --port 4000 --docker-network lambda-local +./deploy_lambdas.sh # Deploy to AWS (requires credentials) +``` + +### UI (`ui/`) + +```bash +cd ui +npm run dev # Vite dev server (copies wasm from build/wasm/ first) +npm run build # Production build +npm test # Jest tests +``` + +## Architecture + +### Go Package Layout + +All packages are at the top level (no `cmd/` or `pkg/` convention): + +- **`main.go`** — CLI entry point using `urfave/cli.v1`. Maps CLI flags to `GOGEN_*` env vars. +- **`internal/`** — Core package. Config singleton, `Sample` struct, `Token` processing, API client, sharing. Imported as `config` throughout (`config "github.com/coccyx/gogen/internal"`). +- **`generator/`** — Reads `GenQueueItem` from channel, dispatches to sample-based or Lua generators. +- **`outputter/`** — Reads `OutQueueItem` from channel, dispatches to output destinations (stdout, file, HTTP, Kafka, network, devnull, buf). +- **`run/`** — Orchestrates the pipeline: timers -> generator worker pool -> outputter worker pool. +- **`timer/`** — One timer goroutine per Sample; handles backfill and realtime intervals. +- **`rater/`** — Controls event rate (config-based, time-of-day/weekday, kbps, Lua script). +- **`template/`** — Output formatting (raw, JSON, CSV, splunkhec, syslog, elasticsearch). +- **`logger/`** — Thin logrus wrapper with file/func/line context hook. + +### Data Flow + +``` +YAML/JSON Config -> internal.Config singleton (sync.Once) + -> [Timer goroutine per Sample] + -> GenQueueItem channel -> [Generator worker pool] + -> OutQueueItem channel -> [Outputter worker pool] + -> output destination +``` + +Concurrency is channel + goroutine worker pools. Worker counts set by `GeneratorWorkers` and `OutputWorkers` config fields. + +### Key Interfaces + +- `internal.Generator` — `Gen(item *GenQueueItem) error` +- `internal.Outputter` — `Send(events []map[string]string, sample *Sample, outputTemplate string) error` +- `internal.Rater` — `EventsPerInterval(s *Sample) int` + +### Config System + +Config is a **singleton** via `sync.Once`. Controlled by environment variables: +- `GOGEN_HOME`, `GOGEN_FULLCONFIG`, `GOGEN_CONFIG_DIR`, `GOGEN_SAMPLES_DIR` +- Remote configs fetched from `https://api.gogen.io` (override with `GOGEN_APIURL`) + +In tests, call `config.ResetConfig()` before `config.NewConfig()` to get a fresh instance. Tests commonly use `config.SetupFromString(yamlStr)` to inject inline YAML config. + +### gogen-api (Python Lambda) + +Each Lambda function is a separate `.py` file in `gogen-api/`. Backed by DynamoDB + S3. Originally Python 2.7, being updated to Python 3. AWS SAM template at `gogen-api/template.yaml`. + +### UI (React/TypeScript) + +Vite + React 18 + TypeScript + Tailwind CSS. Components in `src/components/`, pages in `src/pages/`, API clients in `src/api/`, types in `src/types/`. Tests use Jest + React Testing Library, placed adjacent to source as `.test.tsx`. + +## CI/CD + +GitHub Actions (`.github/workflows/ci.yml`): +- Push to `master`/`dev` or any PR: runs `make test`, then on `master`/`dev` cross-compiles, builds Docker, pushes artifacts to S3, deploys UI and Lambdas. +- Tag pushes (`v*.*.*`): full release workflow via `release.yml` — builds, creates GitHub release, pushes Docker images, deploys to production. + +## Lua Scripting + +Generators (`generator/lua.go`) and raters (`rater/script.go`) support embedded Lua via `gopher-lua` + `gopher-luar`. Lua state persists across calls within a run. diff --git a/generator/generator_test.go b/generator/generator_test.go index 73e2e9a..7568314 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -12,23 +12,25 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGenerator(t *testing.T) { - // Setup environment +// setupGenTest resets config, sets env vars, and returns common test fixtures. +func setupGenTest(t *testing.T, samplesDir string, seed int64) (func() time.Time, *rand.Rand) { + t.Helper() + config.ResetConfig() os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") os.Setenv("GOGEN_FULLCONFIG", "") - home := filepath.Join("..", "tests", "tokens") - os.Setenv("GOGEN_SAMPLES_DIR", home) + os.Setenv("GOGEN_SAMPLES_DIR", samplesDir) loc, _ := time.LoadLocation("Local") - source := rand.NewSource(0) - randgen := rand.New(source) - + randgen := rand.New(rand.NewSource(seed)) n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc) - now := func() time.Time { - return n - } + now := func() time.Time { return n } + return now, randgen +} + +func TestGenerator(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) - // gq := make(chan *config.GenQueueItem) oq := make(chan *config.OutQueueItem) s := tests.FindSampleInFile(home, "token-static") if s == nil { @@ -44,23 +46,171 @@ func TestGenerator(t *testing.T) { assert.Equal(t, "foo", oqi.Events[0]["_raw"]) } -func TestGeneratorCache(t *testing.T) { - // Setup environment +func TestGeneratorMultiPass(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + // Force MultiPass + s.SinglePass = false + + // Count > lines: tests the iters > 1 path in genMultiPass + gqi := &config.GenQueueItem{Count: len(s.Lines) + 2, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + err := genMultiPass(gqi) + assert.NoError(t, err) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines)+2, len(oqi.Events)) +} + +func TestGeneratorMultiPassRandomize(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 42) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + s.SinglePass = false + s.RandomizeEvents = true + + gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genMultiPass(gqi) + }() + + oqi := <-oq + assert.Equal(t, 5, len(oqi.Events)) +} + +func TestGeneratorSinglePassCountGtLines(t *testing.T) { + home := filepath.Join("..", "tests", "singlepass") + now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 0) + + c := config.NewConfig() + s := c.FindSampleByName("test1") + if s == nil { + t.Fatalf("Sample test1 not found") + } + assert.True(t, s.SinglePass) + + oq := make(chan *config.OutQueueItem) + // Count > lines: tests the iters > 1 singlepass path + gqi := &config.GenQueueItem{Count: len(s.Lines) + 3, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genSinglePass(gqi) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines)+3, len(oqi.Events)) +} + +func TestGeneratorSinglePassRandomize(t *testing.T) { + home := filepath.Join("..", "tests", "singlepass") + now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 42) + + c := config.NewConfig() + s := c.FindSampleByName("test1") + if s == nil { + t.Fatalf("Sample test1 not found") + } + assert.True(t, s.SinglePass) + s.RandomizeEvents = true + + oq := make(chan *config.OutQueueItem) + gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genSinglePass(gqi) + }() + + oqi := <-oq + assert.Equal(t, 5, len(oqi.Events)) +} + +func TestGeneratorStartWorker(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "token-static") + if s == nil { + t.Fatalf("Sample token-static not found") + } + + gq := make(chan *config.GenQueueItem) + gqs := make(chan int) + go Start(gq, gqs) + + // Send multiple items to test the "generator already set" path + for i := 0; i < 3; i++ { + gqi := &config.GenQueueItem{Count: 1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + gq <- gqi + oqi := <-oq + assert.Equal(t, "foo", oqi.Events[0]["_raw"]) + } + + close(gq) + select { + case <-gqs: + case <-time.After(5 * time.Second): + t.Fatal("Generator worker did not finish in time") + } +} + +func TestGeneratorCountMinusOne(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + // Count=-1 means "use all lines" + gqi := &config.GenQueueItem{Count: -1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + sg := sample{} + sg.Gen(gqi) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines), len(oqi.Events)) +} + +func TestPrimeRaterSetsRater(t *testing.T) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") - os.Setenv("GOGEN_FULLCONFIG", "") - home := filepath.Join("..", "tests", "tokens") - os.Setenv("GOGEN_SAMPLES_DIR", home) - loc, _ := time.LoadLocation("Local") - source := rand.NewSource(0) - randgen := rand.New(source) - n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc) - now := func() time.Time { - return n + s := &config.Sample{ + Name: "primerater_test", + Tokens: []config.Token{ + { + Name: "ratedtoken", + Type: "rated", + RaterString: "default", + }, + { + Name: "normaltoken", + Type: "choice", + }, + }, } - // gq := make(chan *config.GenQueueItem) + PrimeRater(s) + assert.NotNil(t, s.Tokens[0].Rater, "rated token should have rater set") +} + +func TestGeneratorCache(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + oq := make(chan *config.OutQueueItem) s := tests.FindSampleInFile(home, "token-static") if s == nil { diff --git a/generator/lua_test.go b/generator/lua_test.go index dabb53b..cc3534c 100644 --- a/generator/lua_test.go +++ b/generator/lua_test.go @@ -288,6 +288,98 @@ func TestSetTime(t *testing.T) { testLuaGen(t, s, gen, "2001-10-20 11:59:59.000100") } +func TestLuaRound(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("roundTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + found := false + var token config.Token + for _, tk := range gen.tokens { + if tk.Name == "rounded" { + found = true + token = tk + } + } + assert.True(t, found, "Couldn't find token 'rounded' in sample roundTest") + assert.Equal(t, "3.14", token.Replacement) +} + +func TestLuaLogInfo(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("logInfoTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + found := false + var token config.Token + for _, tk := range gen.tokens { + if tk.Name == "logged" { + found = true + token = tk + } + } + assert.True(t, found, "Couldn't find token 'logged' in sample logInfoTest") + assert.Equal(t, "ok", token.Replacement) +} + +func TestRemoveToken(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("removeTokenTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + + foundKeeper := false + foundRemover := false + for _, tk := range gen.tokens { + if tk.Name == "keeper" { + foundKeeper = true + } + if tk.Name == "remover" { + foundRemover = true + } + } + assert.True(t, foundKeeper, "Token 'keeper' should still be present") + assert.False(t, foundRemover, "Token 'remover' should have been removed") +} + +func TestSendEvent(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("sendEventTest") + gen := new(luagen) + testLuaGen(t, s, gen, "sent via sendEvent") +} + func testLuaGen(t *testing.T, s *config.Sample, gen *luagen, expected string) { oq, err := runLuaGen(t, s, gen) timeout := make(chan bool, 1) diff --git a/internal/config_test.go b/internal/config_test.go index 8e565af..5b37c96 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -2,6 +2,8 @@ package internal import ( "math/rand" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" @@ -356,3 +358,1295 @@ func TestParseWebConfig(t *testing.T) { assert.Equal(t, "timestamp", tsToken.Type) assert.Equal(t, "%d/%b/%Y %H:%M:%S:%L", tsToken.Replacement) } + +func TestFindRater(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + r := c.FindRater("testconfigrater") + assert.NotNil(t, r) + assert.Equal(t, "testconfigrater", r.Name) + + CleanupConfigAndEnvironment() +} + +func TestFindRaterNotFound(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + r := c.FindRater("nonexistentrater") + assert.Nil(t, r) + + CleanupConfigAndEnvironment() +} + +func TestFindSampleByNameNotFound(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + s := c.FindSampleByName("nonexistentsample") + assert.Nil(t, s) + + CleanupConfigAndEnvironment() +} + +func TestClean(t *testing.T) { + configStr := ` +samples: + - name: enabled-sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test + - name: disabled-sample + disabled: true + interval: 1 + count: 1 + lines: + - _raw: test +` + ResetConfig() + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + + // After Clean(), only enabled real samples should remain + found := false + for _, s := range c.Samples { + if s.Name == "disabled-sample" { + found = true + } + } + assert.False(t, found, "disabled sample should be removed by Clean()") + + foundEnabled := false + for _, s := range c.Samples { + if s.Name == "enabled-sample" { + foundEnabled = true + } + } + assert.True(t, foundEnabled, "enabled sample should remain after Clean()") +} + +func TestParseBeginEndWithEndIntervals(t *testing.T) { + s := &Sample{ + Name: "test", + EndIntervals: 3, + Interval: 5, + } + + ParseBeginEnd(s) + + assert.Equal(t, "-15s", s.Begin) + assert.Equal(t, "now", s.End) + assert.False(t, s.Realtime) + assert.False(t, s.BeginParsed.IsZero()) + assert.False(t, s.EndParsed.IsZero()) +} + +func TestParseBeginEndEmptyEnd(t *testing.T) { + s := &Sample{ + Name: "test", + End: "", + } + + ParseBeginEnd(s) + + // Empty end means realtime + assert.True(t, s.Realtime) + assert.True(t, s.EndParsed.IsZero()) +} + +func TestParseBeginEndBeginOverridesRealtime(t *testing.T) { + s := &Sample{ + Name: "test", + Begin: "-60s", + End: "", + } + + ParseBeginEnd(s) + + // Begin set without endIntervals: sets Realtime to false via parsing + assert.False(t, s.Realtime) + assert.False(t, s.BeginParsed.IsZero()) +} + +func TestSetupFromFile(t *testing.T) { + SetupFromFile("/tmp/testfile.yml") + defer CleanupConfigAndEnvironment() + + assert.Equal(t, "..", os.Getenv("GOGEN_HOME")) + assert.Equal(t, "1", os.Getenv("GOGEN_ALWAYS_REFRESH")) + assert.Equal(t, "/tmp/testfile.yml", os.Getenv("GOGEN_FULLCONFIG")) +} + +func TestSetupSystemTokensSplunkHEC(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: splunkhec +samples: + - name: hectokensample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("hectokensample") + assert.NotNil(t, s) + + // Should have a _time token added by SetupSystemTokens + foundTime := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "epochtimestamp", tk.Type) + } + } + assert.True(t, foundTime, "splunkhec should add _time token") +} + +func TestSetupSystemTokensElasticsearch(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: elasticsearch +samples: + - name: estokensample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("estokensample") + assert.NotNil(t, s) + + foundTimestamp := false + for _, tk := range s.Tokens { + if tk.Name == "@timestamp" { + foundTimestamp = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTimestamp, "elasticsearch should add @timestamp token") +} + +func TestSetupSystemTokensRFC3164(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: rfc3164 +samples: + - name: rfc3164sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: syslog event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rfc3164sample") + assert.NotNil(t, s) + + foundTime := false + foundPriority := false + foundHost := false + foundTag := false + foundPid := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTime, "rfc3164 should add _time token") + + // Check that syslog fields were added to lines + if len(s.Lines) > 0 { + if _, ok := s.Lines[0]["priority"]; ok { + foundPriority = true + } + if _, ok := s.Lines[0]["host"]; ok { + foundHost = true + } + if _, ok := s.Lines[0]["tag"]; ok { + foundTag = true + } + if _, ok := s.Lines[0]["pid"]; ok { + foundPid = true + } + } + assert.True(t, foundPriority, "rfc3164 should add priority field") + assert.True(t, foundHost, "rfc3164 should add host field") + assert.True(t, foundTag, "rfc3164 should add tag field") + assert.True(t, foundPid, "rfc3164 should add pid field") +} + +func TestSetupSystemTokensRFC5424(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: rfc5424 +samples: + - name: rfc5424sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: syslog5424 event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rfc5424sample") + assert.NotNil(t, s) + + foundTime := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTime, "rfc5424 should add _time token") + + if len(s.Lines) > 0 { + _, hasAppName := s.Lines[0]["appName"] + assert.True(t, hasAppName, "rfc5424 should add appName field") + } +} + +func TestBuildConfigDefaults(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: defaultsample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + + // Check defaults were applied + assert.Equal(t, 1, c.Global.GeneratorWorkers) + assert.Equal(t, 1, c.Global.OutputWorkers) + assert.Equal(t, "stdout", c.Global.Output.Outputter) + assert.Equal(t, "raw", c.Global.Output.OutputTemplate) + assert.Equal(t, 5, c.Global.Output.BackupFiles) + assert.NotZero(t, c.Global.Output.MaxBytes) + assert.NotZero(t, c.Global.Output.BufferBytes) + assert.NotZero(t, c.Global.Output.Timeout) +} + +func TestValidateDisabledNoLines(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: nolines + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // Sample with no lines should be disabled and cleaned away + s := c.FindSampleByName("nolines") + assert.Nil(t, s, "sample with no lines should be removed") +} + +func TestConvertUTC(t *testing.T) { + ResetConfig() + + configStr := ` +global: + utc: true +samples: + - name: utctest + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + _ = NewConfig() + + now := time.Now() + utcTime := convertUTC(now) + assert.Equal(t, now.UTC(), utcTime) +} + +func TestSampleNow(t *testing.T) { + s := &Sample{ + Realtime: true, + } + beforeCall := time.Now() + result := s.Now() + afterCall := time.Now() + assert.True(t, !result.Before(beforeCall) && !result.After(afterCall), + "Realtime Now() should return current time") + + fixedTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s.Realtime = false + s.Current = fixedTime + result = s.Now() + assert.Equal(t, fixedTime, result) +} + +func TestReadSamplesDir(t *testing.T) { + ResetConfig() + + // Use the existing test samples directory + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", "") + os.Setenv("GOGEN_SAMPLES_DIR", filepath.Join("..", "tests", "tokens")) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // Should have loaded samples from the tokens test directory + assert.NotEmpty(t, c.Samples, "should load samples from samples dir") +} + +func TestParseFileConfigJSON(t *testing.T) { + ResetConfig() + + // Create a JSON config file with the Config struct format + dir := t.TempDir() + jsonFile := filepath.Join(dir, "test.json") + jsonContent := `{"samples": [{"name": "jsonsample", "interval": 1, "count": 1, "endIntervals": 1, "lines": [{"_raw": "json test"}]}]}` + os.WriteFile(jsonFile, []byte(jsonContent), 0644) + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", jsonFile) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + assert.NotEmpty(t, c.Samples, "should load samples from JSON config") +} + +func TestNegativeCacheIntervals(t *testing.T) { + ResetConfig() + + configStr := ` +global: + cacheIntervals: -5 +samples: + - name: cachesample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + assert.Equal(t, 0, c.Global.CacheIntervals, "negative cacheIntervals should be clamped to 0") +} + +func TestReadSamplesDirSampleFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + // Create a .sample file + err := os.WriteFile(filepath.Join(dir, "test.sample"), []byte("line one\nline two\nline three\n"), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + // Should have loaded one sample with 3 lines + found := false + for _, s := range c.Samples { + if s.Name == "test.sample" { + found = true + assert.True(t, s.Disabled, ".sample files should be disabled by default") + assert.Equal(t, 3, len(s.Lines)) + assert.Equal(t, "line one", s.Lines[0]["_raw"]) + assert.Equal(t, "line two", s.Lines[1]["_raw"]) + assert.Equal(t, "line three", s.Lines[2]["_raw"]) + } + } + assert.True(t, found, "should find test.sample") +} + +func TestReadSamplesDirCSVFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + // Create a .csv file with header + csvContent := "name,city,state\nalice,NYC,NY\nbob,LA,CA\n" + err := os.WriteFile(filepath.Join(dir, "test.csv"), []byte(csvContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + found := false + for _, s := range c.Samples { + if s.Name == "test.csv" { + found = true + assert.True(t, s.Disabled, ".csv files should be disabled by default") + assert.Equal(t, 2, len(s.Lines)) + assert.Equal(t, "alice", s.Lines[0]["name"]) + assert.Equal(t, "NYC", s.Lines[0]["city"]) + assert.Equal(t, "NY", s.Lines[0]["state"]) + assert.Equal(t, "bob", s.Lines[1]["name"]) + } + } + assert.True(t, found, "should find test.csv") +} + +func TestReadSamplesDirYAMLFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + yamlContent := `name: yamlsample +interval: 1 +count: 1 +lines: + - _raw: yaml test line +` + err := os.WriteFile(filepath.Join(dir, "yamltest.yml"), []byte(yamlContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + found := false + for _, s := range c.Samples { + if s.Name == "yamlsample" { + found = true + assert.True(t, s.realSample) + } + } + assert.True(t, found, "should find yamlsample") +} + +func TestReadSamplesDirEmptyDir(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + // Should not crash and should have no samples + assert.Empty(t, c.Samples) +} + +func TestReadGeneratorFallbackPath(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + genDir := filepath.Join(dir, "generators") + os.MkdirAll(genDir, 0755) + + // Create generator script in the fallback directory + scriptContent := `-- test generator\nsetToken("test", "value")\n` + err := os.WriteFile(filepath.Join(genDir, "testgen.lua"), []byte(scriptContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{ConfigDir: dir}} + g := &GeneratorConfig{Name: "testgen", FileName: "testgen.lua"} + + err = c.readGenerator(dir, g) + assert.NoError(t, err) + assert.Contains(t, g.Script, "test generator") +} + +func TestReadGeneratorNotFound(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + c := &Config{cc: ConfigConfig{ConfigDir: dir}} + g := &GeneratorConfig{Name: "missing", FileName: "nonexistent.lua"} + + err := c.readGenerator(dir, g) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Cannot find generator file") +} + +func TestValidateTokenRandomString(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: randstring + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: rs + format: template + token: $rs$ + type: random + replacement: string + length: 10 + lines: + - _raw: $rs$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("randstring") + assert.NotNil(t, s, "sample with valid random string token should not be disabled") +} + +func TestValidateTokenRandomStringZeroLength(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: randstringbad + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: rs + format: template + token: $rs$ + type: random + replacement: string + length: 0 + lines: + - _raw: $rs$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("randstringbad") + assert.Nil(t, s, "sample with zero-length random string should be disabled") +} + +func TestValidateTokenReplacementTypes(t *testing.T) { + tests := []struct { + name string + replacement string + extra string + valid bool + }{ + {"hex", "hex", "length: 5", true}, + {"guid", "guid", "", true}, + {"ipv4", "ipv4", "", true}, + {"ipv6", "ipv6", "", true}, + {"invalid", "invalid_replacement_xyz", "", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ResetConfig() + extra := "" + if tc.extra != "" { + extra = "\n " + tc.extra + } + configStr := ` +samples: + - name: ` + tc.name + ` + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: tk + format: template + token: $tk$ + type: random + replacement: ` + tc.replacement + extra + ` + lines: + - _raw: $tk$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName(tc.name) + if tc.valid { + assert.NotNil(t, s, "%s should not be disabled", tc.name) + } else { + assert.Nil(t, s, "%s should be disabled", tc.name) + } + }) + } +} + +func TestValidateTokenScript(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: scripttest + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: sc + format: template + token: $sc$ + type: script + init: + myvar: "42" + scriptSrc: | + return "hello" + lines: + - _raw: $sc$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("scripttest") + assert.NotNil(t, s, "sample with script token should not be disabled") + // Check that script token has mutex + for _, tk := range s.Tokens { + if tk.Name == "sc" { + assert.NotNil(t, tk.mutex, "script token should have mutex initialized") + } + } +} + +func TestValidateNoInterval(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: nointerval + count: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("nointerval") + assert.NotNil(t, s) + assert.Equal(t, 1, s.EndIntervals, "no interval should auto-set endIntervals to 1") +} + +func TestValidateEmptyName(t *testing.T) { + ResetConfig() + + c := &Config{ + Global: Global{ + Output: Output{ + Outputter: "stdout", + OutputTemplate: "raw", + }, + }, + } + s := &Sample{ + realSample: true, + Name: "", + Lines: []map[string]string{{"_raw": "test"}}, + } + c.validate(s) + assert.True(t, s.Disabled, "sample with empty name should be disabled") +} + +func TestValidateRaterWithIntValues(t *testing.T) { + ResetConfig() + configStr := ` +raters: + - name: testrater + type: config + options: + HourOfDay: + 0: 1 + 12: 2 + DayOfWeek: + 0: 1.5 + 6: 0.5 +samples: + - name: ratertest + interval: 1 + count: 1 + endIntervals: 1 + rater: testrater + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + r := c.FindRater("testrater") + assert.NotNil(t, r) + // HourOfDay should be converted to map[int]float64 + hod, ok := r.Options["HourOfDay"].(map[int]float64) + assert.True(t, ok, "HourOfDay should be map[int]float64") + assert.Equal(t, 1.0, hod[0]) + assert.Equal(t, 2.0, hod[12]) +} + +func TestValidateWeightedChoice(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: weightsource + disabled: true + lines: + - value: alpha + _weight: "3" + - value: beta + _weight: "7" + - name: weightuser + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: wt + format: template + token: $wt$ + type: weightedChoice + sample: weightsource + srcField: value + lines: + - _raw: $wt$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("weightuser") + assert.NotNil(t, s) + for _, tk := range s.Tokens { + if tk.Name == "wt" { + assert.NotEmpty(t, tk.WeightedChoice, "should have weighted choices resolved") + assert.Equal(t, 2, len(tk.WeightedChoice)) + } + } +} + +func TestValidateTokenSampleResolution(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: choices + disabled: true + lines: + - _raw: alpha + - _raw: beta + - _raw: gamma + - name: resolver + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: pick + format: template + token: $pick$ + type: choice + sample: choices + lines: + - _raw: $pick$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("resolver") + assert.NotNil(t, s) + for _, tk := range s.Tokens { + if tk.Name == "pick" { + assert.Equal(t, 3, len(tk.Choice), "should resolve 3 choices from sample") + assert.Contains(t, tk.Choice, "alpha") + assert.Contains(t, tk.Choice, "beta") + assert.Contains(t, tk.Choice, "gamma") + } + } +} + +func TestValidateExportMode(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + configFile := filepath.Join(dir, "export.yml") + os.WriteFile(configFile, []byte(` +samples: + - name: exportsample + interval: 1 + count: 1 + lines: + - _raw: test +`), 0644) + + cc := ConfigConfig{ + FullConfig: configFile, + Export: true, + } + c := BuildConfig(cc) + + // In export mode, defaults should NOT be set + assert.Equal(t, 0, c.Global.GeneratorWorkers, "export mode should not set defaults") + assert.Equal(t, "", c.Global.Output.Outputter, "export mode should not set output defaults") +} + +func TestMergeMixConfig(t *testing.T) { + c := &Config{} + nc := &Config{ + Samples: []*Sample{ + {Name: "mixsample", Count: 5, Interval: 2}, + }, + } + m := &Mix{ + Count: 10, + Interval: 3, + Begin: "-60s", + End: "now", + } + c.mergeMixConfig(nc, m) + + assert.Equal(t, 1, len(c.Samples)) + assert.Equal(t, "mixsample", c.Samples[0].Name) + assert.Equal(t, 10, c.Samples[0].Count) + assert.Equal(t, 3, c.Samples[0].Interval) +} + +func TestParseFileConfigYAMLError(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + badFile := filepath.Join(dir, "bad.yml") + // Invalid YAML: tabs mixed with spaces in wrong places + os.WriteFile(badFile, []byte("{\n bad yaml content: [unclosed\n"), 0644) + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, badFile) + // parseFileConfig logs errors but returns nil + assert.NoError(t, err) +} + +func TestParseFileConfigJSONError(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + badFile := filepath.Join(dir, "bad.json") + os.WriteFile(badFile, []byte("{invalid json"), 0644) + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, badFile) + assert.NoError(t, err) +} + +func TestParseFileConfigNotExists(t *testing.T) { + ResetConfig() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, "/nonexistent/path/file.yml") + assert.Error(t, err) +} + +func TestParseWebConfigSuccess(t *testing.T) { + ResetConfig() + + yamlContent := ` +name: websample +interval: 1 +count: 1 +lines: + - _raw: web test +` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(yamlContent)) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + assert.NoError(t, err) + assert.Equal(t, "websample", s.Name) +} + +func TestParseWebConfigJSONFallback(t *testing.T) { + ResetConfig() + + jsonContent := `{"name": "jsonsample", "interval": 1}` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(jsonContent)) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + assert.NoError(t, err) + assert.Equal(t, "jsonsample", s.Name) +} + +func TestParseWebConfigBadContent(t *testing.T) { + ResetConfig() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("<<>>")) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + // JSON fallback parse returns an error for non-JSON content + assert.Error(t, err) + assert.Equal(t, "", s.Name, "garbage content should not set sample name") +} + +func TestMergeMixConfigDuplicate(t *testing.T) { + c := &Config{ + Samples: []*Sample{ + {Name: "existing"}, + }, + } + nc := &Config{ + Samples: []*Sample{ + {Name: "existing", Count: 5}, + }, + } + m := &Mix{} + c.mergeMixConfig(nc, m) + + // Should not add duplicate + assert.Equal(t, 1, len(c.Samples)) +} + +func TestGetAPIURLDefault(t *testing.T) { + os.Unsetenv("GOGEN_APIURL") + url := getAPIURL() + assert.Equal(t, "https://api.gogen.io", url) +} + +func TestGetAPIURLCustom(t *testing.T) { + os.Setenv("GOGEN_APIURL", "http://localhost:4000") + defer os.Unsetenv("GOGEN_APIURL") + url := getAPIURL() + assert.Equal(t, "http://localhost:4000", url) +} + +func TestValidateFromSample(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: sourcesample + disabled: true + lines: + - _raw: source line 1 + - _raw: source line 2 + - name: copiedsample + fromSample: sourcesample + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("copiedsample") + assert.NotNil(t, s) + assert.Len(t, s.Lines, 2, "copiedsample should have lines from sourcesample") +} + +func TestNewGeneratorStateNumericInit(t *testing.T) { + s := &Sample{ + CustomGenerator: &GeneratorConfig{ + Init: map[string]string{ + "count": "42", + "rate": "3.14", + "label": "hello", + }, + }, + Lines: []map[string]string{ + {"_raw": "line1", "host": "h1"}, + {"_raw": "line2", "host": "h2"}, + }, + } + + gs := NewGeneratorState(s) + assert.NotNil(t, gs.LuaState) + assert.NotNil(t, gs.LuaLines) + + // Numeric values should be stored as LNumber + countVal := gs.LuaState.RawGetString("count") + assert.NotNil(t, countVal) + + // String values should be stored as LString + labelVal := gs.LuaState.RawGetString("label") + assert.NotNil(t, labelVal) + + // Lines table should have entries + assert.Equal(t, 2, gs.LuaLines.Len()) +} + +func TestNewGeneratorStateEmptyInit(t *testing.T) { + s := &Sample{ + CustomGenerator: &GeneratorConfig{ + Init: map[string]string{}, + }, + Lines: []map[string]string{}, + } + + gs := NewGeneratorState(s) + assert.NotNil(t, gs.LuaState) + assert.NotNil(t, gs.LuaLines) + assert.Equal(t, 0, gs.LuaLines.Len()) +} + +func TestBuildConfigExportMode(t *testing.T) { + ResetConfig() + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + home := filepath.Join("..", "tests", "tokens") + os.Setenv("GOGEN_SAMPLES_DIR", home) + + cc := ConfigConfig{ + SamplesDir: home, + Home: "..", + Export: true, + } + c := BuildConfig(cc) + assert.NotNil(t, c) + // In export mode, samples should have lines populated inline + for _, s := range c.Samples { + if s.Name == "tokens" { + assert.Greater(t, len(s.Lines), 0) + } + } +} + +func TestBuildConfigWithGlobalFile(t *testing.T) { + ResetConfig() + globalFile := filepath.Join("..", "tests", "rater", "defaultrater.yml") + cc := ConfigConfig{ + FullConfig: globalFile, + Home: "..", + } + c := BuildConfig(cc) + assert.NotNil(t, c) +} + +func TestValidateInvalidEarliestTime(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: badtime + description: "Bad earliest time" + earliest: "not_a_valid_time_string!!!" + latest: now + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("badtime") + // With invalid earliest, EarliestParsed should default to 0 + assert.Equal(t, time.Duration(0), s.EarliestParsed) +} + +func TestValidateInvalidLatestTime(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: badlatest + description: "Bad latest time" + earliest: now + latest: "not_a_valid_time_string!!!" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("badlatest") + // With invalid latest, LatestParsed should default to 0 + assert.Equal(t, time.Duration(0), s.LatestParsed) +} + +func TestNewConfigNoGogenHome(t *testing.T) { + ResetConfig() + os.Unsetenv("GOGEN_HOME") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Unsetenv("GOGEN_FULLCONFIG") + os.Unsetenv("GOGEN_CONFIG_DIR") + os.Unsetenv("GOGEN_SAMPLES_DIR") + + c := NewConfig() + assert.NotNil(t, c) + // When GOGEN_HOME is not set, it should default to "." + assert.Equal(t, ".", os.Getenv("GOGEN_HOME")) +} + +func TestValidateNoLinesDisablesSample(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: nolines + description: "Sample with no lines" + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // After Clean(), disabled samples are removed + // So FindSampleByName should return an empty sample (not in the list) + found := false + for _, s := range c.Samples { + if s.Name == "nolines" { + found = true + } + } + assert.False(t, found, "disabled sample with no lines should be removed by Clean()") +} + +func TestValidateRatedTokenDefaultRater(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: rated_test + description: "Rated token default rater" + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: myrated + format: template + type: rated + replacement: int + lower: 0 + upper: 100 + lines: + - _raw: value=$myrated$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rated_test") + // Rated token with no raterString should default to "default" + for _, tok := range s.Tokens { + if tok.Name == "myrated" { + assert.Equal(t, "default", tok.RaterString) + } + } +} + +func TestValidateLuaGenerator(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: luagen + description: "Lua generator sample" + generator: mygen + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +generators: + - name: mygen + script: | + lines = getLines() + return send(lines) +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("luagen") + assert.NotNil(t, s) + assert.Equal(t, "mygen", s.Generator) + assert.NotNil(t, s.CustomGenerator) +} diff --git a/internal/sample_test.go b/internal/sample_test.go index b05e008..2f66e89 100644 --- a/internal/sample_test.go +++ b/internal/sample_test.go @@ -230,6 +230,136 @@ func benchmarkToken(conf string, i int, b *testing.B) { } } +// mockRater implements Rater for testing rated tokens +type mockRater struct { + rate float64 +} + +func (m *mockRater) EventRate(s *Sample, now time.Time, count int) float64 { return m.rate } +func (m *mockRater) TokenRate(t Token, now time.Time) float64 { return m.rate } + +func TestGenReplacementRatedInt(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedint", + Type: "rated", + Replacement: "int", + Lower: 10, + Upper: 20, + Rater: &mockRater{rate: 2.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + // With rate=2.0, value should be roughly doubled + val, _ := fmt.Sscanf(result, "%d", new(int)) + assert.Equal(t, 1, val, "should parse as an integer") +} + +func TestGenReplacementRatedIntEqualBounds(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedintequal", + Type: "rated", + Replacement: "int", + Lower: 5, + Upper: 5, + Rater: &mockRater{rate: 1.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "5", result) +} + +func TestGenReplacementRatedFloat(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedfloat", + Type: "rated", + Replacement: "float", + Lower: 1, + Upper: 10, + Precision: 2, + Rater: &mockRater{rate: 1.5}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + // Should be a float string with 2 decimal places + assert.Contains(t, result, ".") +} + +func TestGenReplacementRatedFloatEqualBounds(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedfloatequal", + Type: "rated", + Replacement: "float", + Lower: 5, + Upper: 5, + Precision: 2, + Rater: &mockRater{rate: 1.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "5.00", result) +} + +func TestGenReplacementFieldChoice(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "fieldchoice", + Type: "fieldChoice", + SrcField: "city", + FieldChoice: []map[string]string{ + {"city": "NYC", "state": "NY"}, + {"city": "LA", "state": "CA"}, + }, + } + result, choice, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.True(t, result == "NYC" || result == "LA") + assert.True(t, choice >= 0) + + // Test with specific choice + result, _, err = token.GenReplacement(0, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "NYC", result) +} + +func TestGenReplacementInvalidType(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "badtype", + Type: "nonexistenttype", + } + _, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid type") +} + func BenchmarkReplacement(b *testing.B) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") diff --git a/internal/share_test.go b/internal/share_test.go index 1793336..188e974 100644 --- a/internal/share_test.go +++ b/internal/share_test.go @@ -146,3 +146,251 @@ func TestSharePullFile(t *testing.T) { _, err = os.Stat(filepath.Join(os.ExpandEnv("$GOGEN_TMPDIR"), ".configcache_testuser%2Ftestconfig")) assert.NoError(t, err, "Couldn't find cache file") } + +func TestSharePullShortName(t *testing.T) { + // Test Pull with a short name (no "/" in gogen string) + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "shortname", + Name: "shortname", + Description: "Short name test", + Owner: "testuser", + Version: 1, + Config: "sample: test\nname: shortname", + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("shortname", dir, false) + _, err := os.Stat(filepath.Join(dir, "shortname.yml")) + assert.NoError(t, err, "Couldn't find file shortname.yml") +} + +func TestSharePullFileCached(t *testing.T) { + // Test PullFile when cache exists and version matches → uses cached content + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/cached", + Name: "cached", + Owner: "testuser", + Version: 5, + Config: "should not be used", + }, nil + } + + tmpdir := t.TempDir() + os.Setenv("GOGEN_TMPDIR", tmpdir) + defer os.Unsetenv("GOGEN_TMPDIR") + + // Pre-create version cache with matching version + versionCacheFile := filepath.Join(tmpdir, ".versioncache_testuser%2Fcached") + os.WriteFile(versionCacheFile, []byte("5"), 0644) + + // Pre-create config cache with different content + cacheFile := filepath.Join(tmpdir, ".configcache_testuser%2Fcached") + os.WriteFile(cacheFile, []byte("cached config content"), 0644) + + outFile := filepath.Join(tmpdir, "output.yml") + PullFile("testuser/cached", outFile) + + // Should use cached content, not the API response + data, err := os.ReadFile(outFile) + assert.NoError(t, err) + assert.Equal(t, "cached config content", string(data)) +} + +func TestSharePullFileVersionMismatch(t *testing.T) { + // Test PullFile when cache version doesn't match → uses API content and updates cache + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/mismatch", + Name: "mismatch", + Owner: "testuser", + Version: 10, + Config: "fresh api content", + }, nil + } + + tmpdir := t.TempDir() + os.Setenv("GOGEN_TMPDIR", tmpdir) + defer os.Unsetenv("GOGEN_TMPDIR") + + // Pre-create version cache with OLD version + versionCacheFile := filepath.Join(tmpdir, ".versioncache_testuser%2Fmismatch") + os.WriteFile(versionCacheFile, []byte("5"), 0644) + + // Pre-create config cache with old content + cacheFile := filepath.Join(tmpdir, ".configcache_testuser%2Fmismatch") + os.WriteFile(cacheFile, []byte("old cached content"), 0644) + + outFile := filepath.Join(tmpdir, "output.yml") + PullFile("testuser/mismatch", outFile) + + // Should use API content since version doesn't match + data, err := os.ReadFile(outFile) + assert.NoError(t, err) + assert.Equal(t, "fresh api content", string(data)) + + // Cache files should be updated + versionData, _ := os.ReadFile(versionCacheFile) + assert.Equal(t, "10", string(versionData)) + + cachedData, _ := os.ReadFile(cacheFile) + assert.Equal(t, "fresh api content", string(cachedData)) +} + +func TestSharePullWithDeconstructCSV(t *testing.T) { + // Test deconstructConfig with CSV fieldChoice tokens + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: csvtest + description: CSV deconstruct test + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: city + format: template + type: fieldChoice + field: _raw + srcField: city + sample: markets.csv + fieldChoice: + - city: NYC + state: NY + - city: LA + state: CA + lines: + - _raw: city=$city$ +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/csvtest", + Name: "csvtest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/csvtest", dir, true) + _, err := os.Stat(filepath.Join(dir, "samples", "markets.csv")) + assert.NoError(t, err, "Couldn't find samples/markets.csv") + _, err = os.Stat(filepath.Join(dir, "samples", "csvtest.yml")) + assert.NoError(t, err, "Couldn't find samples/csvtest.yml") +} + +func TestSharePullWithDeconstructGenerator(t *testing.T) { + // Test deconstructConfig with generator that has a fileName + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: gentest + description: Generator deconstruct test + generator: mygen + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +generators: + - name: mygen + fileName: /path/to/mygen.lua + script: | + lines = getLines() + return send(lines) +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/gentest", + Name: "gentest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/gentest", dir, true) + _, err := os.Stat(filepath.Join(dir, "generators", "mygen.lua")) + assert.NoError(t, err, "Couldn't find generators/mygen.lua") + _, err = os.Stat(filepath.Join(dir, "generators", "mygen.yml")) + assert.NoError(t, err, "Couldn't find generators/mygen.yml") +} + +func TestSharePullWithDeconstructTemplates(t *testing.T) { + // Test deconstructConfig with templates + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: mytemplate +samples: + - name: tmpltest + description: Template deconstruct test + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +templates: + - name: mytemplate + header: "HEADER\n" + row: "{{._raw}}\n" + footer: "FOOTER\n" +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/tmpltest", + Name: "tmpltest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/tmpltest", dir, true) + _, err := os.Stat(filepath.Join(dir, "templates", "mytemplate.yml")) + assert.NoError(t, err, "Couldn't find templates/mytemplate.yml") +} diff --git a/outputter/send_test.go b/outputter/send_test.go new file mode 100644 index 0000000..1f52406 --- /dev/null +++ b/outputter/send_test.go @@ -0,0 +1,682 @@ +package outputter + +import ( + "bytes" + "io" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/stretchr/testify/assert" +) + +func TestSetup(t *testing.T) { + tests := []struct { + name string + outputter string + expectedType interface{} + }{ + {"stdout", "stdout", &stdout{}}, + {"devnull", "devnull", &devnull{}}, + {"file", "file", &file{}}, + {"http", "http", &httpout{}}, + {"buf", "buf", &buf{}}, + {"network", "network", &network{}}, + {"kafka", "kafka", &kafkaout{}}, + {"unknown defaults to stdout", "unknowntype", &stdout{}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Use a unique gout slot for each test + num := 99 // use last slot to avoid conflicts + gout[num] = nil + + s := &config.Sample{ + Name: "test", + Output: &config.Output{ + Outputter: tc.outputter, + }, + } + item := &config.OutQueueItem{S: s} + source := rand.NewSource(0) + gen := rand.New(source) + + result := setup(gen, item, num) + assert.IsType(t, tc.expectedType, result) + + // Clean up + gout[num] = nil + }) + } +} + +func TestDevnullSend(t *testing.T) { + d := &devnull{} + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: &config.Sample{Name: "test"}, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "test data") + oio.W.Close() + }() + + err := d.Send(item) + assert.NoError(t, err) +} + +func TestDevnullClose(t *testing.T) { + d := &devnull{} + err := d.Close() + assert.NoError(t, err) +} + +func TestBufSend(t *testing.T) { + var b bytes.Buffer + s := &config.Sample{ + Name: "test", + Buf: &b, + } + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: s, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "buffered data\n") + oio.W.Close() + }() + + bu := &buf{} + err := bu.Send(item) + assert.NoError(t, err) + assert.Equal(t, "buffered data\n", b.String()) +} + +func TestStdoutSend(t *testing.T) { + // Redirect stdout to a pipe + origStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: &config.Sample{Name: "test"}, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "stdout data\n") + oio.W.Close() + }() + + so := &stdout{} + err := so.Send(item) + assert.NoError(t, err) + + w.Close() + var buf bytes.Buffer + io.Copy(&buf, r) + os.Stdout = origStdout + + assert.Equal(t, "stdout data\n", buf.String()) +} + +func TestStdoutClose(t *testing.T) { + so := &stdout{} + err := so.Close() + assert.NoError(t, err) +} + +func TestFileSendAndRotation(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "testfile.log") + + s := &config.Sample{ + Name: "filesample", + Output: &config.Output{ + FileName: filename, + MaxBytes: 50, // Very small to trigger rotation + BackupFiles: 2, + }, + } + + f := &file{} + + // Write enough data to trigger rotation + for i := 0; i < 5; i++ { + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + + go func() { + io.WriteString(oio.W, strings.Repeat("X", 30)+"\n") + oio.W.Close() + }() + + err := f.Send(item) + assert.NoError(t, err) + } + + // Check that backup files were created + _, err := os.Stat(filename) + assert.NoError(t, err, "main file should exist") + + _, err = os.Stat(filename + ".1") + assert.NoError(t, err, "backup .1 should exist") + + f.Close() +} + +func TestFileSendExistingFile(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "existing.log") + + // Pre-create the file with some content + os.WriteFile(filename, []byte("pre-existing data\n"), 0644) + + s := &config.Sample{ + Name: "fileexisting", + Output: &config.Output{ + FileName: filename, + MaxBytes: 10000000, + BackupFiles: 2, + }, + } + + f := &file{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + + go func() { + io.WriteString(oio.W, "appended data\n") + oio.W.Close() + }() + + err := f.Send(item) + assert.NoError(t, err) + + // Verify the file has both old and new data + data, _ := os.ReadFile(filename) + assert.Contains(t, string(data), "pre-existing data") + assert.Contains(t, string(data), "appended data") + + f.Close() +} + +func TestFileClose(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "closefile.log") + + s := &config.Sample{ + Name: "fileclose", + Output: &config.Output{ + FileName: filename, + MaxBytes: 1000000, + BackupFiles: 2, + }, + } + + f := &file{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, "data\n") + oio.W.Close() + }() + f.Send(item) + + // Close should work + err := f.Close() + assert.NoError(t, err) + + // Close again should be idempotent + err = f.Close() + assert.NoError(t, err) +} + +func TestHTTPSendAndFlush(t *testing.T) { + var received bytes.Buffer + var mu sync.Mutex + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + io.Copy(&received, r.Body) + mu.Unlock() + w.WriteHeader(200) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpsample", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, // Small buffer to trigger flush + Headers: map[string]string{"Content-Type": "application/json"}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + // Send enough data to exceed buffer and trigger flush + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("D", 50)+"\n") + oio.W.Close() + }() + + err := h.Send(item) + assert.NoError(t, err) + + // Verify server received data + mu.Lock() + data := received.String() + mu.Unlock() + assert.NotEmpty(t, data, "HTTP server should have received data") + + // Close flushes remaining data + err = h.Close() + assert.NoError(t, err) +} + +func TestNetworkSend(t *testing.T) { + // Start a TCP listener on a random port + ln, err := net.Listen("tcp", "127.0.0.1:0") + assert.NoError(t, err) + defer ln.Close() + + var received bytes.Buffer + done := make(chan struct{}) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + io.Copy(&received, conn) + conn.Close() + close(done) + }() + + s := &config.Sample{ + Name: "netsample", + Output: &config.Output{ + Endpoints: []string{ln.Addr().String()}, + Protocol: "tcp", + Timeout: 5 * time.Second, + }, + } + + n := &network{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, "network data\n") + oio.W.Close() + }() + + err = n.Send(item) + assert.NoError(t, err) + + // Close the connection so the listener goroutine can finish + n.Close() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for network data") + } + + assert.Equal(t, "network data\n", received.String()) +} + +func TestNetworkClose(t *testing.T) { + n := &network{} + // Close with no connection should not error + err := n.Close() + assert.NoError(t, err) + assert.False(t, n.initialized) +} + +func TestBufClose(t *testing.T) { + b := &buf{} + err := b.Close() + assert.NoError(t, err) +} + +func TestStartDevnullWorker(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Reset gout slot + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + // Send an item through the pipeline + s := &config.Sample{ + Name: "starttest", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + events := []map[string]string{ + {"_raw": "test event for start"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + // Close the queue and wait for worker to finish + close(oq) + select { + case <-oqs: + // Worker finished + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // Verify gout slot was cleared + assert.Nil(t, gout[0]) +} + +func TestStartMultipleItems(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "multistart", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + + for i := 0; i < 5; i++ { + events := []map[string]string{ + {"_raw": "event number"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + } + + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // Check that events were accounted for + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + ew := EventsWritten["multistart"] + Mutex.RUnlock() + assert.Equal(t, int64(5), ew) +} + +func TestStartEmptyEvents(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "emptyevents", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + // Send item with no events - should skip the write/send + item := &config.OutQueueItem{ + S: s, + Events: []map[string]string{}, + Cache: &config.CacheItem{}, + } + oq <- item + + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} + +func TestStartCloseOnChannelClose(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "closetest", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + // Send one real item so lastS is set, then close + events := []map[string]string{ + {"_raw": "test event"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + // Close the channel - should trigger the Close() path on the outputter + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // gout should be cleared + assert.Nil(t, gout[0]) +} + +func TestStartSendError(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Use a network outputter pointed at a bad address to trigger Send error + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "senderror", + Output: &config.Output{ + Outputter: "network", + OutputTemplate: "raw", + Endpoints: []string{"127.0.0.1:1"}, // Should fail to connect + Protocol: "tcp", + }, + } + events := []map[string]string{ + {"_raw": "error event"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + close(oq) + select { + case <-oqs: + case <-time.After(10 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} + +func TestHTTPFlushNon200(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + w.Write([]byte("internal server error")) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpfail", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, + Headers: map[string]string{"Content-Type": "text/plain"}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("X", 50)+"\n") + oio.W.Close() + }() + + err := h.Send(item) + // flush should return an error due to non-200 status + assert.Error(t, err) + assert.Contains(t, err.Error(), "500") +} + +func TestHTTPCloseFlushError(t *testing.T) { + // Use a server that accepts the first request (Send flush) but returns error on the second (Close flush) + calls := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 { + w.WriteHeader(500) + w.Write([]byte("flush error")) + return + } + w.WriteHeader(200) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpclose", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, // Small buffer to trigger flush on first Send + Headers: map[string]string{}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + // First Send: exceeds buffer, triggers flush (call #1 → 200 OK) + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("X", 50)) + oio.W.Close() + }() + + err := h.Send(item) + assert.NoError(t, err) + + // Add more data to buffer for Close to flush + h.buf.WriteString("leftover data") + + // Close should flush remaining data and get 500 error (call #2) + err = h.Close() + assert.Error(t, err) + assert.Contains(t, err.Error(), "500") +} + +func TestStartSendErrorRepeat(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "senderrorrepeat", + Output: &config.Output{ + Outputter: "network", + OutputTemplate: "raw", + Endpoints: []string{"127.0.0.1:1"}, + Protocol: "tcp", + }, + } + // Send multiple items to trigger repeat error path (lasterr[num].count++) + for i := 0; i < 3; i++ { + events := []map[string]string{ + {"_raw": "error event repeat"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + } + + close(oq) + select { + case <-oqs: + case <-time.After(15 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} diff --git a/outputter/write_test.go b/outputter/write_test.go new file mode 100644 index 0000000..d321cbe --- /dev/null +++ b/outputter/write_test.go @@ -0,0 +1,451 @@ +package outputter + +import ( + "bytes" + "encoding/json" + "io" + "strings" + "sync" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/template" + "github.com/stretchr/testify/assert" +) + +// initROT initializes the rotchan and readStats goroutine needed by write(). +// Returns a cleanup function to call via defer. +func initROT() func() { + Mutex.Lock() + BytesWritten = make(map[string]int64) + EventsWritten = make(map[string]int64) + rotwg = sync.WaitGroup{} + rotchan = make(chan *config.OutputStats) + Mutex.Unlock() + rotwg.Add(1) + go readStats() + return func() { + close(rotchan) + rotwg.Wait() + } +} + +func makeOutQueueItem(sampleName, outputTemplate, outputter string, events []map[string]string) *config.OutQueueItem { + s := &config.Sample{ + Name: sampleName, + Output: &config.Output{ + Outputter: outputter, + OutputTemplate: outputTemplate, + }, + } + oio := config.NewOutputIO() + return &config.OutQueueItem{ + S: s, + Events: events, + IO: oio, + Cache: &config.CacheItem{}, + } +} + +func readFromPipe(item *config.OutQueueItem) string { + var buf bytes.Buffer + io.Copy(&buf, item.IO.R) + return buf.String() +} + +func TestWriteRaw(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "hello world"}, + } + item := makeOutQueueItem("rawsample", "raw", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "hello world") +} + +func TestWriteJSON(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "test event", "host": "myhost"}, + } + item := makeOutQueueItem("jsonsample", "json", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + var parsed map[string]string + lines := strings.TrimSpace(result) + err := json.Unmarshal([]byte(strings.Split(lines, "\n")[0]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "test event", parsed["_raw"]) + assert.Equal(t, "myhost", parsed["host"]) +} + +func TestWriteSplunkHEC(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "splunk event", "_time": "1234567890"}, + } + item := makeOutQueueItem("hecsample", "splunkhec", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + var parsed map[string]string + lines := strings.TrimSpace(result) + err := json.Unmarshal([]byte(strings.Split(lines, "\n")[0]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "splunk event", parsed["event"], "splunkhec should remap _raw to event") + assert.Equal(t, "1234567890", parsed["time"], "splunkhec should remap _time to time") + assert.Empty(t, parsed["_raw"], "_raw should be deleted") + assert.Empty(t, parsed["_time"], "_time should be deleted") +} + +func TestWriteRFC3164(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "syslog msg", "_time": "Oct 20 12:00:00", "priority": "13", "host": "myhost", "tag": "gogen", "pid": "1234"}, + } + item := makeOutQueueItem("rfc3164sample", "rfc3164", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "<13>") + assert.Contains(t, result, "Oct 20 12:00:00") + assert.Contains(t, result, "myhost") + assert.Contains(t, result, "gogen[1234]") + assert.Contains(t, result, "syslog msg") +} + +func TestWriteRFC5424(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "syslog5424 msg", "_time": "2001-10-20T12:00:00Z", "priority": "13", "host": "myhost", "appName": "gogen", "pid": "1234", "extra": "val"}, + } + item := makeOutQueueItem("rfc5424sample", "rfc5424", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "<13>1") + assert.Contains(t, result, "myhost") + assert.Contains(t, result, "gogen") + assert.Contains(t, result, "syslog5424 msg") + assert.Contains(t, result, "[meta") + assert.Contains(t, result, `extra="val"`) +} + +func TestWriteElasticsearch(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "es event", "index": "testindex"}, + } + item := makeOutQueueItem("essample", "elasticsearch", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, `"_index": "testindex"`) + assert.Contains(t, result, `"_type": "doc"`) + // _raw should be remapped to message + var parsed map[string]interface{} + lines := strings.Split(strings.TrimSpace(result), "\n") + assert.GreaterOrEqual(t, len(lines), 2, "elasticsearch should produce index header + body") + err := json.Unmarshal([]byte(lines[1]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "es event", parsed["message"]) +} + +func TestWriteDevnull(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "devnull event data"}, + } + item := makeOutQueueItem("devnullsample", "raw", "devnull", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // devnull should not write any content through the pipe + assert.Empty(t, result) + + // But bytes should still be accounted for + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + bw := BytesWritten["devnullsample"] + ew := EventsWritten["devnullsample"] + Mutex.RUnlock() + assert.Greater(t, bw, int64(0), "bytes should be counted even for devnull") + assert.Equal(t, int64(1), ew) +} + +func TestWriteCacheMiss(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "cache miss event"}, + } + item := makeOutQueueItem("cachemiss", "raw", "stdout", events) + item.Cache.UseCache = true // UseCache=true but no cacheBuf exists => cache miss + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Cache miss should still write through the normal pipe + assert.Contains(t, result, "cache miss event") +} + +func TestWriteSetCache(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Clean the cache bufs + cacheMutex.Lock() + delete(cacheBufs, "setcache") + cacheMutex.Unlock() + + events := []map[string]string{ + {"_raw": "cached event"}, + } + item := makeOutQueueItem("setcache", "raw", "stdout", events) + item.Cache.SetCache = true + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // SetCache should write to both cache buffer and the pipe + assert.Contains(t, result, "cached event") + + // Verify cache buffer was populated + cacheMutex.RLock() + cb, ok := cacheBufs["setcache"] + cacheMutex.RUnlock() + assert.True(t, ok, "cache buffer should be created") + assert.Contains(t, cb.String(), "cached event") +} + +func TestWriteUseCache(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Pre-populate the cache + cacheMutex.Lock() + cacheBufs["usecache"] = &bytes.Buffer{} + cacheBufs["usecache"].WriteString("previously cached data\n") + cacheMutex.Unlock() + + events := []map[string]string{ + {"_raw": "new event"}, + } + item := makeOutQueueItem("usecache", "raw", "stdout", events) + item.Cache.UseCache = true + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Should use the cached data, not the new events + assert.Contains(t, result, "previously cached data") + + // Clean up + cacheMutex.Lock() + delete(cacheBufs, "usecache") + cacheMutex.Unlock() +} + +func TestWriteNonExistentTemplate(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "should not appear"}, + } + item := makeOutQueueItem("badtemplate", "nonexistent_template_xyz", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Non-existent template should produce no output + assert.Empty(t, result) +} + +func TestWriteMultipleEvents(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "event1"}, + {"_raw": "event2"}, + {"_raw": "event3"}, + } + item := makeOutQueueItem("multisample", "raw", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "event1") + assert.Contains(t, result, "event2") + assert.Contains(t, result, "event3") + + // Verify accounting + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + ew := EventsWritten["multisample"] + Mutex.RUnlock() + assert.Equal(t, int64(3), ew) +} + +func TestWriteKafkaNoNewlines(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "kafka event"}, + } + item := makeOutQueueItem("kafkasample", "raw", "kafka", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Kafka should not append newlines + assert.Equal(t, "kafka event", result) +} + +func TestWriteCustomTemplate(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Register a custom template + _ = template.New("customtest_header", "HEADER\n") + _ = template.New("customtest_row", "ROW:{{._raw}}\n") + _ = template.New("customtest_footer", "FOOTER\n") + + events := []map[string]string{ + {"_raw": "custom line"}, + } + item := makeOutQueueItem("customsample", "customtest", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "HEADER") + assert.Contains(t, result, "ROW:custom line") + assert.Contains(t, result, "FOOTER") +} diff --git a/rater/rater_test.go b/rater/rater_test.go index 470d3db..047aef5 100644 --- a/rater/rater_test.go +++ b/rater/rater_test.go @@ -1,10 +1,13 @@ package rater import ( + "os" + "path/filepath" "testing" "time" config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" "github.com/stretchr/testify/assert" ) @@ -14,3 +17,144 @@ func TestRandomizeCount(t *testing.T) { count := EventRate(s, time.Now(), 10) assert.Equal(t, 11, count) } + +func TestTokenRateDefault(t *testing.T) { + dr := &DefaultRater{} + token := config.Token{Name: "test"} + rate := dr.TokenRate(token, time.Now()) + assert.Equal(t, 1.0, rate) +} + +func TestTokenRateConfig(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := config.NewConfig() + r := c.FindRater("testconfigrater") + + cr := &ConfigRater{c: r} + token := config.Token{Name: "test"} + + loc, _ := time.LoadLocation("Local") + n := time.Date(2001, 10, 20, 0, 0, 0, 100000, loc) + rate := cr.TokenRate(token, n) + assert.Equal(t, 2.0, rate) +} + +func TestTokenRateKBps(t *testing.T) { + kr := &KBpsRater{c: &config.RaterConfig{Name: "kbps"}} + token := config.Token{Name: "test"} + rate := kr.TokenRate(token, time.Now()) + assert.Equal(t, 1.0, rate) +} + +func TestTokenRateScript(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "luarater.yml")) + + c := config.NewConfig() + r := c.FindRater("multiply") + + sr := &ScriptRater{c: r} + token := config.Token{Name: "test"} + rate := sr.TokenRate(token, time.Now()) + assert.Equal(t, 2.0, rate) +} + +func TestGetRaterFallback(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "defaultrater.yml")) + + r := GetRater("nonexistentrater") + assert.IsType(t, &DefaultRater{}, r, "unknown rater name should fall back to DefaultRater") +} + +func TestEventRateNegativeResult(t *testing.T) { + // A rater returning a negative rate should produce a negative or zero count + s := &config.Sample{RandomizeCount: 0} + // Use a mock rater by pre-setting it + s.Rater = &negativeRater{} + randSource = 2 + count := EventRate(s, time.Now(), 10) + assert.True(t, count <= 0, "negative rate should produce non-positive count, got %d", count) +} + +func TestKBpsEventRateMissingOption(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{}, + }, + } + s := &config.Sample{Name: "test"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateWrongType(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": "not_a_float", + }, + }, + } + s := &config.Sample{Name: "test"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateMissingSample(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": 100.0, + }, + }, + } + s := &config.Sample{Name: "nonexistent_sample_kbps"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateWithData(t *testing.T) { + // Pre-populate outputter stats + outputter.Mutex.Lock() + outputter.BytesWritten["kbps_test_sample"] = 10000 + outputter.EventsWritten["kbps_test_sample"] = 100 + outputter.Mutex.Unlock() + defer func() { + outputter.Mutex.Lock() + delete(outputter.BytesWritten, "kbps_test_sample") + delete(outputter.EventsWritten, "kbps_test_sample") + outputter.Mutex.Unlock() + }() + + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": 100.0, + }, + }, + t: time.Now().Add(-1 * time.Second), // pretend we started 1s ago + } + s := &config.Sample{Name: "kbps_test_sample"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) // always returns 1.0 regardless +} + +// negativeRater always returns a negative rate for testing +type negativeRater struct{} + +func (nr *negativeRater) EventRate(s *config.Sample, now time.Time, count int) float64 { + return -1.0 +} +func (nr *negativeRater) TokenRate(t config.Token, now time.Time) float64 { + return -1.0 +} diff --git a/run/run_test.go b/run/run_test.go new file mode 100644 index 0000000..17b8bc1 --- /dev/null +++ b/run/run_test.go @@ -0,0 +1,200 @@ +package run + +import ( + "bytes" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" + "github.com/stretchr/testify/assert" +) + +// resetRunState resets config and outputter stats for a clean test. +func resetRunState() { + config.ResetConfig() + outputter.Mutex.Lock() + outputter.BytesWritten = make(map[string]int64) + outputter.EventsWritten = make(map[string]int64) + outputter.Mutex.Unlock() +} + +func TestRunCompletesWithEndIntervals(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: devnull + outputTemplate: raw + rotInterval: 1 +samples: + - name: runtest + description: "Run completion test" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: run test event +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.NotEmpty(t, c.Samples) + + done := make(chan struct{}) + go func() { + Run(c) + close(done) + }() + + select { + case <-done: + // Run completed successfully + case <-time.After(10 * time.Second): + t.Fatal("Run() did not complete within timeout") + } +} + +func TestRunMultipleSamples(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: devnull + outputTemplate: raw + rotInterval: 1 +samples: + - name: multi1 + description: "Multi sample 1" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: event from multi1 + - name: multi2 + description: "Multi sample 2" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: event from multi2 +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.Len(t, c.Samples, 2) + + done := make(chan struct{}) + go func() { + Run(c) + close(done) + }() + + select { + case <-done: + outputter.Mutex.RLock() + totalEvents := int64(0) + for _, v := range outputter.EventsWritten { + totalEvents += v + } + outputter.Mutex.RUnlock() + assert.Greater(t, totalEvents, int64(0), "should have generated events") + case <-time.After(10 * time.Second): + t.Fatal("Run() did not complete within timeout") + } +} + +func TestOnceMethod(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: buf + outputTemplate: json + rotInterval: 1 +samples: + - name: oncemethodtest + description: "Once method test" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: once method event +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + r := Runner{} + + done := make(chan struct{}) + go func() { + r.Once("oncemethodtest") + close(done) + }() + + select { + case <-done: + // Once completed without error + case <-time.After(10 * time.Second): + t.Fatal("Once() did not complete within timeout") + } +} + +func TestOncePublic(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: buf + outputTemplate: json + rotInterval: 1 +samples: + - name: oncetest + description: "Once test sample" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: once event data +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.NotEmpty(t, c.Samples) + + // Set up a buffer for the sample + var buf bytes.Buffer + s := c.FindSampleByName("oncetest") + s.Buf = &buf + + r := Runner{} + + // Start ROT before onceWithConfig (Once() normally does this) + go outputter.ROT(c) + // Give ROT goroutine time to create the new rotchan + time.Sleep(50 * time.Millisecond) + + done := make(chan struct{}) + go func() { + r.onceWithConfig("oncetest", c) + close(done) + }() + + select { + case <-done: + assert.Contains(t, buf.String(), "once event data") + case <-time.After(10 * time.Second): + t.Fatal("Once() did not complete within timeout") + } +} diff --git a/run/runonce_test.go b/run/runonce_test.go index e40ec2d..2b83294 100644 --- a/run/runonce_test.go +++ b/run/runonce_test.go @@ -7,12 +7,12 @@ import ( "time" config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" "github.com/stretchr/testify/assert" ) func TestOnceWithConfig(t *testing.T) { - // Clean up any existing config - config.ResetConfig() + resetRunState() // Setup test configuration configStr := ` @@ -58,6 +58,9 @@ samples: config.SetupFromString(configStr) c := config.NewConfig() + // Start ROT in case a previous test closed rotchan + go outputter.ROT(c) + // Record time before and after test to validate timestamp is within range beforeTest := time.Now().Truncate(time.Second) if c.Global.UTC { diff --git a/tests/generator/luaapi2.yml b/tests/generator/luaapi2.yml new file mode 100644 index 0000000..2edf1dc --- /dev/null +++ b/tests/generator/luaapi2.yml @@ -0,0 +1,43 @@ +generators: + - name: roundTest + script: | + val = round(3.14159, 2) + setToken("rounded", tostring(val)) + - name: logInfoTest + script: | + info("test log message from lua") + setToken("logged", "ok") + - name: removeTokenTest + script: | + setToken("keeper", "keep") + setToken("remover", "remove") + removeToken("remover") + - name: sendEventTest + script: | + event = { _raw = "sent via sendEvent" } + sendEvent(event) +samples: + - name: roundTest + generator: roundTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: logInfoTest + generator: logInfoTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: removeTokenTest + generator: removeTokenTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: sendEventTest + generator: sendEventTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused diff --git a/timer/timer_test.go b/timer/timer_test.go index b93ac0c..139e98f 100644 --- a/timer/timer_test.go +++ b/timer/timer_test.go @@ -199,6 +199,40 @@ Loop: } } +func TestBackfillReplay(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + home := filepath.Join("..", "tests", "timer") + os.Setenv("GOGEN_SAMPLES_DIR", home) + + s := tests.FindSampleInFile(home, "realtimereplay") + // Force non-realtime mode to exercise backfill with replay generator + s.Realtime = false + // Set end slightly in the future so backfill runs a few iterations + s.EndParsed = s.Current.Add(5 * time.Second) + + gq := make(chan *config.GenQueueItem, 1000) + oq := make(chan *config.OutQueueItem) + done := make(chan int) + gqs := make([]*config.GenQueueItem, 0, 10) + + timer := &Timer{S: s, GQ: gq, OQ: oq, Done: done} + go timer.NewTimer(0) + <-done + +Loop: + for { + select { + case i := <-gq: + gqs = append(gqs, i) + default: + break Loop + } + } + // Should have generated events using replay offsets via inc() replay path + assert.Greater(t, len(gqs), 0, "should have generated replay events during backfill") +} + func TestTimerClose(t *testing.T) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1")