diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4b48d2a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,102 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Gogen is an open source data generator for generating demo and test data, especially time series log and metric data. It's a Go CLI tool with an embedded Lua scripting engine, a Python AWS Lambda API backend, and a React/TypeScript UI. + +## Common Commands + +### Go (core CLI) + +```bash +make install # Build and install to $GOPATH/bin (default target) +make build # Cross-compile for linux, darwin, windows, wasm +make test # Run all Go tests: go test -v ./... +go test -v ./internal # Run tests for a single package +go test -v -run TestName ./internal # Run a single test +``` + +Version, git summary, build date, and GitHub OAuth credentials are injected via `-ldflags` in the Makefile. Always use `make install` rather than bare `go install`. + +Dependencies are vendored in `vendor/`. After adding deps, run `go mod vendor`. + +### Python API (`gogen-api/`) + +```bash +cd gogen-api +./start_dev.sh # Starts DynamoDB Local + MinIO via docker-compose, then SAM local API on port 4000 +./setup_local_db.sh # Seeds local DynamoDB schema +sam build && sam local start-api --port 4000 --docker-network lambda-local +./deploy_lambdas.sh # Deploy to AWS (requires credentials) +``` + +### UI (`ui/`) + +```bash +cd ui +npm run dev # Vite dev server (copies wasm from build/wasm/ first) +npm run build # Production build +npm test # Jest tests +``` + +## Architecture + +### Go Package Layout + +All packages are at the top level (no `cmd/` or `pkg/` convention): + +- **`main.go`** — CLI entry point using `urfave/cli.v1`. Maps CLI flags to `GOGEN_*` env vars. +- **`internal/`** — Core package. Config singleton, `Sample` struct, `Token` processing, API client, sharing. Imported as `config` throughout (`config "github.com/coccyx/gogen/internal"`). +- **`generator/`** — Reads `GenQueueItem` from channel, dispatches to sample-based or Lua generators. +- **`outputter/`** — Reads `OutQueueItem` from channel, dispatches to output destinations (stdout, file, HTTP, Kafka, network, devnull, buf). +- **`run/`** — Orchestrates the pipeline: timers -> generator worker pool -> outputter worker pool. +- **`timer/`** — One timer goroutine per Sample; handles backfill and realtime intervals. +- **`rater/`** — Controls event rate (config-based, time-of-day/weekday, kbps, Lua script). +- **`template/`** — Output formatting (raw, JSON, CSV, splunkhec, syslog, elasticsearch). +- **`logger/`** — Thin logrus wrapper with file/func/line context hook. + +### Data Flow + +``` +YAML/JSON Config -> internal.Config singleton (sync.Once) + -> [Timer goroutine per Sample] + -> GenQueueItem channel -> [Generator worker pool] + -> OutQueueItem channel -> [Outputter worker pool] + -> output destination +``` + +Concurrency is channel + goroutine worker pools. Worker counts set by `GeneratorWorkers` and `OutputWorkers` config fields. + +### Key Interfaces + +- `internal.Generator` — `Gen(item *GenQueueItem) error` +- `internal.Outputter` — `Send(events []map[string]string, sample *Sample, outputTemplate string) error` +- `internal.Rater` — `EventsPerInterval(s *Sample) int` + +### Config System + +Config is a **singleton** via `sync.Once`. Controlled by environment variables: +- `GOGEN_HOME`, `GOGEN_FULLCONFIG`, `GOGEN_CONFIG_DIR`, `GOGEN_SAMPLES_DIR` +- Remote configs fetched from `https://api.gogen.io` (override with `GOGEN_APIURL`) + +In tests, call `config.ResetConfig()` before `config.NewConfig()` to get a fresh instance. Tests commonly use `config.SetupFromString(yamlStr)` to inject inline YAML config. + +### gogen-api (Python Lambda) + +Each Lambda function is a separate `.py` file in `gogen-api/`. Backed by DynamoDB + S3. Originally Python 2.7, being updated to Python 3. AWS SAM template at `gogen-api/template.yaml`. + +### UI (React/TypeScript) + +Vite + React 18 + TypeScript + Tailwind CSS. Components in `src/components/`, pages in `src/pages/`, API clients in `src/api/`, types in `src/types/`. Tests use Jest + React Testing Library, placed adjacent to source as `.test.tsx`. + +## CI/CD + +GitHub Actions (`.github/workflows/ci.yml`): +- Push to `master`/`dev` or any PR: runs `make test`, then on `master`/`dev` cross-compiles, builds Docker, pushes artifacts to S3, deploys UI and Lambdas. +- Tag pushes (`v*.*.*`): full release workflow via `release.yml` — builds, creates GitHub release, pushes Docker images, deploys to production. + +## Lua Scripting + +Generators (`generator/lua.go`) and raters (`rater/script.go`) support embedded Lua via `gopher-lua` + `gopher-luar`. Lua state persists across calls within a run. diff --git a/generator/generator_test.go b/generator/generator_test.go index 73e2e9a..7568314 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -12,23 +12,25 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGenerator(t *testing.T) { - // Setup environment +// setupGenTest resets config, sets env vars, and returns common test fixtures. +func setupGenTest(t *testing.T, samplesDir string, seed int64) (func() time.Time, *rand.Rand) { + t.Helper() + config.ResetConfig() os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") os.Setenv("GOGEN_FULLCONFIG", "") - home := filepath.Join("..", "tests", "tokens") - os.Setenv("GOGEN_SAMPLES_DIR", home) + os.Setenv("GOGEN_SAMPLES_DIR", samplesDir) loc, _ := time.LoadLocation("Local") - source := rand.NewSource(0) - randgen := rand.New(source) - + randgen := rand.New(rand.NewSource(seed)) n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc) - now := func() time.Time { - return n - } + now := func() time.Time { return n } + return now, randgen +} + +func TestGenerator(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) - // gq := make(chan *config.GenQueueItem) oq := make(chan *config.OutQueueItem) s := tests.FindSampleInFile(home, "token-static") if s == nil { @@ -44,23 +46,171 @@ func TestGenerator(t *testing.T) { assert.Equal(t, "foo", oqi.Events[0]["_raw"]) } -func TestGeneratorCache(t *testing.T) { - // Setup environment +func TestGeneratorMultiPass(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + // Force MultiPass + s.SinglePass = false + + // Count > lines: tests the iters > 1 path in genMultiPass + gqi := &config.GenQueueItem{Count: len(s.Lines) + 2, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + err := genMultiPass(gqi) + assert.NoError(t, err) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines)+2, len(oqi.Events)) +} + +func TestGeneratorMultiPassRandomize(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 42) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + s.SinglePass = false + s.RandomizeEvents = true + + gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genMultiPass(gqi) + }() + + oqi := <-oq + assert.Equal(t, 5, len(oqi.Events)) +} + +func TestGeneratorSinglePassCountGtLines(t *testing.T) { + home := filepath.Join("..", "tests", "singlepass") + now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 0) + + c := config.NewConfig() + s := c.FindSampleByName("test1") + if s == nil { + t.Fatalf("Sample test1 not found") + } + assert.True(t, s.SinglePass) + + oq := make(chan *config.OutQueueItem) + // Count > lines: tests the iters > 1 singlepass path + gqi := &config.GenQueueItem{Count: len(s.Lines) + 3, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genSinglePass(gqi) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines)+3, len(oqi.Events)) +} + +func TestGeneratorSinglePassRandomize(t *testing.T) { + home := filepath.Join("..", "tests", "singlepass") + now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 42) + + c := config.NewConfig() + s := c.FindSampleByName("test1") + if s == nil { + t.Fatalf("Sample test1 not found") + } + assert.True(t, s.SinglePass) + s.RandomizeEvents = true + + oq := make(chan *config.OutQueueItem) + gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + genSinglePass(gqi) + }() + + oqi := <-oq + assert.Equal(t, 5, len(oqi.Events)) +} + +func TestGeneratorStartWorker(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "token-static") + if s == nil { + t.Fatalf("Sample token-static not found") + } + + gq := make(chan *config.GenQueueItem) + gqs := make(chan int) + go Start(gq, gqs) + + // Send multiple items to test the "generator already set" path + for i := 0; i < 3; i++ { + gqi := &config.GenQueueItem{Count: 1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + gq <- gqi + oqi := <-oq + assert.Equal(t, "foo", oqi.Events[0]["_raw"]) + } + + close(gq) + select { + case <-gqs: + case <-time.After(5 * time.Second): + t.Fatal("Generator worker did not finish in time") + } +} + +func TestGeneratorCountMinusOne(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + + oq := make(chan *config.OutQueueItem) + s := tests.FindSampleInFile(home, "tokens") + if s == nil { + t.Fatalf("Sample tokens not found") + } + // Count=-1 means "use all lines" + gqi := &config.GenQueueItem{Count: -1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}} + go func() { + sg := sample{} + sg.Gen(gqi) + }() + + oqi := <-oq + assert.Equal(t, len(s.Lines), len(oqi.Events)) +} + +func TestPrimeRaterSetsRater(t *testing.T) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") - os.Setenv("GOGEN_FULLCONFIG", "") - home := filepath.Join("..", "tests", "tokens") - os.Setenv("GOGEN_SAMPLES_DIR", home) - loc, _ := time.LoadLocation("Local") - source := rand.NewSource(0) - randgen := rand.New(source) - n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc) - now := func() time.Time { - return n + s := &config.Sample{ + Name: "primerater_test", + Tokens: []config.Token{ + { + Name: "ratedtoken", + Type: "rated", + RaterString: "default", + }, + { + Name: "normaltoken", + Type: "choice", + }, + }, } - // gq := make(chan *config.GenQueueItem) + PrimeRater(s) + assert.NotNil(t, s.Tokens[0].Rater, "rated token should have rater set") +} + +func TestGeneratorCache(t *testing.T) { + home := filepath.Join("..", "tests", "tokens") + now, randgen := setupGenTest(t, home, 0) + oq := make(chan *config.OutQueueItem) s := tests.FindSampleInFile(home, "token-static") if s == nil { diff --git a/generator/lua_test.go b/generator/lua_test.go index dabb53b..cc3534c 100644 --- a/generator/lua_test.go +++ b/generator/lua_test.go @@ -288,6 +288,98 @@ func TestSetTime(t *testing.T) { testLuaGen(t, s, gen, "2001-10-20 11:59:59.000100") } +func TestLuaRound(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("roundTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + found := false + var token config.Token + for _, tk := range gen.tokens { + if tk.Name == "rounded" { + found = true + token = tk + } + } + assert.True(t, found, "Couldn't find token 'rounded' in sample roundTest") + assert.Equal(t, "3.14", token.Replacement) +} + +func TestLuaLogInfo(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("logInfoTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + found := false + var token config.Token + for _, tk := range gen.tokens { + if tk.Name == "logged" { + found = true + token = tk + } + } + assert.True(t, found, "Couldn't find token 'logged' in sample logInfoTest") + assert.Equal(t, "ok", token.Replacement) +} + +func TestRemoveToken(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("removeTokenTest") + gen := new(luagen) + runLuaGen(t, s, gen) + time.Sleep(100 * time.Millisecond) + + foundKeeper := false + foundRemover := false + for _, tk := range gen.tokens { + if tk.Name == "keeper" { + foundKeeper = true + } + if tk.Name == "remover" { + foundRemover = true + } + } + assert.True(t, foundKeeper, "Token 'keeper' should still be present") + assert.False(t, foundRemover, "Token 'remover' should have been removed") +} + +func TestSendEvent(t *testing.T) { + config.ResetConfig() + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "") + home := ".." + os.Setenv("GOGEN_FULLCONFIG", filepath.Join(home, "tests", "generator", "luaapi2.yml")) + + c := config.NewConfig() + s := c.FindSampleByName("sendEventTest") + gen := new(luagen) + testLuaGen(t, s, gen, "sent via sendEvent") +} + func testLuaGen(t *testing.T, s *config.Sample, gen *luagen, expected string) { oq, err := runLuaGen(t, s, gen) timeout := make(chan bool, 1) diff --git a/internal/config_test.go b/internal/config_test.go index 8e565af..5b37c96 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -2,6 +2,8 @@ package internal import ( "math/rand" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" @@ -356,3 +358,1295 @@ func TestParseWebConfig(t *testing.T) { assert.Equal(t, "timestamp", tsToken.Type) assert.Equal(t, "%d/%b/%Y %H:%M:%S:%L", tsToken.Replacement) } + +func TestFindRater(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + r := c.FindRater("testconfigrater") + assert.NotNil(t, r) + assert.Equal(t, "testconfigrater", r.Name) + + CleanupConfigAndEnvironment() +} + +func TestFindRaterNotFound(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + r := c.FindRater("nonexistentrater") + assert.Nil(t, r) + + CleanupConfigAndEnvironment() +} + +func TestFindSampleByNameNotFound(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := NewConfig() + + s := c.FindSampleByName("nonexistentsample") + assert.Nil(t, s) + + CleanupConfigAndEnvironment() +} + +func TestClean(t *testing.T) { + configStr := ` +samples: + - name: enabled-sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test + - name: disabled-sample + disabled: true + interval: 1 + count: 1 + lines: + - _raw: test +` + ResetConfig() + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + + // After Clean(), only enabled real samples should remain + found := false + for _, s := range c.Samples { + if s.Name == "disabled-sample" { + found = true + } + } + assert.False(t, found, "disabled sample should be removed by Clean()") + + foundEnabled := false + for _, s := range c.Samples { + if s.Name == "enabled-sample" { + foundEnabled = true + } + } + assert.True(t, foundEnabled, "enabled sample should remain after Clean()") +} + +func TestParseBeginEndWithEndIntervals(t *testing.T) { + s := &Sample{ + Name: "test", + EndIntervals: 3, + Interval: 5, + } + + ParseBeginEnd(s) + + assert.Equal(t, "-15s", s.Begin) + assert.Equal(t, "now", s.End) + assert.False(t, s.Realtime) + assert.False(t, s.BeginParsed.IsZero()) + assert.False(t, s.EndParsed.IsZero()) +} + +func TestParseBeginEndEmptyEnd(t *testing.T) { + s := &Sample{ + Name: "test", + End: "", + } + + ParseBeginEnd(s) + + // Empty end means realtime + assert.True(t, s.Realtime) + assert.True(t, s.EndParsed.IsZero()) +} + +func TestParseBeginEndBeginOverridesRealtime(t *testing.T) { + s := &Sample{ + Name: "test", + Begin: "-60s", + End: "", + } + + ParseBeginEnd(s) + + // Begin set without endIntervals: sets Realtime to false via parsing + assert.False(t, s.Realtime) + assert.False(t, s.BeginParsed.IsZero()) +} + +func TestSetupFromFile(t *testing.T) { + SetupFromFile("/tmp/testfile.yml") + defer CleanupConfigAndEnvironment() + + assert.Equal(t, "..", os.Getenv("GOGEN_HOME")) + assert.Equal(t, "1", os.Getenv("GOGEN_ALWAYS_REFRESH")) + assert.Equal(t, "/tmp/testfile.yml", os.Getenv("GOGEN_FULLCONFIG")) +} + +func TestSetupSystemTokensSplunkHEC(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: splunkhec +samples: + - name: hectokensample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("hectokensample") + assert.NotNil(t, s) + + // Should have a _time token added by SetupSystemTokens + foundTime := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "epochtimestamp", tk.Type) + } + } + assert.True(t, foundTime, "splunkhec should add _time token") +} + +func TestSetupSystemTokensElasticsearch(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: elasticsearch +samples: + - name: estokensample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("estokensample") + assert.NotNil(t, s) + + foundTimestamp := false + for _, tk := range s.Tokens { + if tk.Name == "@timestamp" { + foundTimestamp = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTimestamp, "elasticsearch should add @timestamp token") +} + +func TestSetupSystemTokensRFC3164(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: rfc3164 +samples: + - name: rfc3164sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: syslog event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rfc3164sample") + assert.NotNil(t, s) + + foundTime := false + foundPriority := false + foundHost := false + foundTag := false + foundPid := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTime, "rfc3164 should add _time token") + + // Check that syslog fields were added to lines + if len(s.Lines) > 0 { + if _, ok := s.Lines[0]["priority"]; ok { + foundPriority = true + } + if _, ok := s.Lines[0]["host"]; ok { + foundHost = true + } + if _, ok := s.Lines[0]["tag"]; ok { + foundTag = true + } + if _, ok := s.Lines[0]["pid"]; ok { + foundPid = true + } + } + assert.True(t, foundPriority, "rfc3164 should add priority field") + assert.True(t, foundHost, "rfc3164 should add host field") + assert.True(t, foundTag, "rfc3164 should add tag field") + assert.True(t, foundPid, "rfc3164 should add pid field") +} + +func TestSetupSystemTokensRFC5424(t *testing.T) { + ResetConfig() + + configStr := ` +global: + output: + outputter: stdout + outputTemplate: rfc5424 +samples: + - name: rfc5424sample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: syslog5424 event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rfc5424sample") + assert.NotNil(t, s) + + foundTime := false + for _, tk := range s.Tokens { + if tk.Name == "_time" { + foundTime = true + assert.Equal(t, "gotimestamp", tk.Type) + } + } + assert.True(t, foundTime, "rfc5424 should add _time token") + + if len(s.Lines) > 0 { + _, hasAppName := s.Lines[0]["appName"] + assert.True(t, hasAppName, "rfc5424 should add appName field") + } +} + +func TestBuildConfigDefaults(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: defaultsample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + + // Check defaults were applied + assert.Equal(t, 1, c.Global.GeneratorWorkers) + assert.Equal(t, 1, c.Global.OutputWorkers) + assert.Equal(t, "stdout", c.Global.Output.Outputter) + assert.Equal(t, "raw", c.Global.Output.OutputTemplate) + assert.Equal(t, 5, c.Global.Output.BackupFiles) + assert.NotZero(t, c.Global.Output.MaxBytes) + assert.NotZero(t, c.Global.Output.BufferBytes) + assert.NotZero(t, c.Global.Output.Timeout) +} + +func TestValidateDisabledNoLines(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: nolines + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // Sample with no lines should be disabled and cleaned away + s := c.FindSampleByName("nolines") + assert.Nil(t, s, "sample with no lines should be removed") +} + +func TestConvertUTC(t *testing.T) { + ResetConfig() + + configStr := ` +global: + utc: true +samples: + - name: utctest + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + _ = NewConfig() + + now := time.Now() + utcTime := convertUTC(now) + assert.Equal(t, now.UTC(), utcTime) +} + +func TestSampleNow(t *testing.T) { + s := &Sample{ + Realtime: true, + } + beforeCall := time.Now() + result := s.Now() + afterCall := time.Now() + assert.True(t, !result.Before(beforeCall) && !result.After(afterCall), + "Realtime Now() should return current time") + + fixedTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s.Realtime = false + s.Current = fixedTime + result = s.Now() + assert.Equal(t, fixedTime, result) +} + +func TestReadSamplesDir(t *testing.T) { + ResetConfig() + + // Use the existing test samples directory + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", "") + os.Setenv("GOGEN_SAMPLES_DIR", filepath.Join("..", "tests", "tokens")) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // Should have loaded samples from the tokens test directory + assert.NotEmpty(t, c.Samples, "should load samples from samples dir") +} + +func TestParseFileConfigJSON(t *testing.T) { + ResetConfig() + + // Create a JSON config file with the Config struct format + dir := t.TempDir() + jsonFile := filepath.Join(dir, "test.json") + jsonContent := `{"samples": [{"name": "jsonsample", "interval": 1, "count": 1, "endIntervals": 1, "lines": [{"_raw": "json test"}]}]}` + os.WriteFile(jsonFile, []byte(jsonContent), 0644) + + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", jsonFile) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + assert.NotEmpty(t, c.Samples, "should load samples from JSON config") +} + +func TestNegativeCacheIntervals(t *testing.T) { + ResetConfig() + + configStr := ` +global: + cacheIntervals: -5 +samples: + - name: cachesample + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + assert.Equal(t, 0, c.Global.CacheIntervals, "negative cacheIntervals should be clamped to 0") +} + +func TestReadSamplesDirSampleFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + // Create a .sample file + err := os.WriteFile(filepath.Join(dir, "test.sample"), []byte("line one\nline two\nline three\n"), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + // Should have loaded one sample with 3 lines + found := false + for _, s := range c.Samples { + if s.Name == "test.sample" { + found = true + assert.True(t, s.Disabled, ".sample files should be disabled by default") + assert.Equal(t, 3, len(s.Lines)) + assert.Equal(t, "line one", s.Lines[0]["_raw"]) + assert.Equal(t, "line two", s.Lines[1]["_raw"]) + assert.Equal(t, "line three", s.Lines[2]["_raw"]) + } + } + assert.True(t, found, "should find test.sample") +} + +func TestReadSamplesDirCSVFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + // Create a .csv file with header + csvContent := "name,city,state\nalice,NYC,NY\nbob,LA,CA\n" + err := os.WriteFile(filepath.Join(dir, "test.csv"), []byte(csvContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + found := false + for _, s := range c.Samples { + if s.Name == "test.csv" { + found = true + assert.True(t, s.Disabled, ".csv files should be disabled by default") + assert.Equal(t, 2, len(s.Lines)) + assert.Equal(t, "alice", s.Lines[0]["name"]) + assert.Equal(t, "NYC", s.Lines[0]["city"]) + assert.Equal(t, "NY", s.Lines[0]["state"]) + assert.Equal(t, "bob", s.Lines[1]["name"]) + } + } + assert.True(t, found, "should find test.csv") +} + +func TestReadSamplesDirYAMLFile(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + yamlContent := `name: yamlsample +interval: 1 +count: 1 +lines: + - _raw: yaml test line +` + err := os.WriteFile(filepath.Join(dir, "yamltest.yml"), []byte(yamlContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + found := false + for _, s := range c.Samples { + if s.Name == "yamlsample" { + found = true + assert.True(t, s.realSample) + } + } + assert.True(t, found, "should find yamlsample") +} + +func TestReadSamplesDirEmptyDir(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + + c := &Config{cc: ConfigConfig{}} + c.readSamplesDir(dir) + + // Should not crash and should have no samples + assert.Empty(t, c.Samples) +} + +func TestReadGeneratorFallbackPath(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + genDir := filepath.Join(dir, "generators") + os.MkdirAll(genDir, 0755) + + // Create generator script in the fallback directory + scriptContent := `-- test generator\nsetToken("test", "value")\n` + err := os.WriteFile(filepath.Join(genDir, "testgen.lua"), []byte(scriptContent), 0644) + assert.NoError(t, err) + + c := &Config{cc: ConfigConfig{ConfigDir: dir}} + g := &GeneratorConfig{Name: "testgen", FileName: "testgen.lua"} + + err = c.readGenerator(dir, g) + assert.NoError(t, err) + assert.Contains(t, g.Script, "test generator") +} + +func TestReadGeneratorNotFound(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + c := &Config{cc: ConfigConfig{ConfigDir: dir}} + g := &GeneratorConfig{Name: "missing", FileName: "nonexistent.lua"} + + err := c.readGenerator(dir, g) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Cannot find generator file") +} + +func TestValidateTokenRandomString(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: randstring + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: rs + format: template + token: $rs$ + type: random + replacement: string + length: 10 + lines: + - _raw: $rs$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("randstring") + assert.NotNil(t, s, "sample with valid random string token should not be disabled") +} + +func TestValidateTokenRandomStringZeroLength(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: randstringbad + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: rs + format: template + token: $rs$ + type: random + replacement: string + length: 0 + lines: + - _raw: $rs$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("randstringbad") + assert.Nil(t, s, "sample with zero-length random string should be disabled") +} + +func TestValidateTokenReplacementTypes(t *testing.T) { + tests := []struct { + name string + replacement string + extra string + valid bool + }{ + {"hex", "hex", "length: 5", true}, + {"guid", "guid", "", true}, + {"ipv4", "ipv4", "", true}, + {"ipv6", "ipv6", "", true}, + {"invalid", "invalid_replacement_xyz", "", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ResetConfig() + extra := "" + if tc.extra != "" { + extra = "\n " + tc.extra + } + configStr := ` +samples: + - name: ` + tc.name + ` + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: tk + format: template + token: $tk$ + type: random + replacement: ` + tc.replacement + extra + ` + lines: + - _raw: $tk$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName(tc.name) + if tc.valid { + assert.NotNil(t, s, "%s should not be disabled", tc.name) + } else { + assert.Nil(t, s, "%s should be disabled", tc.name) + } + }) + } +} + +func TestValidateTokenScript(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: scripttest + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: sc + format: template + token: $sc$ + type: script + init: + myvar: "42" + scriptSrc: | + return "hello" + lines: + - _raw: $sc$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("scripttest") + assert.NotNil(t, s, "sample with script token should not be disabled") + // Check that script token has mutex + for _, tk := range s.Tokens { + if tk.Name == "sc" { + assert.NotNil(t, tk.mutex, "script token should have mutex initialized") + } + } +} + +func TestValidateNoInterval(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: nointerval + count: 1 + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("nointerval") + assert.NotNil(t, s) + assert.Equal(t, 1, s.EndIntervals, "no interval should auto-set endIntervals to 1") +} + +func TestValidateEmptyName(t *testing.T) { + ResetConfig() + + c := &Config{ + Global: Global{ + Output: Output{ + Outputter: "stdout", + OutputTemplate: "raw", + }, + }, + } + s := &Sample{ + realSample: true, + Name: "", + Lines: []map[string]string{{"_raw": "test"}}, + } + c.validate(s) + assert.True(t, s.Disabled, "sample with empty name should be disabled") +} + +func TestValidateRaterWithIntValues(t *testing.T) { + ResetConfig() + configStr := ` +raters: + - name: testrater + type: config + options: + HourOfDay: + 0: 1 + 12: 2 + DayOfWeek: + 0: 1.5 + 6: 0.5 +samples: + - name: ratertest + interval: 1 + count: 1 + endIntervals: 1 + rater: testrater + lines: + - _raw: test +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + r := c.FindRater("testrater") + assert.NotNil(t, r) + // HourOfDay should be converted to map[int]float64 + hod, ok := r.Options["HourOfDay"].(map[int]float64) + assert.True(t, ok, "HourOfDay should be map[int]float64") + assert.Equal(t, 1.0, hod[0]) + assert.Equal(t, 2.0, hod[12]) +} + +func TestValidateWeightedChoice(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: weightsource + disabled: true + lines: + - value: alpha + _weight: "3" + - value: beta + _weight: "7" + - name: weightuser + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: wt + format: template + token: $wt$ + type: weightedChoice + sample: weightsource + srcField: value + lines: + - _raw: $wt$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("weightuser") + assert.NotNil(t, s) + for _, tk := range s.Tokens { + if tk.Name == "wt" { + assert.NotEmpty(t, tk.WeightedChoice, "should have weighted choices resolved") + assert.Equal(t, 2, len(tk.WeightedChoice)) + } + } +} + +func TestValidateTokenSampleResolution(t *testing.T) { + ResetConfig() + configStr := ` +samples: + - name: choices + disabled: true + lines: + - _raw: alpha + - _raw: beta + - _raw: gamma + - name: resolver + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: pick + format: template + token: $pick$ + type: choice + sample: choices + lines: + - _raw: $pick$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("resolver") + assert.NotNil(t, s) + for _, tk := range s.Tokens { + if tk.Name == "pick" { + assert.Equal(t, 3, len(tk.Choice), "should resolve 3 choices from sample") + assert.Contains(t, tk.Choice, "alpha") + assert.Contains(t, tk.Choice, "beta") + assert.Contains(t, tk.Choice, "gamma") + } + } +} + +func TestValidateExportMode(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + configFile := filepath.Join(dir, "export.yml") + os.WriteFile(configFile, []byte(` +samples: + - name: exportsample + interval: 1 + count: 1 + lines: + - _raw: test +`), 0644) + + cc := ConfigConfig{ + FullConfig: configFile, + Export: true, + } + c := BuildConfig(cc) + + // In export mode, defaults should NOT be set + assert.Equal(t, 0, c.Global.GeneratorWorkers, "export mode should not set defaults") + assert.Equal(t, "", c.Global.Output.Outputter, "export mode should not set output defaults") +} + +func TestMergeMixConfig(t *testing.T) { + c := &Config{} + nc := &Config{ + Samples: []*Sample{ + {Name: "mixsample", Count: 5, Interval: 2}, + }, + } + m := &Mix{ + Count: 10, + Interval: 3, + Begin: "-60s", + End: "now", + } + c.mergeMixConfig(nc, m) + + assert.Equal(t, 1, len(c.Samples)) + assert.Equal(t, "mixsample", c.Samples[0].Name) + assert.Equal(t, 10, c.Samples[0].Count) + assert.Equal(t, 3, c.Samples[0].Interval) +} + +func TestParseFileConfigYAMLError(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + badFile := filepath.Join(dir, "bad.yml") + // Invalid YAML: tabs mixed with spaces in wrong places + os.WriteFile(badFile, []byte("{\n bad yaml content: [unclosed\n"), 0644) + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, badFile) + // parseFileConfig logs errors but returns nil + assert.NoError(t, err) +} + +func TestParseFileConfigJSONError(t *testing.T) { + ResetConfig() + + dir := t.TempDir() + badFile := filepath.Join(dir, "bad.json") + os.WriteFile(badFile, []byte("{invalid json"), 0644) + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, badFile) + assert.NoError(t, err) +} + +func TestParseFileConfigNotExists(t *testing.T) { + ResetConfig() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseFileConfig(s, "/nonexistent/path/file.yml") + assert.Error(t, err) +} + +func TestParseWebConfigSuccess(t *testing.T) { + ResetConfig() + + yamlContent := ` +name: websample +interval: 1 +count: 1 +lines: + - _raw: web test +` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(yamlContent)) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + assert.NoError(t, err) + assert.Equal(t, "websample", s.Name) +} + +func TestParseWebConfigJSONFallback(t *testing.T) { + ResetConfig() + + jsonContent := `{"name": "jsonsample", "interval": 1}` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(jsonContent)) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + assert.NoError(t, err) + assert.Equal(t, "jsonsample", s.Name) +} + +func TestParseWebConfigBadContent(t *testing.T) { + ResetConfig() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("<<>>")) + })) + defer ts.Close() + + c := &Config{cc: ConfigConfig{}} + s := &Sample{} + err := c.parseWebConfig(s, ts.URL) + // JSON fallback parse returns an error for non-JSON content + assert.Error(t, err) + assert.Equal(t, "", s.Name, "garbage content should not set sample name") +} + +func TestMergeMixConfigDuplicate(t *testing.T) { + c := &Config{ + Samples: []*Sample{ + {Name: "existing"}, + }, + } + nc := &Config{ + Samples: []*Sample{ + {Name: "existing", Count: 5}, + }, + } + m := &Mix{} + c.mergeMixConfig(nc, m) + + // Should not add duplicate + assert.Equal(t, 1, len(c.Samples)) +} + +func TestGetAPIURLDefault(t *testing.T) { + os.Unsetenv("GOGEN_APIURL") + url := getAPIURL() + assert.Equal(t, "https://api.gogen.io", url) +} + +func TestGetAPIURLCustom(t *testing.T) { + os.Setenv("GOGEN_APIURL", "http://localhost:4000") + defer os.Unsetenv("GOGEN_APIURL") + url := getAPIURL() + assert.Equal(t, "http://localhost:4000", url) +} + +func TestValidateFromSample(t *testing.T) { + ResetConfig() + + configStr := ` +samples: + - name: sourcesample + disabled: true + lines: + - _raw: source line 1 + - _raw: source line 2 + - name: copiedsample + fromSample: sourcesample + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("copiedsample") + assert.NotNil(t, s) + assert.Len(t, s.Lines, 2, "copiedsample should have lines from sourcesample") +} + +func TestNewGeneratorStateNumericInit(t *testing.T) { + s := &Sample{ + CustomGenerator: &GeneratorConfig{ + Init: map[string]string{ + "count": "42", + "rate": "3.14", + "label": "hello", + }, + }, + Lines: []map[string]string{ + {"_raw": "line1", "host": "h1"}, + {"_raw": "line2", "host": "h2"}, + }, + } + + gs := NewGeneratorState(s) + assert.NotNil(t, gs.LuaState) + assert.NotNil(t, gs.LuaLines) + + // Numeric values should be stored as LNumber + countVal := gs.LuaState.RawGetString("count") + assert.NotNil(t, countVal) + + // String values should be stored as LString + labelVal := gs.LuaState.RawGetString("label") + assert.NotNil(t, labelVal) + + // Lines table should have entries + assert.Equal(t, 2, gs.LuaLines.Len()) +} + +func TestNewGeneratorStateEmptyInit(t *testing.T) { + s := &Sample{ + CustomGenerator: &GeneratorConfig{ + Init: map[string]string{}, + }, + Lines: []map[string]string{}, + } + + gs := NewGeneratorState(s) + assert.NotNil(t, gs.LuaState) + assert.NotNil(t, gs.LuaLines) + assert.Equal(t, 0, gs.LuaLines.Len()) +} + +func TestBuildConfigExportMode(t *testing.T) { + ResetConfig() + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + home := filepath.Join("..", "tests", "tokens") + os.Setenv("GOGEN_SAMPLES_DIR", home) + + cc := ConfigConfig{ + SamplesDir: home, + Home: "..", + Export: true, + } + c := BuildConfig(cc) + assert.NotNil(t, c) + // In export mode, samples should have lines populated inline + for _, s := range c.Samples { + if s.Name == "tokens" { + assert.Greater(t, len(s.Lines), 0) + } + } +} + +func TestBuildConfigWithGlobalFile(t *testing.T) { + ResetConfig() + globalFile := filepath.Join("..", "tests", "rater", "defaultrater.yml") + cc := ConfigConfig{ + FullConfig: globalFile, + Home: "..", + } + c := BuildConfig(cc) + assert.NotNil(t, c) +} + +func TestValidateInvalidEarliestTime(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: badtime + description: "Bad earliest time" + earliest: "not_a_valid_time_string!!!" + latest: now + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("badtime") + // With invalid earliest, EarliestParsed should default to 0 + assert.Equal(t, time.Duration(0), s.EarliestParsed) +} + +func TestValidateInvalidLatestTime(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: badlatest + description: "Bad latest time" + earliest: now + latest: "not_a_valid_time_string!!!" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("badlatest") + // With invalid latest, LatestParsed should default to 0 + assert.Equal(t, time.Duration(0), s.LatestParsed) +} + +func TestNewConfigNoGogenHome(t *testing.T) { + ResetConfig() + os.Unsetenv("GOGEN_HOME") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Unsetenv("GOGEN_FULLCONFIG") + os.Unsetenv("GOGEN_CONFIG_DIR") + os.Unsetenv("GOGEN_SAMPLES_DIR") + + c := NewConfig() + assert.NotNil(t, c) + // When GOGEN_HOME is not set, it should default to "." + assert.Equal(t, ".", os.Getenv("GOGEN_HOME")) +} + +func TestValidateNoLinesDisablesSample(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: nolines + description: "Sample with no lines" + interval: 1 + count: 1 + endIntervals: 1 +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + // After Clean(), disabled samples are removed + // So FindSampleByName should return an empty sample (not in the list) + found := false + for _, s := range c.Samples { + if s.Name == "nolines" { + found = true + } + } + assert.False(t, found, "disabled sample with no lines should be removed by Clean()") +} + +func TestValidateRatedTokenDefaultRater(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: rated_test + description: "Rated token default rater" + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: myrated + format: template + type: rated + replacement: int + lower: 0 + upper: 100 + lines: + - _raw: value=$myrated$ +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("rated_test") + // Rated token with no raterString should default to "default" + for _, tok := range s.Tokens { + if tok.Name == "myrated" { + assert.Equal(t, "default", tok.RaterString) + } + } +} + +func TestValidateLuaGenerator(t *testing.T) { + ResetConfig() + configStr := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: luagen + description: "Lua generator sample" + generator: mygen + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +generators: + - name: mygen + script: | + lines = getLines() + return send(lines) +` + SetupFromString(configStr) + defer CleanupConfigAndEnvironment() + + c := NewConfig() + s := c.FindSampleByName("luagen") + assert.NotNil(t, s) + assert.Equal(t, "mygen", s.Generator) + assert.NotNil(t, s.CustomGenerator) +} diff --git a/internal/sample_test.go b/internal/sample_test.go index b05e008..2f66e89 100644 --- a/internal/sample_test.go +++ b/internal/sample_test.go @@ -230,6 +230,136 @@ func benchmarkToken(conf string, i int, b *testing.B) { } } +// mockRater implements Rater for testing rated tokens +type mockRater struct { + rate float64 +} + +func (m *mockRater) EventRate(s *Sample, now time.Time, count int) float64 { return m.rate } +func (m *mockRater) TokenRate(t Token, now time.Time) float64 { return m.rate } + +func TestGenReplacementRatedInt(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedint", + Type: "rated", + Replacement: "int", + Lower: 10, + Upper: 20, + Rater: &mockRater{rate: 2.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + // With rate=2.0, value should be roughly doubled + val, _ := fmt.Sscanf(result, "%d", new(int)) + assert.Equal(t, 1, val, "should parse as an integer") +} + +func TestGenReplacementRatedIntEqualBounds(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedintequal", + Type: "rated", + Replacement: "int", + Lower: 5, + Upper: 5, + Rater: &mockRater{rate: 1.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "5", result) +} + +func TestGenReplacementRatedFloat(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedfloat", + Type: "rated", + Replacement: "float", + Lower: 1, + Upper: 10, + Precision: 2, + Rater: &mockRater{rate: 1.5}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + // Should be a float string with 2 decimal places + assert.Contains(t, result, ".") +} + +func TestGenReplacementRatedFloatEqualBounds(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "ratedfloatequal", + Type: "rated", + Replacement: "float", + Lower: 5, + Upper: 5, + Precision: 2, + Rater: &mockRater{rate: 1.0}, + } + result, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "5.00", result) +} + +func TestGenReplacementFieldChoice(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "fieldchoice", + Type: "fieldChoice", + SrcField: "city", + FieldChoice: []map[string]string{ + {"city": "NYC", "state": "NY"}, + {"city": "LA", "state": "CA"}, + }, + } + result, choice, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.True(t, result == "NYC" || result == "LA") + assert.True(t, choice >= 0) + + // Test with specific choice + result, _, err = token.GenReplacement(0, now, now, now, randgen, fullevent) + assert.NoError(t, err) + assert.Equal(t, "NYC", result) +} + +func TestGenReplacementInvalidType(t *testing.T) { + source := rand.NewSource(0) + randgen := rand.New(source) + fullevent := make(map[string]string) + now := time.Now() + + token := Token{ + Name: "badtype", + Type: "nonexistenttype", + } + _, _, err := token.GenReplacement(-1, now, now, now, randgen, fullevent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid type") +} + func BenchmarkReplacement(b *testing.B) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1") diff --git a/internal/share_test.go b/internal/share_test.go index 1793336..188e974 100644 --- a/internal/share_test.go +++ b/internal/share_test.go @@ -146,3 +146,251 @@ func TestSharePullFile(t *testing.T) { _, err = os.Stat(filepath.Join(os.ExpandEnv("$GOGEN_TMPDIR"), ".configcache_testuser%2Ftestconfig")) assert.NoError(t, err, "Couldn't find cache file") } + +func TestSharePullShortName(t *testing.T) { + // Test Pull with a short name (no "/" in gogen string) + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "shortname", + Name: "shortname", + Description: "Short name test", + Owner: "testuser", + Version: 1, + Config: "sample: test\nname: shortname", + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("shortname", dir, false) + _, err := os.Stat(filepath.Join(dir, "shortname.yml")) + assert.NoError(t, err, "Couldn't find file shortname.yml") +} + +func TestSharePullFileCached(t *testing.T) { + // Test PullFile when cache exists and version matches → uses cached content + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/cached", + Name: "cached", + Owner: "testuser", + Version: 5, + Config: "should not be used", + }, nil + } + + tmpdir := t.TempDir() + os.Setenv("GOGEN_TMPDIR", tmpdir) + defer os.Unsetenv("GOGEN_TMPDIR") + + // Pre-create version cache with matching version + versionCacheFile := filepath.Join(tmpdir, ".versioncache_testuser%2Fcached") + os.WriteFile(versionCacheFile, []byte("5"), 0644) + + // Pre-create config cache with different content + cacheFile := filepath.Join(tmpdir, ".configcache_testuser%2Fcached") + os.WriteFile(cacheFile, []byte("cached config content"), 0644) + + outFile := filepath.Join(tmpdir, "output.yml") + PullFile("testuser/cached", outFile) + + // Should use cached content, not the API response + data, err := os.ReadFile(outFile) + assert.NoError(t, err) + assert.Equal(t, "cached config content", string(data)) +} + +func TestSharePullFileVersionMismatch(t *testing.T) { + // Test PullFile when cache version doesn't match → uses API content and updates cache + originalGet := Get + defer func() { Get = originalGet }() + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/mismatch", + Name: "mismatch", + Owner: "testuser", + Version: 10, + Config: "fresh api content", + }, nil + } + + tmpdir := t.TempDir() + os.Setenv("GOGEN_TMPDIR", tmpdir) + defer os.Unsetenv("GOGEN_TMPDIR") + + // Pre-create version cache with OLD version + versionCacheFile := filepath.Join(tmpdir, ".versioncache_testuser%2Fmismatch") + os.WriteFile(versionCacheFile, []byte("5"), 0644) + + // Pre-create config cache with old content + cacheFile := filepath.Join(tmpdir, ".configcache_testuser%2Fmismatch") + os.WriteFile(cacheFile, []byte("old cached content"), 0644) + + outFile := filepath.Join(tmpdir, "output.yml") + PullFile("testuser/mismatch", outFile) + + // Should use API content since version doesn't match + data, err := os.ReadFile(outFile) + assert.NoError(t, err) + assert.Equal(t, "fresh api content", string(data)) + + // Cache files should be updated + versionData, _ := os.ReadFile(versionCacheFile) + assert.Equal(t, "10", string(versionData)) + + cachedData, _ := os.ReadFile(cacheFile) + assert.Equal(t, "fresh api content", string(cachedData)) +} + +func TestSharePullWithDeconstructCSV(t *testing.T) { + // Test deconstructConfig with CSV fieldChoice tokens + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: csvtest + description: CSV deconstruct test + interval: 1 + count: 1 + endIntervals: 1 + tokens: + - name: city + format: template + type: fieldChoice + field: _raw + srcField: city + sample: markets.csv + fieldChoice: + - city: NYC + state: NY + - city: LA + state: CA + lines: + - _raw: city=$city$ +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/csvtest", + Name: "csvtest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/csvtest", dir, true) + _, err := os.Stat(filepath.Join(dir, "samples", "markets.csv")) + assert.NoError(t, err, "Couldn't find samples/markets.csv") + _, err = os.Stat(filepath.Join(dir, "samples", "csvtest.yml")) + assert.NoError(t, err, "Couldn't find samples/csvtest.yml") +} + +func TestSharePullWithDeconstructGenerator(t *testing.T) { + // Test deconstructConfig with generator that has a fileName + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: raw +samples: + - name: gentest + description: Generator deconstruct test + generator: mygen + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +generators: + - name: mygen + fileName: /path/to/mygen.lua + script: | + lines = getLines() + return send(lines) +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/gentest", + Name: "gentest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/gentest", dir, true) + _, err := os.Stat(filepath.Join(dir, "generators", "mygen.lua")) + assert.NoError(t, err, "Couldn't find generators/mygen.lua") + _, err = os.Stat(filepath.Join(dir, "generators", "mygen.yml")) + assert.NoError(t, err, "Couldn't find generators/mygen.yml") +} + +func TestSharePullWithDeconstructTemplates(t *testing.T) { + // Test deconstructConfig with templates + originalGet := Get + defer func() { Get = originalGet }() + + configYaml := ` +global: + rotInterval: 1 + output: + outputter: devnull + outputTemplate: mytemplate +samples: + - name: tmpltest + description: Template deconstruct test + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: test event +templates: + - name: mytemplate + header: "HEADER\n" + row: "{{._raw}}\n" + footer: "FOOTER\n" +` + + Get = func(q string) (GogenInfo, error) { + return GogenInfo{ + Gogen: "testuser/tmpltest", + Name: "tmpltest", + Owner: "testuser", + Version: 1, + Config: configYaml, + }, nil + } + + os.Setenv("GOGEN_HOME", "..") + dir := t.TempDir() + + Pull("testuser/tmpltest", dir, true) + _, err := os.Stat(filepath.Join(dir, "templates", "mytemplate.yml")) + assert.NoError(t, err, "Couldn't find templates/mytemplate.yml") +} diff --git a/outputter/send_test.go b/outputter/send_test.go new file mode 100644 index 0000000..1f52406 --- /dev/null +++ b/outputter/send_test.go @@ -0,0 +1,682 @@ +package outputter + +import ( + "bytes" + "io" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/stretchr/testify/assert" +) + +func TestSetup(t *testing.T) { + tests := []struct { + name string + outputter string + expectedType interface{} + }{ + {"stdout", "stdout", &stdout{}}, + {"devnull", "devnull", &devnull{}}, + {"file", "file", &file{}}, + {"http", "http", &httpout{}}, + {"buf", "buf", &buf{}}, + {"network", "network", &network{}}, + {"kafka", "kafka", &kafkaout{}}, + {"unknown defaults to stdout", "unknowntype", &stdout{}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Use a unique gout slot for each test + num := 99 // use last slot to avoid conflicts + gout[num] = nil + + s := &config.Sample{ + Name: "test", + Output: &config.Output{ + Outputter: tc.outputter, + }, + } + item := &config.OutQueueItem{S: s} + source := rand.NewSource(0) + gen := rand.New(source) + + result := setup(gen, item, num) + assert.IsType(t, tc.expectedType, result) + + // Clean up + gout[num] = nil + }) + } +} + +func TestDevnullSend(t *testing.T) { + d := &devnull{} + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: &config.Sample{Name: "test"}, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "test data") + oio.W.Close() + }() + + err := d.Send(item) + assert.NoError(t, err) +} + +func TestDevnullClose(t *testing.T) { + d := &devnull{} + err := d.Close() + assert.NoError(t, err) +} + +func TestBufSend(t *testing.T) { + var b bytes.Buffer + s := &config.Sample{ + Name: "test", + Buf: &b, + } + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: s, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "buffered data\n") + oio.W.Close() + }() + + bu := &buf{} + err := bu.Send(item) + assert.NoError(t, err) + assert.Equal(t, "buffered data\n", b.String()) +} + +func TestStdoutSend(t *testing.T) { + // Redirect stdout to a pipe + origStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + oio := config.NewOutputIO() + item := &config.OutQueueItem{ + S: &config.Sample{Name: "test"}, + IO: oio, + } + + go func() { + io.WriteString(oio.W, "stdout data\n") + oio.W.Close() + }() + + so := &stdout{} + err := so.Send(item) + assert.NoError(t, err) + + w.Close() + var buf bytes.Buffer + io.Copy(&buf, r) + os.Stdout = origStdout + + assert.Equal(t, "stdout data\n", buf.String()) +} + +func TestStdoutClose(t *testing.T) { + so := &stdout{} + err := so.Close() + assert.NoError(t, err) +} + +func TestFileSendAndRotation(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "testfile.log") + + s := &config.Sample{ + Name: "filesample", + Output: &config.Output{ + FileName: filename, + MaxBytes: 50, // Very small to trigger rotation + BackupFiles: 2, + }, + } + + f := &file{} + + // Write enough data to trigger rotation + for i := 0; i < 5; i++ { + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + + go func() { + io.WriteString(oio.W, strings.Repeat("X", 30)+"\n") + oio.W.Close() + }() + + err := f.Send(item) + assert.NoError(t, err) + } + + // Check that backup files were created + _, err := os.Stat(filename) + assert.NoError(t, err, "main file should exist") + + _, err = os.Stat(filename + ".1") + assert.NoError(t, err, "backup .1 should exist") + + f.Close() +} + +func TestFileSendExistingFile(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "existing.log") + + // Pre-create the file with some content + os.WriteFile(filename, []byte("pre-existing data\n"), 0644) + + s := &config.Sample{ + Name: "fileexisting", + Output: &config.Output{ + FileName: filename, + MaxBytes: 10000000, + BackupFiles: 2, + }, + } + + f := &file{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + + go func() { + io.WriteString(oio.W, "appended data\n") + oio.W.Close() + }() + + err := f.Send(item) + assert.NoError(t, err) + + // Verify the file has both old and new data + data, _ := os.ReadFile(filename) + assert.Contains(t, string(data), "pre-existing data") + assert.Contains(t, string(data), "appended data") + + f.Close() +} + +func TestFileClose(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "closefile.log") + + s := &config.Sample{ + Name: "fileclose", + Output: &config.Output{ + FileName: filename, + MaxBytes: 1000000, + BackupFiles: 2, + }, + } + + f := &file{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, "data\n") + oio.W.Close() + }() + f.Send(item) + + // Close should work + err := f.Close() + assert.NoError(t, err) + + // Close again should be idempotent + err = f.Close() + assert.NoError(t, err) +} + +func TestHTTPSendAndFlush(t *testing.T) { + var received bytes.Buffer + var mu sync.Mutex + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + io.Copy(&received, r.Body) + mu.Unlock() + w.WriteHeader(200) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpsample", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, // Small buffer to trigger flush + Headers: map[string]string{"Content-Type": "application/json"}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + // Send enough data to exceed buffer and trigger flush + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("D", 50)+"\n") + oio.W.Close() + }() + + err := h.Send(item) + assert.NoError(t, err) + + // Verify server received data + mu.Lock() + data := received.String() + mu.Unlock() + assert.NotEmpty(t, data, "HTTP server should have received data") + + // Close flushes remaining data + err = h.Close() + assert.NoError(t, err) +} + +func TestNetworkSend(t *testing.T) { + // Start a TCP listener on a random port + ln, err := net.Listen("tcp", "127.0.0.1:0") + assert.NoError(t, err) + defer ln.Close() + + var received bytes.Buffer + done := make(chan struct{}) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + io.Copy(&received, conn) + conn.Close() + close(done) + }() + + s := &config.Sample{ + Name: "netsample", + Output: &config.Output{ + Endpoints: []string{ln.Addr().String()}, + Protocol: "tcp", + Timeout: 5 * time.Second, + }, + } + + n := &network{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, "network data\n") + oio.W.Close() + }() + + err = n.Send(item) + assert.NoError(t, err) + + // Close the connection so the listener goroutine can finish + n.Close() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for network data") + } + + assert.Equal(t, "network data\n", received.String()) +} + +func TestNetworkClose(t *testing.T) { + n := &network{} + // Close with no connection should not error + err := n.Close() + assert.NoError(t, err) + assert.False(t, n.initialized) +} + +func TestBufClose(t *testing.T) { + b := &buf{} + err := b.Close() + assert.NoError(t, err) +} + +func TestStartDevnullWorker(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Reset gout slot + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + // Send an item through the pipeline + s := &config.Sample{ + Name: "starttest", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + events := []map[string]string{ + {"_raw": "test event for start"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + // Close the queue and wait for worker to finish + close(oq) + select { + case <-oqs: + // Worker finished + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // Verify gout slot was cleared + assert.Nil(t, gout[0]) +} + +func TestStartMultipleItems(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "multistart", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + + for i := 0; i < 5; i++ { + events := []map[string]string{ + {"_raw": "event number"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + } + + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // Check that events were accounted for + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + ew := EventsWritten["multistart"] + Mutex.RUnlock() + assert.Equal(t, int64(5), ew) +} + +func TestStartEmptyEvents(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "emptyevents", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + // Send item with no events - should skip the write/send + item := &config.OutQueueItem{ + S: s, + Events: []map[string]string{}, + Cache: &config.CacheItem{}, + } + oq <- item + + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} + +func TestStartCloseOnChannelClose(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "closetest", + Output: &config.Output{ + Outputter: "devnull", + OutputTemplate: "raw", + }, + } + // Send one real item so lastS is set, then close + events := []map[string]string{ + {"_raw": "test event"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + // Close the channel - should trigger the Close() path on the outputter + close(oq) + select { + case <-oqs: + case <-time.After(5 * time.Second): + t.Fatal("Start worker did not finish in time") + } + + // gout should be cleared + assert.Nil(t, gout[0]) +} + +func TestStartSendError(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Use a network outputter pointed at a bad address to trigger Send error + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "senderror", + Output: &config.Output{ + Outputter: "network", + OutputTemplate: "raw", + Endpoints: []string{"127.0.0.1:1"}, // Should fail to connect + Protocol: "tcp", + }, + } + events := []map[string]string{ + {"_raw": "error event"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + + close(oq) + select { + case <-oqs: + case <-time.After(10 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} + +func TestHTTPFlushNon200(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + w.Write([]byte("internal server error")) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpfail", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, + Headers: map[string]string{"Content-Type": "text/plain"}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("X", 50)+"\n") + oio.W.Close() + }() + + err := h.Send(item) + // flush should return an error due to non-200 status + assert.Error(t, err) + assert.Contains(t, err.Error(), "500") +} + +func TestHTTPCloseFlushError(t *testing.T) { + // Use a server that accepts the first request (Send flush) but returns error on the second (Close flush) + calls := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 { + w.WriteHeader(500) + w.Write([]byte("flush error")) + return + } + w.WriteHeader(200) + })) + defer ts.Close() + + s := &config.Sample{ + Name: "httpclose", + Output: &config.Output{ + Endpoints: []string{ts.URL}, + BufferBytes: 10, // Small buffer to trigger flush on first Send + Headers: map[string]string{}, + Timeout: 5 * time.Second, + }, + } + + h := &httpout{} + + // First Send: exceeds buffer, triggers flush (call #1 → 200 OK) + oio := config.NewOutputIO() + item := &config.OutQueueItem{S: s, IO: oio} + go func() { + io.WriteString(oio.W, strings.Repeat("X", 50)) + oio.W.Close() + }() + + err := h.Send(item) + assert.NoError(t, err) + + // Add more data to buffer for Close to flush + h.buf.WriteString("leftover data") + + // Close should flush remaining data and get 500 error (call #2) + err = h.Close() + assert.Error(t, err) + assert.Contains(t, err.Error(), "500") +} + +func TestStartSendErrorRepeat(t *testing.T) { + cleanup := initROT() + defer cleanup() + + gout[0] = nil + + oq := make(chan *config.OutQueueItem) + oqs := make(chan int) + + go Start(oq, oqs, 0) + + s := &config.Sample{ + Name: "senderrorrepeat", + Output: &config.Output{ + Outputter: "network", + OutputTemplate: "raw", + Endpoints: []string{"127.0.0.1:1"}, + Protocol: "tcp", + }, + } + // Send multiple items to trigger repeat error path (lasterr[num].count++) + for i := 0; i < 3; i++ { + events := []map[string]string{ + {"_raw": "error event repeat"}, + } + item := &config.OutQueueItem{ + S: s, + Events: events, + Cache: &config.CacheItem{}, + } + oq <- item + } + + close(oq) + select { + case <-oqs: + case <-time.After(15 * time.Second): + t.Fatal("Start worker did not finish in time") + } +} diff --git a/outputter/write_test.go b/outputter/write_test.go new file mode 100644 index 0000000..d321cbe --- /dev/null +++ b/outputter/write_test.go @@ -0,0 +1,451 @@ +package outputter + +import ( + "bytes" + "encoding/json" + "io" + "strings" + "sync" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/template" + "github.com/stretchr/testify/assert" +) + +// initROT initializes the rotchan and readStats goroutine needed by write(). +// Returns a cleanup function to call via defer. +func initROT() func() { + Mutex.Lock() + BytesWritten = make(map[string]int64) + EventsWritten = make(map[string]int64) + rotwg = sync.WaitGroup{} + rotchan = make(chan *config.OutputStats) + Mutex.Unlock() + rotwg.Add(1) + go readStats() + return func() { + close(rotchan) + rotwg.Wait() + } +} + +func makeOutQueueItem(sampleName, outputTemplate, outputter string, events []map[string]string) *config.OutQueueItem { + s := &config.Sample{ + Name: sampleName, + Output: &config.Output{ + Outputter: outputter, + OutputTemplate: outputTemplate, + }, + } + oio := config.NewOutputIO() + return &config.OutQueueItem{ + S: s, + Events: events, + IO: oio, + Cache: &config.CacheItem{}, + } +} + +func readFromPipe(item *config.OutQueueItem) string { + var buf bytes.Buffer + io.Copy(&buf, item.IO.R) + return buf.String() +} + +func TestWriteRaw(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "hello world"}, + } + item := makeOutQueueItem("rawsample", "raw", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "hello world") +} + +func TestWriteJSON(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "test event", "host": "myhost"}, + } + item := makeOutQueueItem("jsonsample", "json", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + var parsed map[string]string + lines := strings.TrimSpace(result) + err := json.Unmarshal([]byte(strings.Split(lines, "\n")[0]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "test event", parsed["_raw"]) + assert.Equal(t, "myhost", parsed["host"]) +} + +func TestWriteSplunkHEC(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "splunk event", "_time": "1234567890"}, + } + item := makeOutQueueItem("hecsample", "splunkhec", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + var parsed map[string]string + lines := strings.TrimSpace(result) + err := json.Unmarshal([]byte(strings.Split(lines, "\n")[0]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "splunk event", parsed["event"], "splunkhec should remap _raw to event") + assert.Equal(t, "1234567890", parsed["time"], "splunkhec should remap _time to time") + assert.Empty(t, parsed["_raw"], "_raw should be deleted") + assert.Empty(t, parsed["_time"], "_time should be deleted") +} + +func TestWriteRFC3164(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "syslog msg", "_time": "Oct 20 12:00:00", "priority": "13", "host": "myhost", "tag": "gogen", "pid": "1234"}, + } + item := makeOutQueueItem("rfc3164sample", "rfc3164", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "<13>") + assert.Contains(t, result, "Oct 20 12:00:00") + assert.Contains(t, result, "myhost") + assert.Contains(t, result, "gogen[1234]") + assert.Contains(t, result, "syslog msg") +} + +func TestWriteRFC5424(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "syslog5424 msg", "_time": "2001-10-20T12:00:00Z", "priority": "13", "host": "myhost", "appName": "gogen", "pid": "1234", "extra": "val"}, + } + item := makeOutQueueItem("rfc5424sample", "rfc5424", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "<13>1") + assert.Contains(t, result, "myhost") + assert.Contains(t, result, "gogen") + assert.Contains(t, result, "syslog5424 msg") + assert.Contains(t, result, "[meta") + assert.Contains(t, result, `extra="val"`) +} + +func TestWriteElasticsearch(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "es event", "index": "testindex"}, + } + item := makeOutQueueItem("essample", "elasticsearch", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, `"_index": "testindex"`) + assert.Contains(t, result, `"_type": "doc"`) + // _raw should be remapped to message + var parsed map[string]interface{} + lines := strings.Split(strings.TrimSpace(result), "\n") + assert.GreaterOrEqual(t, len(lines), 2, "elasticsearch should produce index header + body") + err := json.Unmarshal([]byte(lines[1]), &parsed) + assert.NoError(t, err) + assert.Equal(t, "es event", parsed["message"]) +} + +func TestWriteDevnull(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "devnull event data"}, + } + item := makeOutQueueItem("devnullsample", "raw", "devnull", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // devnull should not write any content through the pipe + assert.Empty(t, result) + + // But bytes should still be accounted for + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + bw := BytesWritten["devnullsample"] + ew := EventsWritten["devnullsample"] + Mutex.RUnlock() + assert.Greater(t, bw, int64(0), "bytes should be counted even for devnull") + assert.Equal(t, int64(1), ew) +} + +func TestWriteCacheMiss(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "cache miss event"}, + } + item := makeOutQueueItem("cachemiss", "raw", "stdout", events) + item.Cache.UseCache = true // UseCache=true but no cacheBuf exists => cache miss + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Cache miss should still write through the normal pipe + assert.Contains(t, result, "cache miss event") +} + +func TestWriteSetCache(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Clean the cache bufs + cacheMutex.Lock() + delete(cacheBufs, "setcache") + cacheMutex.Unlock() + + events := []map[string]string{ + {"_raw": "cached event"}, + } + item := makeOutQueueItem("setcache", "raw", "stdout", events) + item.Cache.SetCache = true + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // SetCache should write to both cache buffer and the pipe + assert.Contains(t, result, "cached event") + + // Verify cache buffer was populated + cacheMutex.RLock() + cb, ok := cacheBufs["setcache"] + cacheMutex.RUnlock() + assert.True(t, ok, "cache buffer should be created") + assert.Contains(t, cb.String(), "cached event") +} + +func TestWriteUseCache(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Pre-populate the cache + cacheMutex.Lock() + cacheBufs["usecache"] = &bytes.Buffer{} + cacheBufs["usecache"].WriteString("previously cached data\n") + cacheMutex.Unlock() + + events := []map[string]string{ + {"_raw": "new event"}, + } + item := makeOutQueueItem("usecache", "raw", "stdout", events) + item.Cache.UseCache = true + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Should use the cached data, not the new events + assert.Contains(t, result, "previously cached data") + + // Clean up + cacheMutex.Lock() + delete(cacheBufs, "usecache") + cacheMutex.Unlock() +} + +func TestWriteNonExistentTemplate(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "should not appear"}, + } + item := makeOutQueueItem("badtemplate", "nonexistent_template_xyz", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Non-existent template should produce no output + assert.Empty(t, result) +} + +func TestWriteMultipleEvents(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "event1"}, + {"_raw": "event2"}, + {"_raw": "event3"}, + } + item := makeOutQueueItem("multisample", "raw", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "event1") + assert.Contains(t, result, "event2") + assert.Contains(t, result, "event3") + + // Verify accounting + time.Sleep(50 * time.Millisecond) + Mutex.RLock() + ew := EventsWritten["multisample"] + Mutex.RUnlock() + assert.Equal(t, int64(3), ew) +} + +func TestWriteKafkaNoNewlines(t *testing.T) { + cleanup := initROT() + defer cleanup() + + events := []map[string]string{ + {"_raw": "kafka event"}, + } + item := makeOutQueueItem("kafkasample", "raw", "kafka", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + // Kafka should not append newlines + assert.Equal(t, "kafka event", result) +} + +func TestWriteCustomTemplate(t *testing.T) { + cleanup := initROT() + defer cleanup() + + // Register a custom template + _ = template.New("customtest_header", "HEADER\n") + _ = template.New("customtest_row", "ROW:{{._raw}}\n") + _ = template.New("customtest_footer", "FOOTER\n") + + events := []map[string]string{ + {"_raw": "custom line"}, + } + item := makeOutQueueItem("customsample", "customtest", "stdout", events) + + var result string + done := make(chan struct{}) + go func() { + result = readFromPipe(item) + close(done) + }() + + write(item) + <-done + + assert.Contains(t, result, "HEADER") + assert.Contains(t, result, "ROW:custom line") + assert.Contains(t, result, "FOOTER") +} diff --git a/rater/rater_test.go b/rater/rater_test.go index 470d3db..047aef5 100644 --- a/rater/rater_test.go +++ b/rater/rater_test.go @@ -1,10 +1,13 @@ package rater import ( + "os" + "path/filepath" "testing" "time" config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" "github.com/stretchr/testify/assert" ) @@ -14,3 +17,144 @@ func TestRandomizeCount(t *testing.T) { count := EventRate(s, time.Now(), 10) assert.Equal(t, 11, count) } + +func TestTokenRateDefault(t *testing.T) { + dr := &DefaultRater{} + token := config.Token{Name: "test"} + rate := dr.TokenRate(token, time.Now()) + assert.Equal(t, 1.0, rate) +} + +func TestTokenRateConfig(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "configrater.yml")) + + c := config.NewConfig() + r := c.FindRater("testconfigrater") + + cr := &ConfigRater{c: r} + token := config.Token{Name: "test"} + + loc, _ := time.LoadLocation("Local") + n := time.Date(2001, 10, 20, 0, 0, 0, 100000, loc) + rate := cr.TokenRate(token, n) + assert.Equal(t, 2.0, rate) +} + +func TestTokenRateKBps(t *testing.T) { + kr := &KBpsRater{c: &config.RaterConfig{Name: "kbps"}} + token := config.Token{Name: "test"} + rate := kr.TokenRate(token, time.Now()) + assert.Equal(t, 1.0, rate) +} + +func TestTokenRateScript(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "luarater.yml")) + + c := config.NewConfig() + r := c.FindRater("multiply") + + sr := &ScriptRater{c: r} + token := config.Token{Name: "test"} + rate := sr.TokenRate(token, time.Now()) + assert.Equal(t, 2.0, rate) +} + +func TestGetRaterFallback(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + os.Setenv("GOGEN_FULLCONFIG", filepath.Join("..", "tests", "rater", "defaultrater.yml")) + + r := GetRater("nonexistentrater") + assert.IsType(t, &DefaultRater{}, r, "unknown rater name should fall back to DefaultRater") +} + +func TestEventRateNegativeResult(t *testing.T) { + // A rater returning a negative rate should produce a negative or zero count + s := &config.Sample{RandomizeCount: 0} + // Use a mock rater by pre-setting it + s.Rater = &negativeRater{} + randSource = 2 + count := EventRate(s, time.Now(), 10) + assert.True(t, count <= 0, "negative rate should produce non-positive count, got %d", count) +} + +func TestKBpsEventRateMissingOption(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{}, + }, + } + s := &config.Sample{Name: "test"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateWrongType(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": "not_a_float", + }, + }, + } + s := &config.Sample{Name: "test"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateMissingSample(t *testing.T) { + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": 100.0, + }, + }, + } + s := &config.Sample{Name: "nonexistent_sample_kbps"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) +} + +func TestKBpsEventRateWithData(t *testing.T) { + // Pre-populate outputter stats + outputter.Mutex.Lock() + outputter.BytesWritten["kbps_test_sample"] = 10000 + outputter.EventsWritten["kbps_test_sample"] = 100 + outputter.Mutex.Unlock() + defer func() { + outputter.Mutex.Lock() + delete(outputter.BytesWritten, "kbps_test_sample") + delete(outputter.EventsWritten, "kbps_test_sample") + outputter.Mutex.Unlock() + }() + + kr := &KBpsRater{ + c: &config.RaterConfig{ + Name: "kbps", + Options: map[string]interface{}{ + "KBps": 100.0, + }, + }, + t: time.Now().Add(-1 * time.Second), // pretend we started 1s ago + } + s := &config.Sample{Name: "kbps_test_sample"} + rate := kr.EventRate(s, time.Now(), 10) + assert.Equal(t, 1.0, rate) // always returns 1.0 regardless +} + +// negativeRater always returns a negative rate for testing +type negativeRater struct{} + +func (nr *negativeRater) EventRate(s *config.Sample, now time.Time, count int) float64 { + return -1.0 +} +func (nr *negativeRater) TokenRate(t config.Token, now time.Time) float64 { + return -1.0 +} diff --git a/run/run_test.go b/run/run_test.go new file mode 100644 index 0000000..17b8bc1 --- /dev/null +++ b/run/run_test.go @@ -0,0 +1,200 @@ +package run + +import ( + "bytes" + "testing" + "time" + + config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" + "github.com/stretchr/testify/assert" +) + +// resetRunState resets config and outputter stats for a clean test. +func resetRunState() { + config.ResetConfig() + outputter.Mutex.Lock() + outputter.BytesWritten = make(map[string]int64) + outputter.EventsWritten = make(map[string]int64) + outputter.Mutex.Unlock() +} + +func TestRunCompletesWithEndIntervals(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: devnull + outputTemplate: raw + rotInterval: 1 +samples: + - name: runtest + description: "Run completion test" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: run test event +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.NotEmpty(t, c.Samples) + + done := make(chan struct{}) + go func() { + Run(c) + close(done) + }() + + select { + case <-done: + // Run completed successfully + case <-time.After(10 * time.Second): + t.Fatal("Run() did not complete within timeout") + } +} + +func TestRunMultipleSamples(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: devnull + outputTemplate: raw + rotInterval: 1 +samples: + - name: multi1 + description: "Multi sample 1" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: event from multi1 + - name: multi2 + description: "Multi sample 2" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: event from multi2 +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.Len(t, c.Samples, 2) + + done := make(chan struct{}) + go func() { + Run(c) + close(done) + }() + + select { + case <-done: + outputter.Mutex.RLock() + totalEvents := int64(0) + for _, v := range outputter.EventsWritten { + totalEvents += v + } + outputter.Mutex.RUnlock() + assert.Greater(t, totalEvents, int64(0), "should have generated events") + case <-time.After(10 * time.Second): + t.Fatal("Run() did not complete within timeout") + } +} + +func TestOnceMethod(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: buf + outputTemplate: json + rotInterval: 1 +samples: + - name: oncemethodtest + description: "Once method test" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: once method event +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + r := Runner{} + + done := make(chan struct{}) + go func() { + r.Once("oncemethodtest") + close(done) + }() + + select { + case <-done: + // Once completed without error + case <-time.After(10 * time.Second): + t.Fatal("Once() did not complete within timeout") + } +} + +func TestOncePublic(t *testing.T) { + resetRunState() + + configStr := ` +global: + utc: true + output: + outputter: buf + outputTemplate: json + rotInterval: 1 +samples: + - name: oncetest + description: "Once test sample" + interval: 1 + count: 1 + endIntervals: 1 + lines: + - _raw: once event data +` + config.SetupFromString(configStr) + defer config.CleanupConfigAndEnvironment() + + c := config.NewConfig() + assert.NotEmpty(t, c.Samples) + + // Set up a buffer for the sample + var buf bytes.Buffer + s := c.FindSampleByName("oncetest") + s.Buf = &buf + + r := Runner{} + + // Start ROT before onceWithConfig (Once() normally does this) + go outputter.ROT(c) + // Give ROT goroutine time to create the new rotchan + time.Sleep(50 * time.Millisecond) + + done := make(chan struct{}) + go func() { + r.onceWithConfig("oncetest", c) + close(done) + }() + + select { + case <-done: + assert.Contains(t, buf.String(), "once event data") + case <-time.After(10 * time.Second): + t.Fatal("Once() did not complete within timeout") + } +} diff --git a/run/runonce_test.go b/run/runonce_test.go index e40ec2d..2b83294 100644 --- a/run/runonce_test.go +++ b/run/runonce_test.go @@ -7,12 +7,12 @@ import ( "time" config "github.com/coccyx/gogen/internal" + "github.com/coccyx/gogen/outputter" "github.com/stretchr/testify/assert" ) func TestOnceWithConfig(t *testing.T) { - // Clean up any existing config - config.ResetConfig() + resetRunState() // Setup test configuration configStr := ` @@ -58,6 +58,9 @@ samples: config.SetupFromString(configStr) c := config.NewConfig() + // Start ROT in case a previous test closed rotchan + go outputter.ROT(c) + // Record time before and after test to validate timestamp is within range beforeTest := time.Now().Truncate(time.Second) if c.Global.UTC { diff --git a/tests/generator/luaapi2.yml b/tests/generator/luaapi2.yml new file mode 100644 index 0000000..2edf1dc --- /dev/null +++ b/tests/generator/luaapi2.yml @@ -0,0 +1,43 @@ +generators: + - name: roundTest + script: | + val = round(3.14159, 2) + setToken("rounded", tostring(val)) + - name: logInfoTest + script: | + info("test log message from lua") + setToken("logged", "ok") + - name: removeTokenTest + script: | + setToken("keeper", "keep") + setToken("remover", "remove") + removeToken("remover") + - name: sendEventTest + script: | + event = { _raw = "sent via sendEvent" } + sendEvent(event) +samples: + - name: roundTest + generator: roundTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: logInfoTest + generator: logInfoTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: removeTokenTest + generator: removeTokenTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused + - name: sendEventTest + generator: sendEventTest + interval: 1 + endIntervals: 1 + lines: + - _raw: notused diff --git a/timer/timer_test.go b/timer/timer_test.go index b93ac0c..139e98f 100644 --- a/timer/timer_test.go +++ b/timer/timer_test.go @@ -199,6 +199,40 @@ Loop: } } +func TestBackfillReplay(t *testing.T) { + os.Setenv("GOGEN_HOME", "..") + os.Setenv("GOGEN_ALWAYS_REFRESH", "1") + home := filepath.Join("..", "tests", "timer") + os.Setenv("GOGEN_SAMPLES_DIR", home) + + s := tests.FindSampleInFile(home, "realtimereplay") + // Force non-realtime mode to exercise backfill with replay generator + s.Realtime = false + // Set end slightly in the future so backfill runs a few iterations + s.EndParsed = s.Current.Add(5 * time.Second) + + gq := make(chan *config.GenQueueItem, 1000) + oq := make(chan *config.OutQueueItem) + done := make(chan int) + gqs := make([]*config.GenQueueItem, 0, 10) + + timer := &Timer{S: s, GQ: gq, OQ: oq, Done: done} + go timer.NewTimer(0) + <-done + +Loop: + for { + select { + case i := <-gq: + gqs = append(gqs, i) + default: + break Loop + } + } + // Should have generated events using replay offsets via inc() replay path + assert.Greater(t, len(gqs), 0, "should have generated replay events during backfill") +} + func TestTimerClose(t *testing.T) { os.Setenv("GOGEN_HOME", "..") os.Setenv("GOGEN_ALWAYS_REFRESH", "1")