Improve test coverage from 52% to 75% with simplified test helpers

CLAUDE.md

@@ -0,0 +1,102 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Gogen is an open source data generator for generating demo and test data, especially time series log and metric data. It's a Go CLI tool with an embedded Lua scripting engine, a Python AWS Lambda API backend, and a React/TypeScript UI.
+
+## Common Commands
+
+### Go (core CLI)
+
+```bash
+make install          # Build and install to $GOPATH/bin (default target)
+make build            # Cross-compile for linux, darwin, windows, wasm
+make test             # Run all Go tests: go test -v ./...
+go test -v ./internal # Run tests for a single package
+go test -v -run TestName ./internal  # Run a single test
+```
+
+Version, git summary, build date, and GitHub OAuth credentials are injected via `-ldflags` in the Makefile. Always use `make install` rather than bare `go install`.
+
+Dependencies are vendored in `vendor/`. After adding deps, run `go mod vendor`.
+
+### Python API (`gogen-api/`)
+
+```bash
+cd gogen-api
+./start_dev.sh        # Starts DynamoDB Local + MinIO via docker-compose, then SAM local API on port 4000
+./setup_local_db.sh   # Seeds local DynamoDB schema
+sam build && sam local start-api --port 4000 --docker-network lambda-local
+./deploy_lambdas.sh   # Deploy to AWS (requires credentials)
+```
+
+### UI (`ui/`)
+
+```bash
+cd ui
+npm run dev           # Vite dev server (copies wasm from build/wasm/ first)
+npm run build         # Production build
+npm test              # Jest tests
+```
+
+## Architecture
+
+### Go Package Layout
+
+All packages are at the top level (no `cmd/` or `pkg/` convention):
+
+- **`main.go`** — CLI entry point using `urfave/cli.v1`. Maps CLI flags to `GOGEN_*` env vars.
+- **`internal/`** — Core package. Config singleton, `Sample` struct, `Token` processing, API client, sharing. Imported as `config` throughout (`config "github.com/coccyx/gogen/internal"`).
+- **`generator/`** — Reads `GenQueueItem` from channel, dispatches to sample-based or Lua generators.
+- **`outputter/`** — Reads `OutQueueItem` from channel, dispatches to output destinations (stdout, file, HTTP, Kafka, network, devnull, buf).
+- **`run/`** — Orchestrates the pipeline: timers -> generator worker pool -> outputter worker pool.
+- **`timer/`** — One timer goroutine per Sample; handles backfill and realtime intervals.
+- **`rater/`** — Controls event rate (config-based, time-of-day/weekday, kbps, Lua script).
+- **`template/`** — Output formatting (raw, JSON, CSV, splunkhec, syslog, elasticsearch).
+- **`logger/`** — Thin logrus wrapper with file/func/line context hook.
+
+### Data Flow
+
+```
+YAML/JSON Config -> internal.Config singleton (sync.Once)
+    -> [Timer goroutine per Sample]
+    -> GenQueueItem channel -> [Generator worker pool]
+    -> OutQueueItem channel -> [Outputter worker pool]
+    -> output destination
+```
+
+Concurrency is channel + goroutine worker pools. Worker counts set by `GeneratorWorkers` and `OutputWorkers` config fields.
+
+### Key Interfaces
+
+- `internal.Generator` — `Gen(item *GenQueueItem) error`
+- `internal.Outputter` — `Send(events []map[string]string, sample *Sample, outputTemplate string) error`
+- `internal.Rater` — `EventsPerInterval(s *Sample) int`
+
+### Config System
+
+Config is a **singleton** via `sync.Once`. Controlled by environment variables:
+- `GOGEN_HOME`, `GOGEN_FULLCONFIG`, `GOGEN_CONFIG_DIR`, `GOGEN_SAMPLES_DIR`
+- Remote configs fetched from `https://api.gogen.io` (override with `GOGEN_APIURL`)
+
+In tests, call `config.ResetConfig()` before `config.NewConfig()` to get a fresh instance. Tests commonly use `config.SetupFromString(yamlStr)` to inject inline YAML config.
+
+### gogen-api (Python Lambda)
+
+Each Lambda function is a separate `.py` file in `gogen-api/`. Backed by DynamoDB + S3. Originally Python 2.7, being updated to Python 3. AWS SAM template at `gogen-api/template.yaml`.
+
+### UI (React/TypeScript)
+
+Vite + React 18 + TypeScript + Tailwind CSS. Components in `src/components/`, pages in `src/pages/`, API clients in `src/api/`, types in `src/types/`. Tests use Jest + React Testing Library, placed adjacent to source as `.test.tsx`.
+
+## CI/CD
+
+GitHub Actions (`.github/workflows/ci.yml`):
+- Push to `master`/`dev` or any PR: runs `make test`, then on `master`/`dev` cross-compiles, builds Docker, pushes artifacts to S3, deploys UI and Lambdas.
+- Tag pushes (`v*.*.*`): full release workflow via `release.yml` — builds, creates GitHub release, pushes Docker images, deploys to production.
+
+## Lua Scripting
+
+Generators (`generator/lua.go`) and raters (`rater/script.go`) support embedded Lua via `gopher-lua` + `gopher-luar`. Lua state persists across calls within a run.

generator/generator_test.go

@@ -12,23 +12,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGenerator(t *testing.T) {
-	// Setup environment
+// setupGenTest resets config, sets env vars, and returns common test fixtures.
+func setupGenTest(t *testing.T, samplesDir string, seed int64) (func() time.Time, *rand.Rand) {
+	t.Helper()
+	config.ResetConfig()
 	os.Setenv("GOGEN_HOME", "..")
 	os.Setenv("GOGEN_ALWAYS_REFRESH", "1")
 	os.Setenv("GOGEN_FULLCONFIG", "")
-	home := filepath.Join("..", "tests", "tokens")
-	os.Setenv("GOGEN_SAMPLES_DIR", home)
+	os.Setenv("GOGEN_SAMPLES_DIR", samplesDir)
 	loc, _ := time.LoadLocation("Local")
-	source := rand.NewSource(0)
-	randgen := rand.New(source)
-
+	randgen := rand.New(rand.NewSource(seed))
 	n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc)
-	now := func() time.Time {
-		return n
-	}
+	now := func() time.Time { return n }
+	return now, randgen
+}
+
+func TestGenerator(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 0)
 
-	// gq := make(chan *config.GenQueueItem)
 	oq := make(chan *config.OutQueueItem)
 	s := tests.FindSampleInFile(home, "token-static")
 	if s == nil {
@@ -44,23 +46,171 @@ func TestGenerator(t *testing.T) {
 	assert.Equal(t, "foo", oqi.Events[0]["_raw"])
 }
 
-func TestGeneratorCache(t *testing.T) {
-	// Setup environment
+func TestGeneratorMultiPass(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 0)
+
+	oq := make(chan *config.OutQueueItem)
+	s := tests.FindSampleInFile(home, "tokens")
+	if s == nil {
+		t.Fatalf("Sample tokens not found")
+	}
+	// Force MultiPass
+	s.SinglePass = false
+
+	// Count > lines: tests the iters > 1 path in genMultiPass
+	gqi := &config.GenQueueItem{Count: len(s.Lines) + 2, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+	go func() {
+		err := genMultiPass(gqi)
+		assert.NoError(t, err)
+	}()
+
+	oqi := <-oq
+	assert.Equal(t, len(s.Lines)+2, len(oqi.Events))
+}
+
+func TestGeneratorMultiPassRandomize(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 42)
+
+	oq := make(chan *config.OutQueueItem)
+	s := tests.FindSampleInFile(home, "tokens")
+	if s == nil {
+		t.Fatalf("Sample tokens not found")
+	}
+	s.SinglePass = false
+	s.RandomizeEvents = true
+
+	gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+	go func() {
+		genMultiPass(gqi)
+	}()
+
+	oqi := <-oq
+	assert.Equal(t, 5, len(oqi.Events))
+}
+
+func TestGeneratorSinglePassCountGtLines(t *testing.T) {
+	home := filepath.Join("..", "tests", "singlepass")
+	now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 0)
+
+	c := config.NewConfig()
+	s := c.FindSampleByName("test1")
+	if s == nil {
+		t.Fatalf("Sample test1 not found")
+	}
+	assert.True(t, s.SinglePass)
+
+	oq := make(chan *config.OutQueueItem)
+	// Count > lines: tests the iters > 1 singlepass path
+	gqi := &config.GenQueueItem{Count: len(s.Lines) + 3, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+	go func() {
+		genSinglePass(gqi)
+	}()
+
+	oqi := <-oq
+	assert.Equal(t, len(s.Lines)+3, len(oqi.Events))
+}
+
+func TestGeneratorSinglePassRandomize(t *testing.T) {
+	home := filepath.Join("..", "tests", "singlepass")
+	now, randgen := setupGenTest(t, filepath.Join(home, "test1.yml"), 42)
+
+	c := config.NewConfig()
+	s := c.FindSampleByName("test1")
+	if s == nil {
+		t.Fatalf("Sample test1 not found")
+	}
+	assert.True(t, s.SinglePass)
+	s.RandomizeEvents = true
+
+	oq := make(chan *config.OutQueueItem)
+	gqi := &config.GenQueueItem{Count: 5, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+	go func() {
+		genSinglePass(gqi)
+	}()
+
+	oqi := <-oq
+	assert.Equal(t, 5, len(oqi.Events))
+}
+
+func TestGeneratorStartWorker(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 0)
+
+	oq := make(chan *config.OutQueueItem)
+	s := tests.FindSampleInFile(home, "token-static")
+	if s == nil {
+		t.Fatalf("Sample token-static not found")
+	}
+
+	gq := make(chan *config.GenQueueItem)
+	gqs := make(chan int)
+	go Start(gq, gqs)
+
+	// Send multiple items to test the "generator already set" path
+	for i := 0; i < 3; i++ {
+		gqi := &config.GenQueueItem{Count: 1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+		gq <- gqi
+		oqi := <-oq
+		assert.Equal(t, "foo", oqi.Events[0]["_raw"])
+	}
+
+	close(gq)
+	select {
+	case <-gqs:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Generator worker did not finish in time")
+	}
+}
+
+func TestGeneratorCountMinusOne(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 0)
+
+	oq := make(chan *config.OutQueueItem)
+	s := tests.FindSampleInFile(home, "tokens")
+	if s == nil {
+		t.Fatalf("Sample tokens not found")
+	}
+	// Count=-1 means "use all lines"
+	gqi := &config.GenQueueItem{Count: -1, Earliest: now(), Latest: now(), Now: now(), S: s, OQ: oq, Rand: randgen, Cache: &config.CacheItem{}}
+	go func() {
+		sg := sample{}
+		sg.Gen(gqi)
+	}()
+
+	oqi := <-oq
+	assert.Equal(t, len(s.Lines), len(oqi.Events))
+}
+
+func TestPrimeRaterSetsRater(t *testing.T) {
 	os.Setenv("GOGEN_HOME", "..")
 	os.Setenv("GOGEN_ALWAYS_REFRESH", "1")
-	os.Setenv("GOGEN_FULLCONFIG", "")
-	home := filepath.Join("..", "tests", "tokens")
-	os.Setenv("GOGEN_SAMPLES_DIR", home)
-	loc, _ := time.LoadLocation("Local")
-	source := rand.NewSource(0)
-	randgen := rand.New(source)
 
-	n := time.Date(2001, 10, 20, 12, 0, 0, 100000, loc)
-	now := func() time.Time {
-		return n
+	s := &config.Sample{
+		Name: "primerater_test",
+		Tokens: []config.Token{
+			{
+				Name:        "ratedtoken",
+				Type:        "rated",
+				RaterString: "default",
+			},
+			{
+				Name: "normaltoken",
+				Type: "choice",
+			},
+		},
 	}
 
-	// gq := make(chan *config.GenQueueItem)
+	PrimeRater(s)
+	assert.NotNil(t, s.Tokens[0].Rater, "rated token should have rater set")
+}
+
+func TestGeneratorCache(t *testing.T) {
+	home := filepath.Join("..", "tests", "tokens")
+	now, randgen := setupGenTest(t, home, 0)
+
 	oq := make(chan *config.OutQueueItem)
 	s := tests.FindSampleInFile(home, "token-static")
 	if s == nil {