Skip to content

Commit

Permalink
slow section added in the test block to build smaller dataset
Browse files Browse the repository at this point in the history 
Signed-off-by: bazooka3000 <[email protected]>
  • Loading branch information
@bazooka3000
bazooka3000 committed Nov 19, 2023
1 parent fcf8381 commit f4de032
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 39 deletions.
15 changes: 13 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,30 @@ benchmarks:
.PHONY: bench-old
bench-old: benchmarks
@echo "Benchmarking old engine"
@go test ./... -bench 'BenchmarkRangeQuery/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out
@go test ./... -bench 'BenchmarkRangeQuery/slow/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out
@go test ./... -bench 'BenchmarkNativeHistograms/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' >> benchmarks/old.out

.PHONY: bench-new
bench-new: benchmarks
@echo "Benchmarking new engine"
@go test ./... -bench 'BenchmarkRangeQuery/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out
@go test ./... -bench 'BenchmarkRangeQuery/slow/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out
@go test ./... -bench 'BenchmarkNativeHistograms/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' >> benchmarks/new.out

.PHONY: benchmark
benchmark: bench-old bench-new
@benchstat benchmarks/old.out benchmarks/new.out

.PHONY : bench-fast
bench-fast : benchmarks
@mkdir -p benchmarks
@echo "Benchmarking old engine"
@go test ./engine -bench 'BenchmarkRangeQuery/fast/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out
@go test ./engine -bench 'BenchmarkNativeHistograms/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' >> benchmarks/old.out
@echo "Benchmarking new engine"
@go test ./engine -bench 'BenchmarkRangeQuery/fast/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out
@go test ./engine -bench 'BenchmarkNativeHistograms/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' >> benchmarks/new.out
@benchstat benchmarks/old.out benchmarks/new.out

.PHONY: sync-parser
sync-parser:
@echo "Cleaning existing directories"
Expand Down
124 changes: 87 additions & 37 deletions engine/bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,11 @@ func BenchmarkSingleQuery(b *testing.B) {

func BenchmarkRangeQuery(b *testing.B) {
samplesPerHour := 60 * 2
sixHourDataset := setupStorage(b, 1000, 3, 6*samplesPerHour)
defer sixHourDataset.Close()

largeSixHourDataset := setupStorage(b, 10000, 10, 6*samplesPerHour)
defer largeSixHourDataset.Close()

sevenDaysAndTwoHoursDataset := setupStorage(b, 1000, 3, (7*24+2)*samplesPerHour)
defer sevenDaysAndTwoHoursDataset.Close()
var (
sixHourDataset *teststorage.TestStorage
largeSixHourDataset *teststorage.TestStorage
sevenDaysAndTwoHoursDataset *teststorage.TestStorage
)

start := time.Unix(0, 0)
end := start.Add(2 * time.Hour)
Expand Down Expand Up @@ -273,42 +270,95 @@ func BenchmarkRangeQuery(b *testing.B) {
storage: sixHourDataset,
},
}
b.Run("fast", func(b *testing.B) {
sixHourDataset = setupStorage(b, 1000, 3, 6*samplesPerHour)
defer sixHourDataset.Close()

for _, tc := range cases {
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
b.Run("old_engine", func(b *testing.B) {
opts := promql.EngineOpts{
Logger: nil,
Reg: nil,
MaxSamples: 50000000,
Timeout: 100 * time.Second,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)

b.ResetTimer()
for _, tc := range cases {
if tc.storage != sixHourDataset {
b.Skip()
}
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
qry, err := engine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step)
testutil.Ok(b, err)
b.Run("old_engine", func(b *testing.B) {
opts := promql.EngineOpts{
Logger: nil,
Reg: nil,
MaxSamples: 50000000,
Timeout: 100 * time.Second,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)

oldResult := qry.Exec(context.Background())
testutil.Ok(b, oldResult.Err)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
qry, err := engine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step)
testutil.Ok(b, err)

oldResult := qry.Exec(context.Background())
testutil.Ok(b, oldResult.Err)
}
})
b.Run("new_engine", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step)
testutil.Ok(b, newResult.Err)
}
})
})
b.Run("new_engine", func(b *testing.B) {
b.ResetTimer()
}
})
b.Run("slow", func(b *testing.B) {
sixHourDataset = setupStorage(b, 1000, 3, 6*samplesPerHour)
defer sixHourDataset.Close()

largeSixHourDataset = setupStorage(b, 10000, 10, 6*samplesPerHour)
defer sixHourDataset.Close()

sevenDaysAndTwoHoursDataset = setupStorage(b, 1000, 3, (7*24+2)*samplesPerHour)
defer sixHourDataset.Close()

for _, tc := range cases {
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
b.Run("old_engine", func(b *testing.B) {
opts := promql.EngineOpts{
Logger: nil,
Reg: nil,
MaxSamples: 50000000,
Timeout: 100 * time.Second,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)

for i := 0; i < b.N; i++ {
newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step)
testutil.Ok(b, newResult.Err)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
qry, err := engine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step)
testutil.Ok(b, err)

oldResult := qry.Exec(context.Background())
testutil.Ok(b, oldResult.Err)
}
})
b.Run("new_engine", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step)
testutil.Ok(b, newResult.Err)
}
})
})
})
}
}
})
}

func BenchmarkNativeHistograms(b *testing.B) {
Expand Down

0 comments on commit f4de032

Please sign in to comment.