Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix work_mem recommendation to not be too low #39

Merged
merged 1 commit into from
Mar 5, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions pkg/pgtune/memory.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
maintenanceWorkMemLimit = 2047 * parse.Megabyte
sharedBuffersWindows = 512 * parse.Megabyte
baseConns = 20
workMemMin = 64 * parse.Kilobyte
workMemPerGigPerConn = 6.4 * baseConns // derived from pgtune results
workMemPerGigPerConnWindows = 8.53336 * baseConns // derived from pgtune results
)
Expand Down Expand Up @@ -80,9 +81,13 @@ func (r *MemoryRecommender) Recommend(key string) string {
} else {
cpuFactor := math.Round(float64(r.cpus) / 2.0)
gigs := float64(r.totalMemory) / float64(parse.Gigabyte)
temp := gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor
val = parse.BytesToPGFormat(uint64(temp))
temp := uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor)
if temp < workMemMin {
temp = workMemMin
}
val = parse.BytesToPGFormat(temp)
}

} else {
panic(fmt.Sprintf("unknown key: %s", key))
}
Expand All @@ -91,16 +96,20 @@ func (r *MemoryRecommender) Recommend(key string) string {

func (r *MemoryRecommender) recommendWindows() string {
cpuFactor := math.Round(float64(r.cpus) / 2.0)
var temp uint64

if r.totalMemory <= 2*parse.Gigabyte {
gigs := float64(r.totalMemory) / float64(parse.Gigabyte)
temp := gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor
return parse.BytesToPGFormat(uint64(temp))
temp = uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor)
} else {
base := 2.0 * workMemPerGigPerConn * float64(parse.Megabyte)
gigs := float64(r.totalMemory)/float64(parse.Gigabyte) - 2.0
temp = uint64(((gigs*(workMemPerGigPerConnWindows*float64(parse.Megabyte)) + base) / float64(r.conns)) / cpuFactor)
}
if temp < workMemMin {
temp = workMemMin
}
base := 2.0 * workMemPerGigPerConn * float64(parse.Megabyte)
gigs := float64(r.totalMemory)/float64(parse.Gigabyte) - 2.0
temp := ((gigs*(workMemPerGigPerConnWindows*float64(parse.Megabyte)) + base) / float64(r.conns)) / cpuFactor
return parse.BytesToPGFormat(uint64(temp))
return parse.BytesToPGFormat(temp)
}

// MemorySettingsGroup is the SettingsGroup to represent settings that affect memory usage.
Expand Down
53 changes: 34 additions & 19 deletions pkg/pgtune/memory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,19 @@ var memoryToBaseVals = map[uint64]map[string]uint64{
},
}

// cpuVals is the different amounts of CPUs to test
var cpuVals = []int{1, 4, 5}

// connVals is the different number of conns to test
var connVals = []uint64{0, 19, 20, 50}

// memorySettingsMatrix stores the test cases for MemoryRecommend along with
// the expected values
var memorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{}
// highCPUs is the number of CPUs that is high enough that work_mem would normally
// fall below the minimum (64KB) using the standard formula
const highCPUs = 9000

var (
// cpuVals is the different amounts of CPUs to test
cpuVals = []int{1, 4, 5, highCPUs}
// connVals is the different number of conns to test
connVals = []uint64{0, 19, 20, 50}
// memorySettingsMatrix stores the test cases for MemoryRecommend along with
// the expected values
memorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{}
)

func init() {
for mem, baseMatrix := range memoryToBaseVals {
Expand All @@ -57,17 +61,21 @@ func init() {
memorySettingsMatrix[mem][cpus][conns][EffectiveCacheKey] = parse.BytesToPGFormat(baseMatrix[EffectiveCacheKey])
memorySettingsMatrix[mem][cpus][conns][MaintenanceWorkMemKey] = parse.BytesToPGFormat(baseMatrix[MaintenanceWorkMemKey])

// CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same)
cpuFactor := math.Round(float64(cpus) / 2.0)
// Our work_mem values are derivied by a certain amount of memory lost/gained when
// moving away from baseConns
connFactor := float64(MaxConnectionsDefault) / float64(baseConns)
if conns != 0 {
connFactor = float64(conns) / float64(baseConns)
if cpus == highCPUs {
memorySettingsMatrix[mem][cpus][conns][WorkMemKey] = parse.BytesToPGFormat(workMemMin)
} else {
// CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same)
cpuFactor := math.Round(float64(cpus) / 2.0)
// Our work_mem values are derivied by a certain amount of memory lost/gained when
// moving away from baseConns
connFactor := float64(MaxConnectionsDefault) / float64(baseConns)
if conns != 0 {
connFactor = float64(conns) / float64(baseConns)
}

memorySettingsMatrix[mem][cpus][conns][WorkMemKey] =
parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor))
}

memorySettingsMatrix[mem][cpus][conns][WorkMemKey] =
parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor))
}
}
}
Expand Down Expand Up @@ -179,6 +187,13 @@ func TestMemoryRecommenderRecommendWindows(t *testing.T) {
conns: baseConns,
want: "27088" + parse.KB, // from pgtune
},
{
desc: "1GB, 9000 cpus",
totalMemory: parse.Gigabyte,
cpus: highCPUs,
conns: baseConns,
want: "64" + parse.KB,
},
}

for _, c := range cases {
Expand Down
3 changes: 0 additions & 3 deletions pkg/tstune/tuner.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
Expand Down Expand Up @@ -77,8 +76,6 @@ var (
getPGConfigVersionFn = getPGConfigVersion
filepathAbsFn = filepath.Abs

pgVersionRegex = regexp.MustCompile("^PostgreSQL ([0-9]+?).([0-9]+?).*")

// ValidPGVersions is a slice representing the major versions of PostgreSQL
// for which recommendations can be generated.
ValidPGVersions = []string{pgMajor11, pgMajor10, pgMajor96}
Expand Down