Skip to content

Commit

Permalink
Fix work_mem recommendation to not be too low
Browse files Browse the repository at this point in the history
Given the inputs, the work_mem formulas could potentially return
values below 64KB, which is invalid and blocks PostgreSQL from
starting. These changes ensure that work_mem is always at least
64KB.

Also remove an unused regex in pkg/tstune
  • Loading branch information
RobAtticus committed Mar 5, 2019
1 parent 257004a commit baaec8c
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 30 deletions.
25 changes: 17 additions & 8 deletions pkg/pgtune/memory.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
maintenanceWorkMemLimit = 2047 * parse.Megabyte
sharedBuffersWindows = 512 * parse.Megabyte
baseConns = 20
workMemMin = 64 * parse.Kilobyte
workMemPerGigPerConn = 6.4 * baseConns // derived from pgtune results
workMemPerGigPerConnWindows = 8.53336 * baseConns // derived from pgtune results
)
Expand Down Expand Up @@ -80,9 +81,13 @@ func (r *MemoryRecommender) Recommend(key string) string {
} else {
cpuFactor := math.Round(float64(r.cpus) / 2.0)
gigs := float64(r.totalMemory) / float64(parse.Gigabyte)
temp := gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor
val = parse.BytesToPGFormat(uint64(temp))
temp := uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor)
if temp < workMemMin {
temp = workMemMin
}
val = parse.BytesToPGFormat(temp)
}

} else {
panic(fmt.Sprintf("unknown key: %s", key))
}
Expand All @@ -91,16 +96,20 @@ func (r *MemoryRecommender) Recommend(key string) string {

func (r *MemoryRecommender) recommendWindows() string {
cpuFactor := math.Round(float64(r.cpus) / 2.0)
var temp uint64

if r.totalMemory <= 2*parse.Gigabyte {
gigs := float64(r.totalMemory) / float64(parse.Gigabyte)
temp := gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor
return parse.BytesToPGFormat(uint64(temp))
temp = uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor)
} else {
base := 2.0 * workMemPerGigPerConn * float64(parse.Megabyte)
gigs := float64(r.totalMemory)/float64(parse.Gigabyte) - 2.0
temp = uint64(((gigs*(workMemPerGigPerConnWindows*float64(parse.Megabyte)) + base) / float64(r.conns)) / cpuFactor)
}
if temp < workMemMin {
temp = workMemMin
}
base := 2.0 * workMemPerGigPerConn * float64(parse.Megabyte)
gigs := float64(r.totalMemory)/float64(parse.Gigabyte) - 2.0
temp := ((gigs*(workMemPerGigPerConnWindows*float64(parse.Megabyte)) + base) / float64(r.conns)) / cpuFactor
return parse.BytesToPGFormat(uint64(temp))
return parse.BytesToPGFormat(temp)
}

// MemorySettingsGroup is the SettingsGroup to represent settings that affect memory usage.
Expand Down
53 changes: 34 additions & 19 deletions pkg/pgtune/memory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,19 @@ var memoryToBaseVals = map[uint64]map[string]uint64{
},
}

// cpuVals is the different amounts of CPUs to test
var cpuVals = []int{1, 4, 5}

// connVals is the different number of conns to test
var connVals = []uint64{0, 19, 20, 50}

// memorySettingsMatrix stores the test cases for MemoryRecommend along with
// the expected values
var memorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{}
// highCPUs is the number of CPUs that is high enough that work_mem would normally
// fall below the minimum (64KB) using the standard formula
const highCPUs = 9000

var (
// cpuVals is the different amounts of CPUs to test
cpuVals = []int{1, 4, 5, highCPUs}
// connVals is the different number of conns to test
connVals = []uint64{0, 19, 20, 50}
// memorySettingsMatrix stores the test cases for MemoryRecommend along with
// the expected values
memorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{}
)

func init() {
for mem, baseMatrix := range memoryToBaseVals {
Expand All @@ -57,17 +61,21 @@ func init() {
memorySettingsMatrix[mem][cpus][conns][EffectiveCacheKey] = parse.BytesToPGFormat(baseMatrix[EffectiveCacheKey])
memorySettingsMatrix[mem][cpus][conns][MaintenanceWorkMemKey] = parse.BytesToPGFormat(baseMatrix[MaintenanceWorkMemKey])

// CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same)
cpuFactor := math.Round(float64(cpus) / 2.0)
// Our work_mem values are derivied by a certain amount of memory lost/gained when
// moving away from baseConns
connFactor := float64(MaxConnectionsDefault) / float64(baseConns)
if conns != 0 {
connFactor = float64(conns) / float64(baseConns)
if cpus == highCPUs {
memorySettingsMatrix[mem][cpus][conns][WorkMemKey] = parse.BytesToPGFormat(workMemMin)
} else {
// CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same)
cpuFactor := math.Round(float64(cpus) / 2.0)
// Our work_mem values are derivied by a certain amount of memory lost/gained when
// moving away from baseConns
connFactor := float64(MaxConnectionsDefault) / float64(baseConns)
if conns != 0 {
connFactor = float64(conns) / float64(baseConns)
}

memorySettingsMatrix[mem][cpus][conns][WorkMemKey] =
parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor))
}

memorySettingsMatrix[mem][cpus][conns][WorkMemKey] =
parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor))
}
}
}
Expand Down Expand Up @@ -179,6 +187,13 @@ func TestMemoryRecommenderRecommendWindows(t *testing.T) {
conns: baseConns,
want: "27088" + parse.KB, // from pgtune
},
{
desc: "1GB, 9000 cpus",
totalMemory: parse.Gigabyte,
cpus: highCPUs,
conns: baseConns,
want: "64" + parse.KB,
},
}

for _, c := range cases {
Expand Down
3 changes: 0 additions & 3 deletions pkg/tstune/tuner.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
Expand Down Expand Up @@ -77,8 +76,6 @@ var (
getPGConfigVersionFn = getPGConfigVersion
filepathAbsFn = filepath.Abs

pgVersionRegex = regexp.MustCompile("^PostgreSQL ([0-9]+?).([0-9]+?).*")

// ValidPGVersions is a slice representing the major versions of PostgreSQL
// for which recommendations can be generated.
ValidPGVersions = []string{pgMajor11, pgMajor10, pgMajor96}
Expand Down

0 comments on commit baaec8c

Please sign in to comment.