diff --git a/Makefile b/Makefile index 7e85b35..d3008f7 100644 --- a/Makefile +++ b/Makefile @@ -13,18 +13,6 @@ VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) -# -# systemd files -# - -HAVE_SYSTEMD := $(shell pkg-config --exists systemd 2>/dev/null && echo 'yes') - -ifeq ($(HAVE_SYSTEMD),yes) -UNIT_DIR := $(shell pkg-config --variable=systemdsystemunitdir systemd) -UNIT_FILES = cc-proxy.service cc-proxy.socket -GENERATED_FILES += $(UNIT_FILES) -endif - # # Pretty printing # @@ -35,7 +23,7 @@ QUIET_GOBUILD = $(Q:@=@echo ' GOBUILD '$@;) QUIET_GEN = $(Q:@=@echo ' GEN '$@;) # Entry point -all: cc-proxy $(UNIT_FILES) +all: cc-proxy # # proxy @@ -81,22 +69,13 @@ define INSTALL_FILE endef -all-installable: cc-proxy $(UNIT_FILES) +all-installable: cc-proxy install: all-installable $(call INSTALL_EXEC,cc-proxy,$(LIBEXECDIR)/clear-containers) - $(foreach f,$(UNIT_FILES),$(call INSTALL_FILE,$f,$(UNIT_DIR))) clean: - rm -f cc-proxy $(GENERATED_FILES) - -$(GENERATED_FILES): %: %.in Makefile - @mkdir -p `dirname $@` - $(QUIET_GEN)sed \ - -e 's|[@]bindir[@]|$(BINDIR)|g' \ - -e 's|[@]libexecdir[@]|$(LIBEXECDIR)|' \ - -e "s|[@]localstatedir[@]|$(LOCALSTATEDIR)|" \ - "$<" > "$@" + rm -f cc-proxy # # dist diff --git a/cc-proxy.service.in b/cc-proxy.service.in deleted file mode 100644 index 1c8a5c8..0000000 --- a/cc-proxy.service.in +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Clear Containers Proxy -Documentation=https://github.com/clearcontainers/proxy - -[Service] -ExecStart=@libexecdir@/clear-containers/cc-proxy -LimitNOFILE=infinity -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/cc-proxy.socket.in b/cc-proxy.socket.in deleted file mode 100644 index e4419c2..0000000 --- a/cc-proxy.socket.in +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Clear Containers Proxy Socket -Documentation=https://github.com/clearcontainers/proxy -PartOf=cc-proxy.service - -[Socket] -ListenStream=@localstatedir@/run/clear-containers/proxy.sock -DirectoryMode=0770 -SocketMode=0660 - -[Install] -WantedBy=sockets.target diff --git a/docs/ksm.md b/docs/ksm.md deleted file mode 100644 index adcd910..0000000 --- a/docs/ksm.md +++ /dev/null @@ -1,129 +0,0 @@ -# Kernel Same-Page tuning - -This section describes why `cc-proxy` tunes the host kernel -Kernel Same-Page ([KSM](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/vm/ksm.txt)) -settings and how it does it. - -## What is KSM? - -KSM is a host Linux* kernel feature for de-duplicating memory pages. -Although it was initially designed as a KVM specific feature, it is -now part of the generic Linux memory management subsystem and can -be leveraged by any userspace component or application looking for -memory to save. - -A daemon (`ksmd`) periodically scans userspace memory, looking for -identical pages that can be replaced by a single, write-protected -page. When a process tries to modify this shared page content, it -gets a private copy into its memory space. KSM only scans and merges -pages that are both anonymous and that have been explictly tagged as -mergeable by applications calling into the `madvise` system call -(`int madvice(addr, length, MADV_MERGEABLE)`). - -KSM is customizable through a set of Linux kernel `sysfs` attributes, -the most interesting ones being: - - * `/sys/kernel/mm/ksm/run`: Turns KSM on (`1`) and off (`0`). - * `/sys/kernel/mm/ksm/sleep_millisec`: Knob that specifies the KSM - scanning period. - * `/sys/kernel/mm/ksm/pages_to_scan`: Sets the number of - pages KSM will scan per scanning cycle. - -The memory density improvements that KSM can provide come at a cost. -Depending on the number of anonymous pages it will scan, it can be -relatively expensive on CPU utilization. - -## `cc-proxy` and KSM - -The ideal KSM use case is when a given system runs a large amount -of identical and relatively long running virtual machines (VMs). -This is because when all the VMs run the same code, share the same -data and start the same processes, they will be allocating a lot of -identical pages that KSM can process and de-duplicate into shared -single pages. - -A system running Intel® Clear Containers fits the above use case -precisely. Using KSM appropriately in Clear Container sytems can -significantly reduce memory footprint. This is why `cc-proxy` tries -to optimally tune KSM. - -`cc-proxy` is the right component to tune KSM for the following reasons: - - * KSM settings are system wide ones that should be controlled by - a system daemon like `cc-proxy`. - * `cc-proxy` is where you can optimize KSM settings based on - container creation activities. - -## Theory of operation - -`cc-proxy` supports three KSM modes, available through a command line -option: `cc-proxy -ksm`. - - 1. `initial`: In this mode, `cc-proxy` does not take control of the - system KSM settings. It does not turn KSM on or off, and does not - modify any of the KSM settings. - - 2. `off`: When running with `-ksm off`, `cc-proxy` will completely - disable KSM on the system. - - 3. `throttle`: In `throttle` mode, `cc-proxy` will throttle KSM up - and down depending on the Clear Containers creation activity. This - is the `cc-proxy` default KSM mode. - -### KSM throttling - -By default, `cc-proxy` will throttle KSM up and down. Regardless of -the current KSM system settings, `cc-proxy` will move them to the -`aggressive` settings as soon as a new Clear Container is created. -With the `aggressive` setting, ksmd will run every millisecond and -will scan 10% of all available anonymous pages during each scanning -cycle. - -After switching to the `aggressive` KSM settings, `cc-proxy` will -throttle down to the `standard` setting if there are no additional -Clear Containers instances created for 30 seconds. -Then `cc-proxy` will continue throttling down to the `slow` KSM -setting if there no further Clear Container creation for the next -two minutes. -Finally, `cc-proxy` will get back to the initial KSM settings after -two more minutes, unless a new Clear Container is created. - -At any point in time, `cc-proxy` will get back to to the `aggressive` -setting when detecting the creation of a Clear Container: - -``` - +----------------+ - | | - | Initial | - | Settings |<<-------------------------------+ - | | | - +-------+--------+ | - | | - New | | - Container | | - | | - v | - +--------------+ | - | Aggressive |<<--------+ | - +--------------+ | | - | | | - No Container | | | - (30s) | | New | - | | Container | No Container - v | | (2mn) - +--------------+ | | - | Standard |----------+ | - +--------------+ | | - | | | - No Container | | | - (2mn) | | New | - | | Container | - v | | - +--------------+ | | - | Slow |----------+ | - +--------------+ | - | | - | | - +------------------------------------------+ - -``` \ No newline at end of file diff --git a/ksm.go b/ksm.go deleted file mode 100644 index 96d22e1..0000000 --- a/ksm.go +++ /dev/null @@ -1,540 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bufio" - "errors" - "fmt" - "io/ioutil" - "os" - "os/signal" - "path/filepath" - "strconv" - "strings" - "sync" - "syscall" - "time" -) - -type ksmSetting struct { - // run describes if we want KSM to be on or off. - run bool - - // pagesPerScanFactor describes how many pages we want - // to scan per KSM run. - // ksmd will san N pages, where N*pagesPerScanFactor is - // equal to the number of anonymous pages. - pagesPerScanFactor int64 - - // scanIntervalMS is the KSM scan interval in milliseconds. - scanIntervalMS uint32 -} - -func anonPages() (int64, error) { - // We're going to parse meminfo - f, err := os.Open(memInfo) - if err != nil { - return -1, err - } - defer f.Close() - - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - - // We only care about anonymous pages - if !strings.HasPrefix(line, "AnonPages:") { - continue - } - - // Extract the before last (value) and last (unit) fields - fields := strings.Split(line, " ") - value := fields[len(fields)-2] - totalMemory, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return -1, fmt.Errorf("Invalid integer") - } - - // meminfo gives us kB - totalMemory *= 1024 - - // Fetch the system page size - pageSize := (int64)(os.Getpagesize()) - - nPages := totalMemory / pageSize - return nPages, nil - } - - return 0, fmt.Errorf("Could not compute number of pages") -} - -func (s ksmSetting) pagesToScan() (string, error) { - if s.pagesPerScanFactor == 0 { - return "", errors.New("Invalid KSM setting") - } - - nPages, err := anonPages() - if err != nil { - return "", err - } - - pagesToScan := nPages / s.pagesPerScanFactor - - return fmt.Sprintf("%v", pagesToScan), nil -} - -type ksmMode string - -const ( - ksmInitial ksmMode = "initial" - ksmOff ksmMode = "off" - ksmSlow ksmMode = "slow" - ksmStandard ksmMode = "standard" - ksmAggressive ksmMode = "aggressive" - ksmAuto ksmMode = "auto" -) - -var ksmSettings = map[ksmMode]ksmSetting{ - ksmOff: {false, 1000, 500}, // Turn KSM off - ksmSlow: {true, 500, 100}, // Every 100ms, we scan 1 page for every 500 pages available in the system - ksmStandard: {true, 100, 10}, // Every 10ms, we scan 1 page for every 100 pages available in the system - ksmAggressive: {true, 10, 1}, // Every ms, we scan 1 page for every 10 pages available in the system -} - -func (k ksmMode) String() string { - switch k { - case ksmOff: - return "off" - case ksmInitial: - return "initial" - case ksmAuto: - return "auto" - } - - return "" -} - -func (k *ksmMode) Set(value string) error { - for _, r := range strings.Split(value, ",") { - if r == "off" { - *k = ksmOff - return nil - } else if r == "initial" { - *k = ksmInitial - return nil - } else if r == "auto" { - *k = ksmAuto - return nil - } - - return fmt.Errorf("Unsupported KSM knob %v", r) - } - - return nil -} - -var defaultKSMRoot = "/sys/kernel/mm/ksm/" -var errKSMUnavailable = errors.New("KSM is unavailable") -var memInfo = "/proc/meminfo" - -const ( - ksmRunFile = "run" - ksmPagesToScan = "pages_to_scan" - ksmSleepMillisec = "sleep_millisecs" - ksmStart = "1" - ksmStop = "0" - defaultKSMMode = ksmAuto -) - -type sysfsAttribute struct { - path string - file *os.File -} - -func (attr *sysfsAttribute) open() error { - file, err := os.OpenFile(attr.path, os.O_RDWR|syscall.O_NONBLOCK, 0660) - attr.file = file - return err -} - -func (attr *sysfsAttribute) close() error { - err := attr.file.Close() - attr.file = nil - return err -} - -func (attr *sysfsAttribute) read() (string, error) { - _, err := attr.file.Seek(0, os.SEEK_SET) - if err != nil { - return "", err - } - - data, err := ioutil.ReadAll(attr.file) - if err != nil { - return "", err - } - - return string(data), nil -} - -func (attr *sysfsAttribute) write(value string) error { - _, err := attr.file.Seek(0, os.SEEK_SET) - if err != nil { - return err - } - - err = attr.file.Truncate(0) - if err != nil { - return err - } - - _, err = attr.file.WriteString(value) - - return err -} - -type ksm struct { - root string - run sysfsAttribute - pagesToScan sysfsAttribute - sleepInterval sysfsAttribute - initialized bool - - initialPagesToScan string - initialSleepInterval string - initialKSMRun string - - currentKnob ksmMode - throttling bool - kickChannel chan bool - - sync.Mutex -} - -func (k *ksm) isAvailable() error { - info, err := os.Stat(k.root) - if err != nil || !info.IsDir() { - return fmt.Errorf("%s is not available", k.root) - } - - return nil -} - -func newKSM(root string) (*ksm, error) { - var err error - var k ksm - - k.initialized = false - k.throttling = false - k.root = root - - if root == "" { - return nil, errors.New("Invalid KSM root") - } - - if err := k.isAvailable(); err != nil { - return nil, err - } - - k.pagesToScan = sysfsAttribute{ - path: filepath.Join(k.root, ksmPagesToScan), - } - - k.sleepInterval = sysfsAttribute{ - path: filepath.Join(k.root, ksmSleepMillisec), - } - - k.run = sysfsAttribute{ - path: filepath.Join(k.root, ksmRunFile), - } - - defer func(err error) { - if err != nil { - _ = k.run.close() - _ = k.sleepInterval.close() - _ = k.pagesToScan.close() - } - }(err) - - if err := k.run.open(); err != nil { - return nil, err - } - - if err := k.sleepInterval.open(); err != nil { - return nil, err - } - - if err := k.pagesToScan.open(); err != nil { - return nil, err - } - - k.initialPagesToScan, err = k.pagesToScan.read() - if err != nil { - return nil, err - } - - k.initialSleepInterval, err = k.sleepInterval.read() - if err != nil { - return nil, err - } - - k.initialKSMRun, err = k.run.read() - if err != nil { - return nil, err - } - - k.initialized = true - k.kickChannel = make(chan bool) - - return &k, nil -} - -func startKSM(root string, mode ksmMode) (*ksm, error) { - k, err := newKSM(root) - if err != nil { - return k, err - } - - // We just no-op if going for initial settings - if mode != ksmInitial { - // We want to catch termination to restore the initial sysfs values - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - - go func() { - <-c - _ = k.restore() - os.Exit(0) - }() - - if mode == ksmAuto { - k.throttle() - } else { - setting, ok := ksmSettings[mode] - if !ok { - return k, fmt.Errorf("Invalide KSM mode %v", mode) - } - - if err := k.tune(setting); err != nil { - return k, err - } - } - } - - return k, nil -} - -// restoreSysFS is unlocked. You should take the ksm lock before calling it. -func (k *ksm) restoreSysFS() error { - var err error - - if !k.initialized { - return errKSMUnavailable - } - - if err = k.pagesToScan.write(k.initialPagesToScan); err != nil { - return err - } - - if err = k.sleepInterval.write(k.initialSleepInterval); err != nil { - return err - } - - return k.run.write(k.initialKSMRun) -} - -func (k *ksm) restore() error { - var err error - - k.Lock() - defer k.Unlock() - - if !k.initialized { - return errors.New("KSM is unavailable") - } - - if err = k.restoreSysFS(); err != nil { - return err - } - - if err := k.run.close(); err != nil { - return err - } - - if err := k.sleepInterval.close(); err != nil { - return err - } - - if err := k.pagesToScan.close(); err != nil { - return err - } - - k.initialized = false - return nil -} - -func (k *ksm) tune(s ksmSetting) error { - k.Lock() - defer k.Unlock() - - if !k.initialized { - return errKSMUnavailable - } - - if !s.run { - return k.run.write(ksmStop) - } - - newPagesToScan, err := s.pagesToScan() - if err != nil { - return err - } - - if err = k.run.write(ksmStop); err != nil { - return err - } - - if err = k.pagesToScan.write(newPagesToScan); err != nil { - return err - } - - if err = k.sleepInterval.write(fmt.Sprintf("%v", s.scanIntervalMS)); err != nil { - return err - } - - return k.run.write(ksmStart) -} - -type ksmThrottleInterval struct { - interval time.Duration - nextKnob ksmMode -} - -var ksmAggressiveInterval = 30 * time.Second -var ksmStandardInterval = 120 * time.Second -var ksmSlowInterval = 120 * time.Second - -var ksmThrottleIntervals = map[ksmMode]ksmThrottleInterval{ - ksmAggressive: { - // From aggressive: move to standard and wait 120s - interval: ksmStandardInterval, - nextKnob: ksmStandard, - }, - - ksmStandard: { - // From standard: move to slow and wait 120s - interval: ksmSlowInterval, - nextKnob: ksmSlow, - }, - - ksmSlow: { - // From slow: move to the initial settings and stop there - interval: 0, - nextKnob: ksmInitial, - }, - - // We should never make it here - ksmInitial: { - interval: 0, // We stay here unless a new container shows up - }, -} - -func (k *ksm) throttle() { - k.Lock() - defer k.Unlock() - - if !k.initialized { - proxyLog.Error(errors.New("KSM is unavailable")) - return - } - - k.currentKnob = ksmAggressive - k.throttling = true - - go func() { - throttleTimer := time.NewTimer(ksmThrottleIntervals[k.currentKnob].interval) - - for { - select { - case <-k.kickChannel: - // We got kicked, this means a new VM has been created. - // We will enter the aggressive setting until we throttle down. - _ = throttleTimer.Stop() - if err := k.tune(ksmSettings[ksmAggressive]); err != nil { - proxyLog.Error(err) - continue - } - - k.Lock() - k.currentKnob = ksmAggressive - k.Unlock() - - _ = throttleTimer.Reset(ksmAggressiveInterval) - - case <-throttleTimer.C: - // Our throttling down timer kicked in. - // We will move down to the next knob and start the next time, - // if necessary. - var throttle = ksmThrottleIntervals[k.currentKnob] - if throttle.interval == 0 { - if throttle.nextKnob == ksmInitial { - k.Lock() - if err := k.restoreSysFS(); err != nil { - proxyLog.Error(err) - } - k.Unlock() - } - continue - } - - nextKnob := ksmThrottleIntervals[k.currentKnob].nextKnob - interval := ksmThrottleIntervals[k.currentKnob].interval - if err := k.tune(ksmSettings[nextKnob]); err != nil { - proxyLog.Error(err) - continue - } - - k.Lock() - k.currentKnob = nextKnob - k.Unlock() - - _ = throttleTimer.Reset(interval) - } - } - }() -} - -// kick gets us back to the aggressive setting -func (k *ksm) kick() { - k.Lock() - - if !k.initialized { - proxyLog.Error(errors.New("KSM is unavailable")) - k.Unlock() - return - } - - // If we're not throttling, we must not kick. - if !k.throttling { - k.Unlock() - return - } - - k.Unlock() - k.kickChannel <- true -} diff --git a/ksm_test.go b/ksm_test.go deleted file mode 100644 index 3a62582..0000000 --- a/ksm_test.go +++ /dev/null @@ -1,567 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const ksmString = "ksmrules" -const anonPagesMemory = 16777216 // Typically 4096 pages -const run = "1" -const interval = "10" -const scan = "1000" - -func ksmTestPrepare() error { - newKSMRoot, err := ioutil.TempDir("", "cc-ksm-test") - if err != nil { - return err - } - - defaultKSMRoot = newKSMRoot - - memInfoFile, err := ioutil.TempFile("", "cc-ksm-meminfo") - if err != nil { - return err - } - - memInfo = memInfoFile.Name() - - _, err = memInfoFile.WriteString(fmt.Sprintf("AnonPages: %v kB", anonPagesMemory)) - if err != nil { - return err - } - - ksmTestRun, err := os.Create(filepath.Join(defaultKSMRoot, ksmRunFile)) - if err != nil { - return err - } - - ksmTestPagesToScan, err := os.Create(filepath.Join(defaultKSMRoot, ksmPagesToScan)) - if err != nil { - return err - } - - ksmTestSleepMillisec, err := os.Create(filepath.Join(defaultKSMRoot, ksmSleepMillisec)) - if err != nil { - return err - } - - defer ksmTestRun.Close() - defer ksmTestPagesToScan.Close() - defer ksmTestSleepMillisec.Close() - - return nil -} - -func ksmTestCleanup() { - os.RemoveAll(defaultKSMRoot) - os.RemoveAll(memInfo) -} - -func TestKSMSysfsAttributeOpen(t *testing.T) { - pagesToScanSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmPagesToScan), - } - - err := pagesToScanSysFs.open() - defer pagesToScanSysFs.close() - - assert.Nil(t, err) -} - -func TestKSMSysfsAttributeOpenNonExistent(t *testing.T) { - pagesToScanSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, "foo"), - } - - err := pagesToScanSysFs.open() - defer pagesToScanSysFs.close() - - assert.NotNil(t, err) -} - -func TestKSMSysfsAttributeReadWrite(t *testing.T) { - pagesToScanSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmPagesToScan), - } - - err := pagesToScanSysFs.open() - defer pagesToScanSysFs.close() - - assert.Nil(t, err) - - err = pagesToScanSysFs.write(ksmString) - assert.Nil(t, err) - - s, err := pagesToScanSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, ksmString, "Wrong sysfs read: %s", s) -} - -func initKSM(root string, t *testing.T) *ksm { - _, err := newKSM("") - assert.NotNil(t, err) - - k, err := newKSM(root) - assert.Nil(t, err) - - return k -} - -func TestKSMAvailabilityDummy(t *testing.T) { - _, err := newKSM("foo") - assert.NotNil(t, err) -} - -func TestKSMAvailability(t *testing.T) { - k := initKSM(defaultKSMRoot, t) - - err := k.isAvailable() - assert.Nil(t, err) -} - -func TestKSMAnonPages(t *testing.T) { - pageSize := (int64)(os.Getpagesize()) - expectedAnonPages := (anonPagesMemory * 1024) / pageSize - - anon, err := anonPages() - assert.Nil(t, err) - assert.Equal(t, expectedAnonPages, anon, "Anonymous pages mismatch") -} - -func TestKSMPagesToScan(t *testing.T) { - setting, valid := ksmSettings[ksmAggressive] - assert.True(t, valid) - - anonPages, err := anonPages() - assert.Nil(t, err) - expectedPagesToScan := fmt.Sprintf("%v", anonPages/setting.pagesPerScanFactor) - - pagesToScan, err := setting.pagesToScan() - assert.Nil(t, err) - assert.Equal(t, pagesToScan, expectedPagesToScan, "") -} - -func TestKSMPagesToScanInvalidSetting(t *testing.T) { - setting := ksmSetting{ - pagesPerScanFactor: 0, - } - - _, err := setting.pagesToScan() - assert.NotNil(t, err) -} - -func TestKSMInit(t *testing.T) { - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err := runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - err = runSysFs.write(run) - assert.Nil(t, err) - - pagesToScanSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmPagesToScan), - } - - err = pagesToScanSysFs.open() - defer pagesToScanSysFs.close() - assert.Nil(t, err) - - err = pagesToScanSysFs.write(scan) - assert.Nil(t, err) - - sleepIntervalSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmSleepMillisec), - } - - err = sleepIntervalSysFs.open() - defer sleepIntervalSysFs.close() - assert.Nil(t, err) - - err = sleepIntervalSysFs.write(interval) - assert.Nil(t, err) - - k := initKSM(defaultKSMRoot, t) - - assert.Equal(t, k.initialPagesToScan, scan) - assert.Equal(t, k.initialSleepInterval, interval) - assert.Equal(t, k.initialKSMRun, run) -} - -func TestKSMRestore(t *testing.T) { - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err := runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - err = runSysFs.write(run) - assert.Nil(t, err) - - pagesToScanSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmPagesToScan), - } - - err = pagesToScanSysFs.open() - defer pagesToScanSysFs.close() - assert.Nil(t, err) - - err = pagesToScanSysFs.write(scan) - assert.Nil(t, err) - - sleepIntervalSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmSleepMillisec), - } - - err = sleepIntervalSysFs.open() - defer sleepIntervalSysFs.close() - assert.Nil(t, err) - - err = sleepIntervalSysFs.write(interval) - assert.Nil(t, err) - - k := initKSM(defaultKSMRoot, t) - - // Write dummy values and read them back - var newInterval = "foo" - var newRun = "bar" - var newScan = "foobar" - - err = sleepIntervalSysFs.write(newInterval) - assert.Nil(t, err) - - s, err := sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, newInterval) - - err = runSysFs.write(newRun) - assert.Nil(t, err) - s, err = runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, newRun) - - err = pagesToScanSysFs.write(newScan) - assert.Nil(t, err) - s, err = pagesToScanSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, newScan) - - // Now restore and verify that we read the initial values back - k.restore() - - s, err = pagesToScanSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, scan) - - s, err = runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, run) - - s, err = sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, interval) -} - -func TestKSMKick(t *testing.T) { - k := initKSM(defaultKSMRoot, t) - - timer := time.NewTimer(time.Second) - k.throttling = true - go k.kick() - - select { - case <-k.kickChannel: - return - - case <-timer.C: - t.Fatalf("KSM kick timeout") - } -} - -func TestKSMTune(t *testing.T) { - var err error - var s string - - sleepIntervalSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmSleepMillisec), - } - - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err = sleepIntervalSysFs.open() - defer sleepIntervalSysFs.close() - assert.Nil(t, err) - - err = runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - k := initKSM(defaultKSMRoot, t) - - for _, v := range ksmSettings { - err = k.tune(v) - assert.Nil(t, err) - - s, err = runSysFs.read() - - assert.Nil(t, err) - assert.NotNil(t, s) - if v.run { - assert.Equal(t, s, "1", "Wrong run value") - } else { - assert.Equal(t, s, "0", "Wrong run value") - } - - if !v.run { - continue - } - - s, err = sleepIntervalSysFs.read() - - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, fmt.Sprintf("%v", v.scanIntervalMS), "Wrong sleep interval") - } -} - -func TestKSMModeSet(t *testing.T) { - var k ksmMode - var err error - - err = k.Set(string(ksmOff)) - assert.Nil(t, err) - assert.Equal(t, k, ksmOff) - - err = k.Set(string(ksmAuto)) - assert.Nil(t, err) - assert.Equal(t, k, ksmAuto) - - err = k.Set(string(ksmInitial)) - assert.Nil(t, err) - assert.Equal(t, k, ksmInitial) - - err = k.Set(string(ksmStandard)) - assert.NotNil(t, err) - - err = k.Set(fmt.Sprintf("%s,%s", ksmAuto, ksmOff)) - assert.Nil(t, err) - assert.Equal(t, k, ksmAuto) -} - -func TestKSMModeString(t *testing.T) { - var k ksmMode - var s string - - k = ksmOff - s = string(k) - assert.Equal(t, string(k), s) - - k = ksmInitial - s = string(k) - assert.Equal(t, string(k), s) - - k = ksmAuto - s = string(k) - assert.Equal(t, string(k), s) -} - -func boolString(b bool) string { - if b { - return "1" - } - - return "0" -} - -func testThrottle(k *ksm, t *testing.T) { - var err error - var s string - - assert.NotNil(t, k) - - sleepIntervalSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmSleepMillisec), - } - - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err = sleepIntervalSysFs.open() - defer sleepIntervalSysFs.close() - assert.Nil(t, err) - - err = runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - // We should first be in aggressive mode - time.Sleep(100 * time.Millisecond) - k.Lock() - assert.Equal(t, k.currentKnob, ksmAggressive) - k.Unlock() - - // Let's check sysfs values are the aggressive ones - s, err = runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, boolString(ksmSettings[ksmAggressive].run), s, "Wrong sysfs read: %s", s) - - s, err = sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, fmt.Sprintf("%d", ksmSettings[ksmAggressive].scanIntervalMS), s, "Wrong sysfs read: %s", s) - - nextKnob := ksmThrottleIntervals[ksmAggressive].nextKnob - time.Sleep(ksmAggressiveInterval) - - // Let's check for the next knob - time.Sleep(100 * time.Millisecond) - k.Lock() - assert.Equal(t, k.currentKnob, nextKnob) - expectedRun := boolString(ksmSettings[k.currentKnob].run) - expectedScan := ksmSettings[k.currentKnob].scanIntervalMS - k.Unlock() - - // Let's check sysfs values are properly set - s, err = runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, expectedRun, s, "Wrong sysfs read: %s", s) - - s, err = sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, fmt.Sprintf("%d", expectedScan), s, "Wrong sysfs read: %s", s) -} - -func TestKSMThrottle(t *testing.T) { - k := initKSM(defaultKSMRoot, t) - - // Let's make the throttling down faster, for quicker tests purpose. - ksmAggressiveInterval = 500 * time.Millisecond - - k.throttle() - k.kick() - - testThrottle(k, t) -} - -func TestKSMStartInitialMode(t *testing.T) { - var err error - - sleepIntervalSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmSleepMillisec), - } - - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err = sleepIntervalSysFs.open() - defer sleepIntervalSysFs.close() - assert.Nil(t, err) - - err = runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - initialRun, err := runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, initialRun) - - initialSleep, err := sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, initialSleep) - - k, err := startKSM(defaultKSMRoot, ksmInitial) - assert.Nil(t, err) - assert.NotNil(t, k) - - newRun, err := runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, newRun) - assert.Equal(t, newRun, initialRun, "Run sysfs attribute modified") - - newSleep, err := sleepIntervalSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, newSleep) - assert.Equal(t, newSleep, initialSleep, "Sleep sysfs attribute modified") -} - -func TestKSMStartAutoMode(t *testing.T) { - // Let's make the throttling down faster, for quicker tests purpose. - ksmAggressiveInterval = 500 * time.Millisecond - - k, err := startKSM(defaultKSMRoot, ksmAuto) - assert.Nil(t, err) - assert.NotNil(t, k) - - k.kick() - - testThrottle(k, t) -} - -func TestKSMStartOffMode(t *testing.T) { - var err error - - runSysFs := sysfsAttribute{ - path: filepath.Join(defaultKSMRoot, ksmRunFile), - } - - err = runSysFs.open() - defer runSysFs.close() - assert.Nil(t, err) - - k, err := startKSM(defaultKSMRoot, ksmOff) - assert.Nil(t, err) - assert.NotNil(t, k) - - s, err := runSysFs.read() - assert.Nil(t, err) - assert.NotNil(t, s) - assert.Equal(t, s, ksmStop, "KSM not stopped") -} - -func TestKSMStartInvalidMode(t *testing.T) { - k, err := startKSM(defaultKSMRoot, "foo") - assert.NotNil(t, k) - assert.NotNil(t, err) -} diff --git a/main_test.go b/main_test.go index 3251c9a..f159a4b 100644 --- a/main_test.go +++ b/main_test.go @@ -31,14 +31,5 @@ func TestMain(m *testing.M) { fmt.Fprint(os.Stderr, err) } - if err := ksmTestPrepare(); err != nil { - ksmTestCleanup() - fmt.Fprint(os.Stderr, err) - } - - exit := m.Run() - - ksmTestCleanup() - - os.Exit(exit) + os.Exit(m.Run()) } diff --git a/proxy.go b/proxy.go index ce29154..9dfa526 100644 --- a/proxy.go +++ b/proxy.go @@ -69,9 +69,6 @@ var proxyLog = logrus.WithFields(logrus.Fields{ "pid": os.Getpid(), }) -// KSM setting -var proxyKSM *ksm - // Main struct holding the proxy state type proxy struct { // Protect concurrent accesses from separate client goroutines to this @@ -263,10 +260,6 @@ func registerVM(data []byte, userData interface{}, response *handlerResponse) { client.vm = vm - if proxyKSM != nil { - proxyKSM.kick() - } - // We start one goroutine per-VM to monitor the qemu process proxy.wg.Add(1) go func() { @@ -344,10 +337,6 @@ func unregisterVM(data []byte, userData interface{}, response *handlerResponse) client.vm = nil - if !podInstance { - return - } - // Signal the proxy to exit proxy.finished = true if proxy.listener != nil { @@ -648,36 +637,24 @@ func (proxy *proxy) init(uri string) error { if proxy.socketPath, err = getSocketPath(uri); err != nil { return fmt.Errorf("couldn't get a valid socket path: %v", err) } - fds := listenFds() - - if len(fds) > 1 { - return fmt.Errorf("too many activated sockets (%d)", len(fds)) - } else if len(fds) == 1 { - fd := fds[0] - l, err = net.FileListener(fd) - if err != nil { - return fmt.Errorf("couldn't listen on socket: %v", err) - } - - } else { - socketDir := filepath.Dir(proxy.socketPath) - if err = os.MkdirAll(socketDir, 0750); err != nil { - return fmt.Errorf("couldn't create socket directory: %v", err) - } - if err = os.Remove(proxy.socketPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("couldn't remove exiting socket: %v", err) - } - l, err = net.ListenUnix("unix", &net.UnixAddr{Name: proxy.socketPath, Net: "unix"}) - if err != nil { - return fmt.Errorf("couldn't create AF_UNIX socket: %v", err) - } - if err = os.Chmod(proxy.socketPath, 0660|os.ModeSocket); err != nil { - return fmt.Errorf("couldn't set mode on socket: %v", err) - } - proxyLog.Info("listening on ", proxy.socketPath) + socketDir := filepath.Dir(proxy.socketPath) + if err = os.MkdirAll(socketDir, 0750); err != nil { + return fmt.Errorf("couldn't create socket directory: %v", err) + } + if err = os.Remove(proxy.socketPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("couldn't remove exiting socket: %v", err) + } + l, err = net.ListenUnix("unix", &net.UnixAddr{Name: proxy.socketPath, Net: "unix"}) + if err != nil { + return fmt.Errorf("couldn't create AF_UNIX socket: %v", err) + } + if err = os.Chmod(proxy.socketPath, 0660|os.ModeSocket); err != nil { + return fmt.Errorf("couldn't set mode on socket: %v", err) } + proxyLog.Info("listening on ", proxy.socketPath) + proxy.listener = l return nil @@ -725,17 +702,11 @@ func (proxy *proxy) serve() { for { conn, err := proxy.listener.Accept() if err != nil { - if podInstance && proxy.finished { + if proxy.finished { break } - msg := fmt.Sprint("couldn't accept connection:", err) - if podInstance { - proxyLog.Info(msg) - } else { - fmt.Fprintln(os.Stderr, msg) - } - + proxyLog.Infof("couldn't accept connection:", err) continue } @@ -744,45 +715,12 @@ func (proxy *proxy) serve() { } func proxyMain(uri string) { - var err error - - if podInstance { - proxyLog.WithField("mode", "pod").Info("starting") - } else { - fmt.Fprintln(os.Stderr, "starting in system mode") - } - proxy := newProxy() if err := proxy.init(uri); err != nil { - if podInstance { - proxyLog.WithError(err).Error("init failed") - } else { - fmt.Fprint(os.Stderr, "init failed: ", err.Error()) - } - + proxyLog.WithError(err).Error("init failed") os.Exit(1) } - // KSM is only available when running as a system-level daemon - // (where a URI is unecessary). - if !podInstance { - // Init and tune KSM if available - proxyKSM, err = startKSM(defaultKSMRoot, proxyKSMMode) - if err != nil { - // KSM failure should not be fatal - msg := "KSM setup failed" - if podInstance { - proxyLog.WithError(err).Warn(msg) - } else { - fmt.Fprintf(os.Stderr, "%s: %s\n", msg, err.Error()) - } - } else { - defer func() { - _ = proxyKSM.restore() - }() - } - } - proxy.serve() // Wait for all the goroutines started by registerVMHandler to finish. @@ -831,18 +769,16 @@ func SetLoggingParams(logLevel string) error { } // log to syslog - if podInstance { - syslogHook, err := lsyslog.NewSyslogHook("", - "", - syslog.LOG_INFO|syslog.LOG_DAEMON, - name) - if err != nil { - return err - } - - proxyLog.Logger.Hooks.Add(syslogHook) + syslogHook, err := lsyslog.NewSyslogHook("", + "", + syslog.LOG_INFO|syslog.LOG_DAEMON, + name) + if err != nil { + return err } + proxyLog.Logger.Hooks.Add(syslogHook) + return nil } @@ -868,10 +804,6 @@ func (p *profiler) setup() { // Version is the proxy version. This variable is populated at build time. var Version = "unknown" -var proxyKSMMode ksmMode - -// true if associated with a single POD -var podInstance bool func main() { doVersion := flag.Bool("version", false, "display the version") @@ -887,18 +819,10 @@ func main() { "host the pprof server will be bound to") flag.UintVar(&pprof.port, "pprof-port", 6060, "port the pprof server will be bound to") - flag.StringVar(&uri, "uri", "", "proxy socket URI (note: disables KSM entirely)") - - proxyKSMMode = defaultKSMMode - - flag.Var(&proxyKSMMode, "ksm", "KSM settings [off, initial, auto]") + flag.StringVar(&uri, "uri", "", "proxy socket URI") flag.Parse() - if uri != "" { - podInstance = true - } - if err := SetLoggingParams(*logLevel); err != nil { logrus.Fatal(err) } diff --git a/socket_activation.go b/socket_activation.go deleted file mode 100644 index e6e5550..0000000 --- a/socket_activation.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "strconv" - "syscall" -) - -const ( - listenFdsStart = 3 -) - -func listenFds() []*os.File { - pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) - if err != nil || pid != os.Getpid() { - return nil - } - - nfds, err := strconv.Atoi(os.Getenv("LISTEN_FDS")) - if err != nil || nfds == 0 { - return nil - } - - files := []*os.File(nil) - for fd := listenFdsStart; fd < listenFdsStart+nfds; fd++ { - syscall.CloseOnExec(fd) - files = append(files, os.NewFile(uintptr(fd), "")) - } - - return files -} diff --git a/vm.go b/vm.go index 883168e..8beb98f 100644 --- a/vm.go +++ b/vm.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "net" - "os" "strconv" "strings" "sync" @@ -230,13 +229,7 @@ func (vm *vm) ioHyperToClients() { session := vm.findSessionBySeq(msg.Session) if session == nil { - msg := fmt.Sprintf("couldn't find client with seq number %d", msg.Session) - - if podInstance { - vm.logIO.Info(msg) - } else { - fmt.Fprintln(os.Stderr, msg) - } + vm.logIO.Infof("couldn't find client with seq number %d", msg.Session) continue }