remove old thread info from cpu metrics

In case if application starts new threads it's possible that in
result thread info will contain info about all threads that were
started ever. As result size of such metrics will be quite huge.
This patch removes info about dead threads and user will see info
only about relevant threads.

Closes #376

Author: olegrok

CHANGELOG.md

@@ -9,8 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Float numbers in Graphite exporter
 - Signed timestamp in Graphite exporter
-- Increase `Shared.make_key` perfomance in observations with empty label
+- Increase `Shared.make_key` performance in observations with empty label
 - Forbid observation of non-number value in collectors (except `gauge:set`)
+- Clean dead threads from `psutils.cpu` metric
 
 ### Added
 

metrics/psutils/cpu.lua

@@ -12,25 +12,42 @@ local instance_file = arg[0]
 collectors_list.cpu_count = utils.set_gauge('cpu_count', 'The number of processors', psutils.get_cpu_count())
 collectors_list.cpu_number = utils.set_gauge('cpu_number', 'The number of processors', psutils.get_cpu_count())
 
+local threads = {}
+
 local function update_cpu_metrics()
     utils.set_gauge('cpu_total', 'Host CPU time', psutils.get_cpu_time())
     utils.set_gauge('cpu_time', 'Host CPU time', psutils.get_cpu_time())
 
+    local new_threads = {}
     for _, thread_info in ipairs(psutils.get_process_cpu_time()) do
-        collectors_list.cpu_thread = utils.set_gauge('cpu_thread', 'Tarantool thread cpu time', thread_info.utime, {
-            kind = 'user',
+        local labels = {
             thread_name = thread_info.comm,
             thread_pid = thread_info.pid,
             file_name = instance_file,
-        })
+        }
 
-        collectors_list.cpu_thread = utils.set_gauge('cpu_thread', 'Tarantool thread cpu time', thread_info.stime, {
-            kind = 'system',
-            thread_name = thread_info.comm,
-            thread_pid = thread_info.pid,
-            file_name = instance_file,
-        })
+        local utime_labels = table.copy(labels)
+        utime_labels.kind = 'user'
+        collectors_list.cpu_thread = utils.set_gauge('cpu_thread', 'Tarantool thread cpu time',
+            thread_info.utime, utime_labels)
+
+        local stime_labels = table.copy(labels)
+        stime_labels.kind = 'user'
+        collectors_list.cpu_thread = utils.set_gauge('cpu_thread', 'Tarantool thread cpu time',
+            thread_info.stime, stime_labels)
+
+        threads[thread_info.pid] = nil
+        new_threads[thread_info.pid] = labels
+    end
+
+    for _, thread_info in pairs(threads) do
+        thread_info.kind = 'user'
+        collectors_list.cpu_thread:remove(thread_info)
+
+        thread_info.kind = 'system'
+        collectors_list.cpu_thread:remove(thread_info)
     end
+    threads = new_threads
 end
 
 return {

test/psutils_linux_thread_clean_test.lua

@@ -0,0 +1,56 @@
+#!/usr/bin/env tarantool
+
+local t = require('luatest')
+local g = t.group('psutils_linux_clean_info')
+local utils = require('test.utils')
+local metrics = require('metrics')
+local fiber = require('fiber')
+local fio = require('fio')
+local psutils_linux = require('metrics.psutils.psutils_linux')
+local cpu = require('metrics.psutils.cpu')
+
+g.before_all(function()
+    t.skip_if(jit.os ~= 'Linux', 'Linux is the only supported platform')
+    utils.init()
+end)
+
+g.after_each(function()
+    metrics.clear()
+end)
+
+g.test_clean_thread_info = function()
+    box.cfg{worker_pool_threads = 100}
+
+    for _ = 1, 1000 do
+        fiber.new(function() fio.stat(arg[-1]) end)
+    end
+    fiber.sleep(0.1)
+
+    cpu.update()
+    local list1 = psutils_linux.get_process_cpu_time()
+    local observations1 = metrics.collect()
+    local coio_count1 = 0
+    for _, thread_info in ipairs(list1) do
+        if thread_info.comm == 'coio' then
+            coio_count1 = coio_count1 + 1
+        end
+    end
+
+    box.cfg{worker_pool_threads = 1}
+    fiber.sleep(0.1)
+
+    cpu.update()
+    local list2 = psutils_linux.get_process_cpu_time()
+    local observations2 = metrics.collect()
+    local coio_count2 = 0
+    for _, thread_info in ipairs(list2) do
+        if thread_info.comm == 'coio' then
+            coio_count2 = coio_count2 + 1
+        end
+    end
+
+    t.assert_gt(#list1, #list2)
+    t.assert_gt(#observations1, #observations2)
+    t.assert_gt(coio_count1, coio_count2)
+    t.assert_equals(#observations1 - #observations2, coio_count1 - coio_count2)
+end