Add new metrics (#483)

Co-authored-by: Georgy Moiseev <[email protected]>

Author: yngvar-antonsson

.github/workflows/test.yml

@@ -44,6 +44,10 @@ jobs:
             cartridge: "2.9.0"
           - tarantool: "2.11"
             cartridge: "2.10.0"
+          - tarantool: "3.0"
+            cartridge: ""
+          - tarantool: "3.1"
+            cartridge: ""
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -18,6 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Inconsistent metrics descrtiptions for memtx metrics.
 
+- New memory metrics:
+  - tnt_memtx_tuples_data_total
+  - tnt_memtx_tuples_data_read_view
+  - tnt_memtx_tuples_data_garbage
+  - tnt_memtx_index_total
+  - tnt_memtx_index_read_view
+  - tnt_vinyl_memory_tuple
+
 ### Deprecated
 
 - Using `cdata` values with `histogram:observe`.

doc/monitoring/metrics_reference.rst

@@ -665,6 +665,9 @@ The following metrics show state memory areas used by vinyl for caches and write
         *   -   ``tnt_vinyl_memory_bloom_filter``
             -   Amount of memory in bytes used by
                 :ref:`bloom filters <vinyl-lsm_disadvantages_compression_bloom_filters>`.
+        *   -   ``tnt_vinyl_memory_tuple``
+            -   Total size of memory in bytes occupied by Vinyl tuples.
+                It includes cached tuples and tuples pinned by the Lua world.
 
 ..  _metrics-reference-vinyl_scheduler:
 
@@ -956,3 +959,37 @@ So ``stories`` and ``retained`` need to be measured separately.
                     Number of retained ``tracking`` tuples / number of stories.
                 *   ``total``
                     Amount of bytes used by retained ``tracking`` tuples.
+
+
+~~~~~~~~~~~~~~~~~~~~
+Read view statistics
+~~~~~~~~~~~~~~~~~~~~
+
+
+..  container:: table
+
+    ..  list-table::
+        :widths: 25 75
+        :header-rows: 0
+
+        *   -   ``tnt_memtx_tuples_data_total``
+            -   Total amount of memory (in bytes) allocated for data tuples.
+                This includes ``tnt_memtx_tuples_data_read_view`` and
+                ``tnt_memtx_tuples_data_garbage`` metric values plus tuples that
+                are actually stored in memtx spaces.
+
+        *   -   ``tnt_memtx_tuples_data_read_view``
+            -   Memory (in bytes) held for read views.
+
+        *   -   ``tnt_memtx_tuples_data_garbage``
+            -   Memory (in bytes) that is unused and scheduled to be freed
+                (freed lazily on memory allocation).
+
+        *   -   ``tnt_memtx_index_total``
+            -   Total amount of memory (in bytes) allocated for indexing data.
+                This includes ``tnt_memtx_index_read_view`` metric value
+                plus memory used for indexing tuples
+                that are actually stored in memtx spaces.
+
+        *   -   ``tnt_memtx_index_read_view``
+            -   Memory (in bytes) held for read views.

metrics/tarantool/memtx.lua

@@ -122,6 +122,36 @@ local function update()
             'Number of retained `read_view` tuples / number of stories',
             memtx_stat.mvcc.tuples.read_view.retained.total, {kind = "total"}, nil, {default = true})
 
+    -- Tarantool 3.0 memory statistics
+
+    local ok, memtx_stat_3 = pcall(box.stat.memtx)
+    if not ok or memtx_stat_3.data == nil or memtx_stat_3.index == nil then
+        return
+    end
+
+    collectors_list.memtx_tuples_data_total =
+        utils.set_gauge('memtx_tuples_data_total',
+            'Total amount of memory allocated for data tuples',
+            memtx_stat_3.data.total, nil, nil, {default = true})
+    collectors_list.memtx_tuples_data_read_view =
+        utils.set_gauge('memtx_tuples_data_read_view',
+            'Memory held for read views',
+            memtx_stat_3.data.read_view, nil, nil, {default = true})
+    collectors_list.memtx_tuples_data_garbage =
+        utils.set_gauge('memtx_tuples_data_garbage',
+            'Memory that is unused and scheduled to be freed',
+            memtx_stat_3.data.garbage, nil, nil, {default = true})
+
+
+    collectors_list.memtx_index_total =
+        utils.set_gauge('memtx_index_total',
+            'Total amount of memory allocated for indexing data',
+            memtx_stat_3.index.total, nil, nil, {default = true})
+    collectors_list.memtx_index_read_view =
+        utils.set_gauge('memtx_index_read_view',
+            'Memory held for read views',
+            memtx_stat_3.index.read_view, nil, nil, {default = true})
+
 end
 
 return {

metrics/tarantool/vinyl.lua

@@ -60,6 +60,13 @@ local function update()
         utils.set_gauge('vinyl_memory_bloom_filter', 'Size of bloom filter',
             vinyl_stat.memory.bloom_filter, nil, nil, {default = true})
 
+    if vinyl_stat.memory.tuple ~= nil then
+        collectors_list.vinyl_memory_tuple =
+            utils.set_gauge('vinyl_memory_tuple',
+                'Total size of memory in bytes occupied by Vinyl tuples',
+                vinyl_stat.memory.tuple, nil, nil, {default = true})
+    end
+
     collectors_list.vinyl_scheduler_tasks =
         utils.set_gauge('vinyl_scheduler_tasks', 'Vinyl tasks count',
             vinyl_stat.scheduler.tasks_inprogress, {status = 'inprogress'}, nil, {default = true})

test/tarantool/memtx_metrics_test.lua

@@ -63,3 +63,34 @@ g.test_memtx = function(cg)
         end
     end)
 end
+
+g.test_memtx_read_view = function(cg)
+    t.skip_if(utils.is_version_less(_TARANTOOL, '3.1.0'),
+        'Tarantool version is must be v3.1.0 or greater')
+
+    cg.server:exec(function()
+        local metrics = require('metrics')
+        local memtx = require('metrics.tarantool.memtx')
+        local utils = require('test.utils') -- luacheck: ignore 431
+
+        metrics.enable_default_metrics()
+        memtx.update()
+        local default_metrics = metrics.collect()
+        local log = require('log')
+
+        local metrics_list = {
+            'tnt_memtx_tuples_data_total',
+            'tnt_memtx_tuples_data_read_view',
+            'tnt_memtx_tuples_data_garbage',
+            'tnt_memtx_index_total',
+            'tnt_memtx_index_read_view',
+        }
+
+        for _, item in ipairs(metrics_list) do
+            log.info('checking metric: ' .. item)
+            local metric = utils.find_metric(item, default_metrics)
+            t.assert(metric)
+            t.assert_type(metric[1].value, 'number')
+        end
+    end)
+end

test/tarantool/vinyl_test.lua

@@ -31,8 +31,10 @@ g.test_vinyl_metrics_present = function(cg)
         if utils.is_version_less(_TARANTOOL, '2.8.3')
         and utils.is_version_greater(_TARANTOOL, '2.0.0') then
             t.assert_equals(metrics_cnt, 19)
-        else
+        elseif utils.is_version_less(_TARANTOOL, '3.0.0') then
             t.assert_equals(metrics_cnt, 20)
+        else
+            t.assert_equals(metrics_cnt, 21)
         end
     end)
 end