diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ee47e7..d239fa67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `metrics.cfg{}` `"all"` metasection for array `include` and `exclude` (`metrics.cfg{include={'all'}}` can be used instead of `metrics.cfg{include='all'}`, `metrics.cfg{exclude={'all'}}` can be used instead of `metrics.cfg{include='none'}`) +- Allow users to decide on labels serialization scheme themselves. Add `labels_serializer`, which provides alternative efficient labels serialization scheme. ## [1.0.0] - 2023-05-22 ### Changed diff --git a/metrics/api.lua b/metrics/api.lua index d36f1c9a..b9407fd3 100644 --- a/metrics/api.lua +++ b/metrics/api.lua @@ -9,6 +9,8 @@ local Gauge = require('metrics.collectors.gauge') local Histogram = require('metrics.collectors.histogram') local Summary = require('metrics.collectors.summary') +local serializers = require("metrics.serializers") + local registry = rawget(_G, '__metrics_registry') if not registry then registry = Registry.new() @@ -140,4 +142,5 @@ return { unregister_callback = unregister_callback, invoke_callbacks = invoke_callbacks, set_global_labels = set_global_labels, + labels_serializer = serializers.basic_labels_serializer } diff --git a/metrics/collectors/shared.lua b/metrics/collectors/shared.lua index 2cb195b4..1abe5600 100644 --- a/metrics/collectors/shared.lua +++ b/metrics/collectors/shared.lua @@ -1,6 +1,7 @@ local clock = require('clock') local fiber = require('fiber') local log = require('log') +local serializers = require("metrics.serializers") local Shared = {} @@ -47,12 +48,11 @@ function Shared.make_key(label_pairs) if type(label_pairs) ~= 'table' then return "" end - local parts = {} - for k, v in pairs(label_pairs) do - table.insert(parts, k .. '\t' .. v) + -- `label_pairs` provides its own serialization scheme, it must be used instead of default one. + if label_pairs.__metrics_serialize then + return label_pairs:__metrics_serialize() end - table.sort(parts) - return table.concat(parts, '\t') + return serializers.default_labels_serializer(label_pairs) end function Shared:remove(label_pairs) diff --git a/metrics/http_middleware.lua b/metrics/http_middleware.lua index cd8655fc..b873e83a 100644 --- a/metrics/http_middleware.lua +++ b/metrics/http_middleware.lua @@ -69,6 +69,8 @@ function export.configure_default_collector(...) export.set_default_collector(export.build_default_collector(...)) end +local labels_serializer = metrics_api.labels_serializer({ "path", "method", "status" }) + --- Measure latency and invoke collector with labels from given route -- -- @tab collector @@ -86,11 +88,11 @@ function export.observe(collector, route, handler, ...) error(('incorrect http handler for %s %s: expecting return response object'): format(route.method, route.path), 0) end - return { + return labels_serializer.wrap({ path = route.path, method = route.method, status = (not ok and 500) or result.status or 200, - } + }) end, handler, ...) end diff --git a/metrics/init.lua b/metrics/init.lua index 9adb6662..a511e8cf 100644 --- a/metrics/init.lua +++ b/metrics/init.lua @@ -27,6 +27,7 @@ return setmetatable({ unregister_callback = api.unregister_callback, invoke_callbacks = api.invoke_callbacks, set_global_labels = api.set_global_labels, + labels_serializer = api.labels_serializer, enable_default_metrics = tarantool.enable, cfg = cfg.cfg, http_middleware = http_middleware, diff --git a/metrics/serializers.lua b/metrics/serializers.lua new file mode 100644 index 00000000..10505da5 --- /dev/null +++ b/metrics/serializers.lua @@ -0,0 +1,86 @@ +--- Default slow algorithm, polluted with sorting. +--- It is used when nothing is known about `label_pairs`. +local function default_labels_serializer(label_pairs) + local parts = {} + for k, v in pairs(label_pairs) do + table.insert(parts, k .. '\t' .. v) + end + table.sort(parts) + return table.concat(parts, '\t') +end + + +--- Prepares a serializer for label pairs with given keys. +--- +--- `make_key`, which is used during every metric-related operation, is not very efficient itself. +--- To mitigate it, one could add his own serialization implementation. +--- It is done via passing `__metrics_serialize` callback to the label pairs table. +--- +--- This function gives you ready-to-use serializer, so you don't have to create one yourself. +--- +--- BEWARE! If keys of the `label_pairs` somehow change between serialization turns, it would raise error mostlikely. +--- We cover internal cases already, for example, "le" key is always added for the histograms. +--- +--- @class LabelsSerializer +--- @field wrap function(label_pairs: table): table Wraps given `label_pairs` with an efficient serialization. +--- @field serialize function(label_pairs: table): string Serialize given `label_pairs` into the key. +--- Exposed so you can write your own serializers on top of it. +--- +--- @param labels_keys string[] Label keys for the further use. +--- @return LabelsSerializer +local function basic_labels_serializer(labels_keys) + -- we always add keys that are needed for metrics' internals. + local __labels_keys = { "le" } + -- used to protect label_pairs from altering with unexpected keys. + local keys_index = { le = true } + + -- keep only unique labels + for _, key in ipairs(labels_keys) do + if not keys_index[key] then + table.insert(__labels_keys, key) + keys_index[key] = true + end + end + table.sort(__labels_keys) + + local function serialize(label_pairs) + local result = "" + for _, label in ipairs(__labels_keys) do + local value = label_pairs[label] + if value ~= nil then + if result ~= "" then + result = result .. '\t' + end + result = result .. label .. '\t' .. value + end + end + return result + end + + local pairs_metatable = { + __index = { + __metrics_serialize = function(self) + return serialize(self) + end + }, + -- It protects pairs from being altered with unexpected labels. + __newindex = function(table, key, value) + if not keys_index[key] then + error(('Label "%s" is unexpected'):format(key), 2) + end + rawset(table, key, value) + end + } + + return { + wrap = function(label_pairs) + return setmetatable(label_pairs, pairs_metatable) + end, + serialize = serialize + } +end + +return { + default_labels_serializer = default_labels_serializer, + basic_labels_serializer = basic_labels_serializer +} diff --git a/test/metrics_test.lua b/test/metrics_test.lua index 1b373305..2029c99b 100755 --- a/test/metrics_test.lua +++ b/test/metrics_test.lua @@ -269,3 +269,41 @@ g.test_deprecated_version = function(cg) "use require%('metrics'%)._VERSION instead." t.assert_not_equals(cg.server:grep_log(warn), nil) end + +g.test_labels_serializer_consistent = function() + local shared = require("metrics.collectors.shared") + + local label_pairs = { + abc = 123, + cba = "123", + cda = 0, + eda = -1, + acb = 456 + } + local label_keys = {} + for key, _ in pairs(label_pairs) do + table.insert(label_keys, key) + end + + local serializer = metrics.labels_serializer(label_keys) + local actual = serializer.serialize(label_pairs) + local wrapped = serializer.wrap(table.copy(label_pairs)) + + t.assert_equals(actual, shared.make_key(label_pairs)) + t.assert_equals(actual, shared.make_key(wrapped)) + t.assert_not_equals(wrapped.__metrics_serialize, nil) + + -- trying to set unexpected label. + t.assert_error_msg_contains('Label "new_label" is unexpected', function() wrapped.new_label = "123456" end) + + -- check that builtin metrics work. + local hist_serializer = metrics.labels_serializer(label_keys) + local hist = metrics.histogram('hist', 'test histogram', {2}) + + hist:observe(3, hist_serializer.wrap(table.copy(label_pairs))) + local state = table.deepcopy(hist) + hist:observe(3, label_pairs) + + t.assert_equals(hist.observations, state.observations) + t.assert_equals(hist.label_pairs, state.label_pairs) +end