From 32f3192428b6cba9c3374441a14744b35e53b567 Mon Sep 17 00:00:00 2001 From: Coen van Leeuwen Date: Tue, 8 Sep 2020 14:34:02 +0200 Subject: [PATCH 1/5] Fix spaces in measurement name or tags --- influxdb-client/Series.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/influxdb-client/Series.m b/influxdb-client/Series.m index 1a0895a..287e8c1 100644 --- a/influxdb-client/Series.m +++ b/influxdb-client/Series.m @@ -99,7 +99,8 @@ end % Create a line for each sample - prefix = [strjoin([{obj.Name}, obj.Tags], ','), ' ']; + prefix = strjoin([{obj.Name}, obj.Tags], ','); + prefix = [strrep(prefix,' ','\ ') ' ']; builder = ''; for i = 1:field_lengths values = ''; From aa6f25f84f24896bdef3904e7155e807a085c9a3 Mon Sep 17 00:00:00 2001 From: Coen van Leeuwen Date: Mon, 5 Oct 2020 14:58:59 +0200 Subject: [PATCH 2/5] Also escape other special characters --- influxdb-client/Series.m | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/influxdb-client/Series.m b/influxdb-client/Series.m index 287e8c1..a5b3482 100644 --- a/influxdb-client/Series.m +++ b/influxdb-client/Series.m @@ -15,7 +15,13 @@ % Add a tag function obj = tag(obj, key, value) - obj.Tags{end + 1} = [key '=' value]; + if isnumeric(value) || islogical(value) + obj.Tags{end + 1} = [Series.safeKey(key) '=' value]; + elseif ischar(value) + obj.Tags{end + 1} = [Series.safeKey(key) '=' Series.safeKey(value)]; + else + error('unsupported tag type'); + end end % Add multiple tags at once @@ -99,14 +105,15 @@ end % Create a line for each sample - prefix = strjoin([{obj.Name}, obj.Tags], ','); - prefix = [strrep(prefix,' ','\ ') ' ']; + measurement = Series.safeMeasurement(obj.Name); + prefix = [strjoin([{measurement}, obj.Tags], ',') ' ']; + builder = ''; for i = 1:field_lengths values = ''; for f = 1:length(obj.Fields) field = obj.Fields{f}; - name = field.key; + name = Series.safeKey(field.key); value = field.value; if iscell(value) str = obj.fieldFmt(name, value{i}); @@ -143,13 +150,27 @@ elseif isinteger(value) str = sprintf('%s=%ii', key, value); elseif ischar(value) - str = [key '="' value '"']; + str = [key '="' Series.safeValue(value) '"']; elseif islogical(value) str = [key '=' iif(value, 'true', 'false')]; else error('unsupported value type'); end end + + % The following functions escape special characters according to: + % https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_reference/#special-characters + function safe = safeValue(value) + safe = regexprep(value, '["\\]', '\\$0'); + end + + function safe = safeKey(key) + safe = regexprep(key, '[,= ]', '\\$0'); + end + + function safe = safeMeasurement(name) + safe = regexprep(name, '[, ]', '\\$0'); + end end end From 4f9acc4b09920814ffac776ebf6ca55e6cb2fddd Mon Sep 17 00:00:00 2001 From: Coen van Leeuwen Date: Mon, 5 Oct 2020 14:59:13 +0200 Subject: [PATCH 3/5] Tests for special characters --- tests/SeriesTest.m | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/SeriesTest.m b/tests/SeriesTest.m index b74303f..8381314 100644 --- a/tests/SeriesTest.m +++ b/tests/SeriesTest.m @@ -308,6 +308,31 @@ function imports_time_and_fields_from_timetable(test) 'weather temperature=-3.5,wind_direction="west" 1529933581618']; test.verifyEqual(s.toLine(), exp); end + + function measurements_with_spaces_and_commas(test) + s = Series('Hello, world!') ... + .fields('value', 42); + + expected = 'Hello\,\ world! value=42'; + test.verifyEqual(s.toLine(), expected); + end + + function tags_with_commas_spaces_and_equals(test) + s = Series('Series_A') ... + .tags('Annoying, tag = annoying', 'comma="evil", am i right?') ... + .fields('value', 42); + + expected = 'Series_A,Annoying\,\ tag\ \=\ annoying=comma\="evil"\,\ am\ i\ right? value=42'; + test.verifyEqual(s.toLine(), expected); + end + + function fields_with_commas_equals_spaces_quotes_and_slashes(test) + s = Series('JSON') ... + .fields('strange, json=string', '{"w": "\/\/"}'); + + expected = 'JSON strange\,\ json\=string="{\"w\": \"\\/\\/\"}"'; + test.verifyEqual(s.toLine(), expected); + end end end From 4719544e4c6a7f75c69b8d32202a939bb9b564b8 Mon Sep 17 00:00:00 2001 From: Coen van Leeuwen Date: Thu, 30 Sep 2021 09:14:33 +0200 Subject: [PATCH 4/5] Reuse generic formatting code for tags --- influxdb-client/Series.m | 53 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/influxdb-client/Series.m b/influxdb-client/Series.m index a5b3482..e3d2bcb 100644 --- a/influxdb-client/Series.m +++ b/influxdb-client/Series.m @@ -15,13 +15,7 @@ % Add a tag function obj = tag(obj, key, value) - if isnumeric(value) || islogical(value) - obj.Tags{end + 1} = [Series.safeKey(key) '=' value]; - elseif ischar(value) - obj.Tags{end + 1} = [Series.safeKey(key) '=' Series.safeKey(value)]; - else - error('unsupported tag type'); - end + obj.Tags{end + 1} = Series.tagFmt(key, value); end % Add multiple tags at once @@ -108,17 +102,17 @@ measurement = Series.safeMeasurement(obj.Name); prefix = [strjoin([{measurement}, obj.Tags], ',') ' ']; - builder = ''; + lines = cell(100); + n = 1; for i = 1:field_lengths values = ''; for f = 1:length(obj.Fields) field = obj.Fields{f}; - name = Series.safeKey(field.key); value = field.value; if iscell(value) - str = obj.fieldFmt(name, value{i}); + str = Series.fieldFmt(field.key, value{i}); else - str = obj.fieldFmt(name, value(i)); + str = Series.fieldFmt(field.key, value(i)); end if ~isempty(str) values = [values, str, ',']; @@ -128,31 +122,52 @@ values = values(1:end-1); if time_length > 0 time = sprintf(' %i', timestamp(i)); - builder = [builder, prefix, values, time, newline]; + lines{n} = [prefix, values, time]; else - builder = [builder, prefix, values, newline]; + lines{n} = [prefix, values]; + end + n = n + 1; + if (n > numel(lines)) + lines = [lines cell(100)]; end end end - lines = iif(isempty(builder), '', builder(1:end-1)); + lines = lines(1:(n-1)); end end methods(Static, Access = private) % Format a field function str = fieldFmt(key, value) + if ischar(value) + str = [Series.safeKey(key) '="' Series.safeValue(value) '"']; + else + str = Series.genericFmt(key, value); + end + end + + % Format a tag + function str = tagFmt(key, value) + if ischar(value) + str = [Series.safeKey(key) '=' Series.safeKey(value)]; + else + str = Series.genericFmt(key, value); + end + end + + % Generic formatting (field or tag) + function str = genericFmt(key, value) + safeKey = Series.safeKey(key); if isfloat(value) if ~isempty(value) && isfinite(value) - str = sprintf('%s=%.8g', key, value); + str = sprintf('%s=%.8g', safeKey, value); else str = ''; end elseif isinteger(value) - str = sprintf('%s=%ii', key, value); - elseif ischar(value) - str = [key '="' Series.safeValue(value) '"']; + str = sprintf('%s=%ii', safeKey, value); elseif islogical(value) - str = [key '=' iif(value, 'true', 'false')]; + str = [safeKey '=' iif(value, 'true', 'false')]; else error('unsupported value type'); end From 7a9eb4251030f4f24ed3c51cfe449ab07f6018b8 Mon Sep 17 00:00:00 2001 From: Coen van Leeuwen Date: Thu, 30 Sep 2021 09:14:50 +0200 Subject: [PATCH 5/5] Batch write measurements --- influxdb-client/InfluxDB.m | 11 ++++++++++- influxdb-client/WriteBuilder.m | 10 +++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/influxdb-client/InfluxDB.m b/influxdb-client/InfluxDB.m index e3c42fc..442faa5 100644 --- a/influxdb-client/InfluxDB.m +++ b/influxdb-client/InfluxDB.m @@ -107,7 +107,16 @@ url = [obj.Url '/write?' strjoin(params, '&')]; opts = weboptions('Timeout', obj.WriteTimeout, ... 'Username', obj.User, 'Password', obj.Password); - webwrite(url, lines, opts); + + if iscell(lines) + for from = 1:5000:numel(lines) + to = min(from+4999, numel(lines)); + part = strjoin(lines(from:to), '\n'); + webwrite(url, part, opts); + end + else + webwrite(url, lines, opts); + end end % Obtain a write builder diff --git a/influxdb-client/WriteBuilder.m b/influxdb-client/WriteBuilder.m index cda9337..0f070f1 100644 --- a/influxdb-client/WriteBuilder.m +++ b/influxdb-client/WriteBuilder.m @@ -47,14 +47,14 @@ % Build line protocol function lines = build(obj) - if isempty(obj.Items) + f = @(x) x.toLine(obj.Precision); + items = cellfun(f, obj.Items, 'UniformOutput', false); + nonempty = ~cellfun(@isempty, items); + if ~any(nonempty) warning('this writer is empty'); lines = ''; else - f = @(x) x.toLine(obj.Precision); - items = cellfun(f, obj.Items, 'UniformOutput', false); - nonempty = ~cellfun(@isempty, items); - lines = strjoin(items(nonempty), newline); + lines = [items{nonempty}]; end end