From 284fe50d9bcff1b8685e47bc3c70ad3eb23a8085 Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Fri, 5 Jan 2024 15:17:42 +0000 Subject: [PATCH 1/7] numlist: remove unused component There seems to be no use for the numlist object --- Makefile | 1 - numlist.c | 189 ------------------------------------------------------ numlist.h | 40 ------------ 3 files changed, 230 deletions(-) delete mode 100644 numlist.c delete mode 100644 numlist.h diff --git a/Makefile b/Makefile index c3b6041..2ce021a 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ lib := \ histo.o \ logging.o \ loop.o \ - numlist.o \ or_die.o \ parse.o \ percentiles.o \ diff --git a/numlist.c b/numlist.c deleted file mode 100644 index 194a87c..0000000 --- a/numlist.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "numlist.h" -#include -#include -#include -#include -#include "lib.h" -#include "logging.h" - -#define MEMBLOCK_SIZE 500 - -struct memblock { - size_t size; - struct memblock *next; - double data[MEMBLOCK_SIZE]; -}; - -struct numlist { - struct callbacks *cb; - struct memblock *head; -}; - -static void prepend_memblock(struct numlist *lst) -{ - struct memblock *blk; - - blk = malloc(sizeof(struct memblock)); - if (!blk) - PLOG_FATAL(lst->cb, "unable to allocate memblock"); - blk->size = 0; - blk->next = lst->head; - lst->head = blk; -} - -struct numlist *numlist_create(struct callbacks *cb) -{ - struct numlist *lst; - - lst = malloc(sizeof(struct numlist)); - if (!lst) - PLOG_FATAL(cb, "unable to allocate numlist"); - lst->cb = cb; - lst->head = NULL; - prepend_memblock(lst); - return lst; -} - -void numlist_destroy(struct numlist *lst) -{ - struct memblock *block, *next; - - block = lst->head; - while (block) { - next = block->next; - free(block); - block = next; - } - free(lst); -} - -void numlist_add(struct numlist *lst, double val) -{ - if (lst->head->size == MEMBLOCK_SIZE) - prepend_memblock(lst); - lst->head->data[lst->head->size++] = val; -} - -void numlist_concat(struct numlist *lst, struct numlist *tail) -{ - struct memblock *blk = lst->head; - - while (blk->next) - blk = blk->next; - blk->next = tail->head; - tail->head = NULL; -} - -#define for_each_memblock(blk, lst) \ - for (blk = (lst)->head; blk; blk = blk->next) - -#define for_each_number(n, blk) \ - for (n = (blk)->data; n < (blk)->data + blk->size; n++) - -#define for_each(n, blk, lst) \ - for_each_memblock(blk, lst) for_each_number(n, blk) - -size_t numlist_size(struct numlist *lst) -{ - struct memblock *blk; - size_t size = 0; - double *n; - - for_each(n, blk, lst) - size++; - return size; -} - -double numlist_min(struct numlist *lst) -{ - double min = INFINITY, *n; - struct memblock *blk; - - for_each(n, blk, lst) { - if (*n < min) - min = *n; - } - return min; -} - -double numlist_max(struct numlist *lst) -{ - double max = -INFINITY, *n; - struct memblock *blk; - - for_each(n, blk, lst) { - if (*n > max) - max = *n; - } - return max; -} - -double numlist_mean(struct numlist *lst) -{ - double sum = 0, cnt = 0, *n; - struct memblock *blk; - - for_each(n, blk, lst) { - sum += *n; - cnt++; - } - return sum / cnt; -} - -double numlist_stddev(struct numlist *lst) -{ - double sum = 0, cnt = 0, mean, *n; - struct memblock *blk; - - mean = numlist_mean(lst); - for_each(n, blk, lst) { - sum += (*n - mean) * (*n - mean); - cnt++; - } - return sqrt(sum / cnt); -} - -static int compare_doubles(const void *a, const void *b) -{ - const double x = *(const double *)a, y = *(const double *)b; - - if (x < y) - return -1; - if (x > y) - return 1; - return 0; -} - -double numlist_percentile(struct numlist *lst, int percentile) -{ - double *values, *n, result; - struct memblock *blk; - size_t size, i = 0; - - size = numlist_size(lst); - if (size == 0) - return NAN; - values = malloc(sizeof(double) * size); - for_each(n, blk, lst) - values[i++] = *n; - qsort(values, size, sizeof(double), compare_doubles); - result = values[(size - 1) * percentile / 100]; - free(values); - return result; -} diff --git a/numlist.h b/numlist.h deleted file mode 100644 index ed47b26..0000000 --- a/numlist.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef NEPER_NUMLIST_H -#define NEPER_NUMLIST_H - -#include - -struct callbacks; -struct numlist; - -struct numlist *numlist_create(struct callbacks *cb); -void numlist_destroy(struct numlist *lst); -void numlist_add(struct numlist *lst, double val); -/** - * The numbers in @tail are all moved to @lst. - * @tail will become empty after this operation. - */ -void numlist_concat(struct numlist *lst, struct numlist *tail); -size_t numlist_size(struct numlist *lst); -double numlist_min(struct numlist *lst); -double numlist_max(struct numlist *lst); -double numlist_mean(struct numlist *lst); -double numlist_stddev(struct numlist *lst); -double numlist_percentile(struct numlist *lst, int percentile); - -#endif From 313f5e7e446b9b5ad6d9a10c46dc6b5932eed145 Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Thu, 4 Jan 2024 07:41:25 +0000 Subject: [PATCH 2/7] rr: remove incorrect division by MILLION csv printing percentiles in CSV file were incorrectly divided by MILLION, resulting in mostly 0 values. Remove the divisor. Probably the feature was never used, otherwise it would have been noticed. --- rr.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rr.c b/rr.c index 3a7483c..4b35722 100644 --- a/rr.c +++ b/rr.c @@ -39,8 +39,6 @@ #define NEPER_EPOLL_MASK (EPOLLHUP | EPOLLRDHUP | EPOLLERR) -static const int MILLION = 1000000; - typedef ssize_t (*rr_send_t)(struct flow *, const char *, size_t, int); typedef ssize_t (*rr_recv_t)(struct flow *, char *, size_t); @@ -480,8 +478,7 @@ static void rr_print_snap(struct thread *t, int flow_index, const struct rr_snap_opaque *rso = (void *)&snap->opaque; fprintf(csv, ",%f,%f,%f,%f", - rso->min / MILLION, rso->mean / MILLION, - rso->max / MILLION, rso->stddev / MILLION); + rso->min, rso->mean, rso->max, rso->stddev); if (t->percentiles) { const struct options *opts = t->opts; @@ -490,7 +487,7 @@ static void rr_print_snap(struct thread *t, int flow_index, for (i = 0; i < PER_INDEX_COUNT; i++) if (percentiles_chosen(&opts->percentiles, i)) fprintf(csv, ",%f", - rso->percentile[j++] / MILLION); + rso->percentile[j++]); } fprintf(csv, "\n"); From 38d1ff1c45d48cee0aa991daf9c3373cfa40ce2d Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Thu, 4 Jan 2024 08:28:09 +0000 Subject: [PATCH 3/7] histo: de-virtualize histogram methods. No functional change. histogram methods were implemented as virtual functions, but since there is only one possible implementation this was overkill. Simplify the code by exposing the actual methods. The implementation still remains opaque. No functional changes. Tested with ./tcp_rr -c -H 127.0.0.1 -p1,2,10,50,90,999.9999,100 -A/tmp/x.csv -l 4 and verified that the csv file has the correct data. (histograms are only exercised in rr tests) --- histo.c | 147 +++++++++++++------------------------------------------ histo.h | 71 +++++++++++++++------------ rr.c | 29 ++++++----- stats.c | 2 +- thread.c | 4 +- 5 files changed, 91 insertions(+), 162 deletions(-) diff --git a/histo.c b/histo.c index 0ca302f..d88fd83 100644 --- a/histo.c +++ b/histo.c @@ -23,9 +23,7 @@ // use 0.01 us time resolution static const int TIME_RESOLUTION = 100 * 1000000; -struct histo_impl { - struct neper_histo histo; - +struct neper_histo { const struct thread *thread; int num_buckets; /* # of buckets allocated */ @@ -60,55 +58,26 @@ struct histo_impl { bool first_all; /* Is this the first call to all_percent() */ }; -struct histo_factory_impl { - struct neper_histo_factory factory; - +struct neper_histo_factory { const struct thread *thread; int num_buckets; /* # of buckets allocated */ int *ceil; /* Max value that can be hashed into each bucket */ }; -static double histo_all_min(const struct neper_histo *histo) -{ - const struct histo_impl *impl = (void *)histo; - - return impl->all_min; -} - -static double histo_one_min(const struct neper_histo *histo) -{ - const struct histo_impl *impl = (void *)histo; - - return impl->one_min; -} - -static double histo_all_max(const struct neper_histo *histo) -{ - const struct histo_impl *impl = (void *)histo; - - return impl->all_max; -} - -static double histo_one_max(const struct neper_histo *histo) +double neper_histo_min(const struct neper_histo *histo) { - const struct histo_impl *impl = (void *)histo; - - return impl->one_max; + return histo->one_min; } -static double histo_all_mean(const struct neper_histo *histo) +double neper_histo_max(const struct neper_histo *histo) { - const struct histo_impl *impl = (void *)histo; - - return impl->all_sum / impl->all_count; + return histo->one_max; } -static double histo_one_mean(const struct neper_histo *histo) +double neper_histo_mean(const struct neper_histo *histo) { - const struct histo_impl *impl = (void *)histo; - - return impl->one_sum / impl->one_count; + return histo->one_sum / histo->one_count; } static double histo_stddev(long double N, long double S, long double Q) @@ -116,21 +85,12 @@ static double histo_stddev(long double N, long double S, long double Q) return sqrt(N*Q - S*S) / N; } -static double histo_all_stddev(const struct neper_histo *histo) -{ - struct histo_impl *impl = (void *)histo; - - return histo_stddev(impl->all_count, impl->all_sum, impl->all_sum2); -} - -static double histo_one_stddev(const struct neper_histo *histo) +double neper_histo_stddev(const struct neper_histo *histo) { - struct histo_impl *impl = (void *)histo; - - return histo_stddev(impl->one_count, impl->one_sum, impl->one_sum2); + return histo_stddev(histo->one_count, histo->one_sum, histo->one_sum2); } -static void histo_all_finalize(struct histo_impl *impl) +static void histo_all_finalize(struct neper_histo *impl) { double cent = impl->all_count / 100.0; double nnn = (impl->all_count * 99.9) / 100.0; @@ -164,7 +124,7 @@ static void histo_all_finalize(struct histo_impl *impl) } } -static void histo_one_finalize(struct histo_impl *impl) +static void histo_one_finalize(struct neper_histo *impl) { double cent = impl->one_count / 100.0; double nnn = (impl->one_count * 99.9) / 100.0; @@ -197,10 +157,8 @@ static void histo_one_finalize(struct histo_impl *impl) } } -static double histo_all_percent(struct neper_histo *histo, int percentage) +static double histo_all_percent(struct neper_histo *impl, int percentage) { - struct histo_impl *impl = (void *)histo; - histo_all_finalize(impl); switch (percentage) { @@ -220,10 +178,8 @@ static double histo_all_percent(struct neper_histo *histo, int percentage) } } -static double histo_one_percent(const struct neper_histo *histo, int percentage) +double neper_histo_percent(const struct neper_histo *impl, int percentage) { - struct histo_impl *impl = (void *)histo; - switch (percentage) { case 0: return impl->one_min; @@ -241,18 +197,13 @@ static double histo_one_percent(const struct neper_histo *histo, int percentage) } } -static uint64_t histo_events(const struct neper_histo *histo) +uint64_t neper_histo_samples(const struct neper_histo *histo) { - struct histo_impl *impl = (void *)histo; - - return impl->all_count; + return histo->all_count; } -static void histo_add(struct neper_histo *des, const struct neper_histo *src) +void neper_histo_add(struct neper_histo *desi, const struct neper_histo *srci) { - struct histo_impl *desi = (void *)des; - const struct histo_impl *srci = (void *)src; - desi->cur_count += srci->all_count; desi->cur_sum += srci->all_sum; desi->cur_sum2 += srci->all_sum2; @@ -266,7 +217,7 @@ static void histo_add(struct neper_histo *des, const struct neper_histo *src) } // binary search for the correct bucket index -static int histo_find_bucket_idx(struct histo_impl *impl, int ticks) +static int histo_find_bucket_idx(struct neper_histo *impl, int ticks) { int l_idx = 0; int r_idx = impl->num_buckets - 1; @@ -291,9 +242,8 @@ static int histo_find_bucket_idx(struct histo_impl *impl, int ticks) return -1; } -static void histo_event(struct neper_histo *histo, double delta_s) +void neper_histo_event(struct neper_histo *impl, double delta_s) { - struct histo_impl *impl = (void *)histo; int ticks = delta_s * TIME_RESOLUTION; int i; @@ -314,10 +264,8 @@ static void histo_event(struct neper_histo *histo, double delta_s) impl->cur_buckets[i]++; } -static void histo_epoch(struct neper_histo *histo) +void neper_histo_epoch(struct neper_histo *impl) { - struct histo_impl *impl = (void *)histo; - impl->all_count += impl->cur_count; impl->one_count = impl->cur_count; impl->cur_count = 0; @@ -370,16 +318,16 @@ static void histo_hash(int num_buckets, double growth, int *ceils) } } -static void histo_print(struct neper_histo *histo) +void neper_histo_print(struct neper_histo *histo) { - struct histo_impl *impl = (void *)histo; - const struct thread *t = impl->thread; + const struct thread *t = histo->thread; const struct options *opts = t->opts; - PRINT(t->cb, "latency_min", "%.9f", histo_all_min(histo)); - PRINT(t->cb, "latency_max", "%.9f", histo_all_max(histo)); - PRINT(t->cb, "latency_mean", "%.9f", histo_all_mean(histo)); - PRINT(t->cb, "latency_stddev", "%.9f", histo_all_stddev(histo)); + PRINT(t->cb, "latency_min", "%.9f", histo->all_min); + PRINT(t->cb, "latency_max", "%.9f", histo->all_max); + PRINT(t->cb, "latency_mean", "%.9f", histo->all_sum / histo->all_count); + PRINT(t->cb, "latency_stddev", "%.9f", + histo_stddev(histo->all_count, histo->all_sum, histo->all_sum2)); int i; for (i = 0; i < 100; i++) @@ -396,10 +344,8 @@ static void histo_print(struct neper_histo *histo) histo_all_percent(histo, PER_INDEX_99_99)); } -static void histo_fini(struct neper_histo *histo) +void neper_histo_delete(struct neper_histo *impl) { - struct histo_impl *impl = (void *)histo; - if (impl) { free(impl->all_buckets); free(impl->cur_buckets); @@ -408,26 +354,10 @@ static void histo_fini(struct neper_histo *histo) } } -static struct neper_histo *neper_histo_factory_create( - const struct neper_histo_factory *factory) +struct neper_histo *neper_histo_new( const struct neper_histo_factory *fimpl) { - const struct histo_factory_impl *fimpl = (void *)factory; - struct histo_impl *impl = calloc(1, sizeof(struct histo_impl)); - struct neper_histo *histo = &impl->histo; - - histo->min = histo_one_min; - histo->max = histo_one_max; - histo->mean = histo_one_mean; - histo->stddev = histo_one_stddev; - histo->percent = histo_one_percent; - histo->events = histo_events; - - histo->add = histo_add; - histo->event = histo_event; - histo->epoch = histo_epoch; - histo->print = histo_print; - histo->fini = histo_fini; + struct neper_histo *impl = calloc(1, sizeof(*impl)); impl->thread = fimpl->thread; impl->num_buckets = fimpl->num_buckets; @@ -441,28 +371,21 @@ static struct neper_histo *neper_histo_factory_create( impl->first_all = true; - return histo; + return impl; } -void neper_histo_factory_fini(struct neper_histo_factory *factory) +void neper_histo_factory_delete(struct neper_histo_factory *impl) { - struct histo_factory_impl *impl = (void *)factory; - if (impl) { free(impl->ceil); free(impl); } } -struct neper_histo_factory *neper_histo_factory(const struct thread *t, +struct neper_histo_factory *neper_histo_factory_new(const struct thread *t, int num_buckets, double growth) { - struct histo_factory_impl *impl = - calloc(1, sizeof(struct histo_factory_impl)); - struct neper_histo_factory *factory = &impl->factory; - - factory->create = neper_histo_factory_create; - factory->fini = neper_histo_factory_fini; + struct neper_histo_factory *impl = calloc(1, sizeof(*impl)); impl->thread = t; impl->num_buckets = num_buckets; @@ -470,5 +393,5 @@ struct neper_histo_factory *neper_histo_factory(const struct thread *t, histo_hash(num_buckets, growth, impl->ceil); - return factory; + return impl; } diff --git a/histo.h b/histo.h index 571d80d..2b78d5e 100644 --- a/histo.h +++ b/histo.h @@ -23,54 +23,61 @@ struct thread; /* * A simple histogram API for tracking a series of latency measurements: - * - * min() Returns the min of the previous sampling epoch. - * max() Returns the max of the previous sampling epoch. - * mean() Returns the mean of the previous sampling epoch. - * stddev() Returns the stddev of the previous sampling epoch. - * percent() Returns the percent of the previous sampling epoch. - * add() Adds one histogram to the current epoch of another. - * event() Adds a new event to the current sampling epoch. - * events() Returns the event total across all sampling epochs. - * epoch() Commits the current sample set and begins a new one. - * print() Prints the results. - * fini() Deallocates the object. * * An 'event' is a single measurement. * An 'epoch' is all events collected within some time interval. * - * So, typical usage is to call histo->event() many times and histo->epoch() - * perhaps every second or so. + * Typical usage is to call neper_histo_event() many times and + * neper_histo_epoch() perhaps every second or so. */ -struct neper_histo { - uint64_t (*events)(const struct neper_histo *); +struct neper_histo_factory; +struct neper_histo; - double (*min)(const struct neper_histo *); - double (*max)(const struct neper_histo *); - double (*mean)(const struct neper_histo *); - double (*stddev)(const struct neper_histo *); - double (*percent)(const struct neper_histo *, int percentage); +/* Create a new collector */ +struct neper_histo *neper_histo_new(const struct neper_histo_factory *); - void (*add)(struct neper_histo *des, const struct neper_histo *src); +/* Returns the min of the previous sampling epoch. */ +double neper_histo_min(const struct neper_histo *); - void (*event)(struct neper_histo *, double delta_s); - void (*epoch)(struct neper_histo *); - void (*print)(struct neper_histo *); - void (*fini)(struct neper_histo *); -}; +/* Returns the max of the previous sampling epoch. */ +double neper_histo_max(const struct neper_histo *); + +/* Returns the mean of the previous sampling epoch. */ +double neper_histo_mean(const struct neper_histo *); + +/* Returns the stddev of the previous sampling epoch. */ +double neper_histo_stddev(const struct neper_histo *); + +/* Returns the percent of the previous sampling epoch. */ +double neper_histo_percent(const struct neper_histo *, int percentage); + +/* Adds one histogram to the current epoch of another. */ +void neper_histo_add(struct neper_histo *des, const struct neper_histo *src); + +/* Adds a new event to the current sampling epoch. */ +void neper_histo_event(struct neper_histo *, double delta_s); + +/* Returns the event total across all sampling epochs. */ +uint64_t neper_histo_samples(const struct neper_histo *); + +/* Commits the current sample set and begins a new one. */ +void neper_histo_epoch(struct neper_histo *); + +/* Prints the results */ +void neper_histo_print(struct neper_histo *); + +/* Destroy the object */ +void neper_histo_delete(struct neper_histo *); /* * We use a factory to create histo objects so they can all share one set of * common lookup tables, saving a great deal of memory. */ -struct neper_histo_factory { - struct neper_histo *(*create)(const struct neper_histo_factory *); - void (*fini)(struct neper_histo_factory *); -}; +void neper_histo_factory_delete(struct neper_histo_factory *); -struct neper_histo_factory *neper_histo_factory(const struct thread *, +struct neper_histo_factory *neper_histo_factory_new(const struct thread *, int size, double growth); diff --git a/rr.c b/rr.c index 4b35722..00261e8 100644 --- a/rr.c +++ b/rr.c @@ -144,7 +144,7 @@ static struct neper_stat *rr_latency_init(struct flow *f) if (t->opts->nostats) return NULL; - struct neper_histo *histo = t->histo_factory->create(t->histo_factory); + struct neper_histo *histo = neper_histo_new(t->histo_factory); size = sizeof(struct rr_snap_opaque) + t->percentiles * sizeof(double); @@ -298,21 +298,21 @@ static void rr_snapshot(struct thread *t, struct neper_stat *stat, { struct neper_histo *histo = stat->histo(stat); - histo->epoch(histo); + neper_histo_epoch(histo); struct rr_snap_opaque *opaque = (void *)&snap->opaque; - opaque->min = histo->min(histo); - opaque->max = histo->max(histo); - opaque->mean = histo->mean(histo); - opaque->stddev = histo->stddev(histo); + opaque->min = neper_histo_min(histo); + opaque->max = neper_histo_max(histo); + opaque->mean = neper_histo_mean(histo); + opaque->stddev = neper_histo_stddev(histo); if (t->percentiles) { int i, j = 0; for (i = 0; i < PER_INDEX_COUNT; i++) if (percentiles_chosen(&t->opts->percentiles, i)) opaque->percentile[j++] = - histo->percent(histo, i); + neper_histo_percent(histo, i); } } @@ -326,7 +326,7 @@ static bool rr_do_compl(struct flow *f, struct neper_stat *stat = flow_stat(f); struct neper_histo *histo = stat->histo(stat); - histo->event(histo, elapsed); + neper_histo_event(histo, elapsed); if (t->data_pending) { /* data vs time mode, last rr? */ @@ -411,7 +411,7 @@ static void rr_server_state_2(struct flow *f, uint32_t events) if (rr_do_send(f, events, rr->rr_send)) { if (stat) { /* rr server has no meaningful latency to measure. */ - histo->event(histo, 0.0); + neper_histo_event(histo, 0.0); stat->event(t, stat, 1, false, rr_snapshot); } flow_mod(f, rr_server_state_0, EPOLLIN, false); @@ -499,7 +499,7 @@ fn_add(struct neper_stat *stat, void *ptr) { struct neper_histo *src = stat->histo(stat); struct neper_histo *des = ptr; - des->add(des, src); + neper_histo_add(des, src); return 0; } @@ -517,13 +517,12 @@ int rr_report_stats(struct thread *tinfo) int num_events = thread_stats_events(tinfo); PRINT(cb, "num_transactions", "%d", num_events); - struct neper_histo *sum = - tinfo[0].histo_factory->create(tinfo[0].histo_factory); + struct neper_histo *sum = neper_histo_new(tinfo[0].histo_factory); for (i = 0; i < opts->num_threads; i++) tinfo[i].stats->sumforeach(tinfo[i].stats, fn_add, sum); - sum->epoch(sum); - sum->print(sum); - sum->fini(sum); + neper_histo_epoch(sum); + neper_histo_print(sum); + neper_histo_delete(sum); if (path) { csv = print_header(path, "transactions,transactions/s", diff --git a/stats.c b/stats.c index 173e2dd..26e18d9 100644 --- a/stats.c +++ b/stats.c @@ -201,7 +201,7 @@ static void stat_delete(struct neper_stat *stat) if (impl) { if (impl->histo) - impl->histo->fini(impl->histo); + neper_histo_delete(impl->histo); free(impl->snaps); /* TODO: Add a destructor */ free(impl); } diff --git a/thread.c b/thread.c index 3782b05..91c2a60 100644 --- a/thread.c +++ b/thread.c @@ -47,7 +47,7 @@ static int fn_count_events(struct neper_stat *stat, void *ptr) { const struct neper_histo *histo = stat->histo(stat); - return histo->events(histo); + return neper_histo_samples(histo); } static int @@ -392,7 +392,7 @@ void start_worker_threads(struct options *opts, struct callbacks *cb, t[i].stats = neper_stats_init(cb); t[i].rusage = neper_rusage(opts->interval); t[i].data_pending = data_pending; - t[i].histo_factory = neper_histo_factory(&t[i], + t[i].histo_factory = neper_histo_factory_new(&t[i], NEPER_HISTO_SIZE, NEPER_HISTO_GROWTH); t[i].loop_inited = loop_inited; From 80b171f63ab33422387a7918f4dc4b0be1f5df0e Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Thu, 4 Jan 2024 09:27:54 +0000 Subject: [PATCH 4/7] histo: replace threshold table with faster bit-based logarithms histograms store samples in buckets with pseudo logarithmic size. The previous implementation used a table of thresholds, and binary search to locate the correct bucket. This patch replaces the thresholds with the fast pseudo-logarithm algorithm used in lr-cstats and bpftrace so we can locate the bucket in a handful of instructions. This gives memory savings, reduced cache trashing, and better performance. Tests show that with a hot cache a lookup now takes less than 2us compared to 20-25us with the previous approach. Also, we can remove the now-useless neper_histo_factory. The actual resolution of the buckets is approximately the same as in the previous implementation (about 1.5%). On passing, correct a few bugs in the previous implementation: - resolution was supposed to be 0.25% but due to an implementation bug it was around 1% or even bigger at low values, and cause the thresholds to become negative - conversion from double to int for the sample could have unchecked overflows. Tested with tcp_rr and verifying that the distribution and csv files contain correct values. --- histo.c | 174 ++++++++++++++++++++++--------------------------------- histo.h | 22 +++---- rr.c | 4 +- thread.c | 8 --- thread.h | 2 - 5 files changed, 78 insertions(+), 132 deletions(-) diff --git a/histo.c b/histo.c index d88fd83..1e3c326 100644 --- a/histo.c +++ b/histo.c @@ -26,8 +26,9 @@ static const int TIME_RESOLUTION = 100 * 1000000; struct neper_histo { const struct thread *thread; - int num_buckets; /* # of buckets allocated */ - int *ceil; /* Max value that can be hashed into each bucket */ + uint32_t num_buckets; /* # of buckets allocated */ + uint8_t k_bits; /* resolution */ + uint64_t sample_max_value; uint64_t *all_buckets; uint64_t *cur_buckets; @@ -58,12 +59,46 @@ struct neper_histo { bool first_all; /* Is this the first call to all_percent() */ }; -struct neper_histo_factory { - const struct thread *thread; +/* Conversion of a 64-bit value to an approximately logarithmic index + * with k bits of resolution. + * lr_bucket(n, k) computes the log2, followed by the k next significant bits. + * + * lr_bucket_lo(b, k) returns the lower bound of bucket b. + * Translate the index into the starting value for the corresponding interval. + * Each power of 2 is mapped into N = 2**k intervals, each of size + * S = 1 << ((index >> k) - 1), and starting at S * N. + * The last k bits of index indicate which interval we want. + * + * For example, if k = 2 and index = 0b11011 (27) we have: + * - N = 2**2 = 4; + * - interval size S is 1 << ((0b11011 >> 2) - 1) = 1 << (6 - 1) = 32 + * - starting value is S * N = 128 + * - the last 2 bits 11 indicate the third interval so the + * starting value is 128 + 32*3 = 224 + */ - int num_buckets; /* # of buckets allocated */ - int *ceil; /* Max value that can be hashed into each bucket */ -}; +#define fls64(x) ((x) == 0? 0 : (64 - __builtin_clzl(x))) +static int lr_bucket(uint64_t val, int k) +{ + const uint64_t mask = (1ul << k) - 1; + const int bucket = fls64(val >> k); + int slot = bucket == 0 ? val : ((bucket << k) | ((val >> (bucket - 1)) & mask) ); + return slot; +} + +static uint64_t lr_bucket_lo(int index, int k) +{ + const uint32_t n = (1 << k), interval = index & (n - 1); + if (index < n) + return index; + const uint32_t power_of_2 = (index >> k) - 1; + return (1ul << power_of_2) * (n + interval); +} + +static uint64_t lr_bucket_hi(int index, int k) +{ + return lr_bucket_lo(index + 1, k) - 1; +} double neper_histo_min(const struct neper_histo *histo) { @@ -106,17 +141,17 @@ static void histo_all_finalize(struct neper_histo *impl) for (i = 0; i < impl->num_buckets; i++) { sub += impl->all_buckets[i]; while (p < 100 && p * cent <= sub) - impl->all_percent[p++] = impl->ceil[i]; + impl->all_percent[p++] = lr_bucket_hi(i, impl->k_bits); if (p == 100) { if (nnn <= sub) { - int c = impl->ceil[i]; + int c = lr_bucket_hi(i, impl->k_bits); impl->all_percent[PER_INDEX_99_9] = c; p++; } } if (p == 101) { if (nnnn <= sub) { - int c = impl->ceil[i]; + int c = lr_bucket_hi(i, impl->k_bits); impl->all_percent[PER_INDEX_99_99] = c; p++; } @@ -137,17 +172,17 @@ static void histo_one_finalize(struct neper_histo *impl) int n = impl->cur_buckets[i]; sub += n; while (p < 100 && p * cent <= sub) - impl->one_percent[p++] = impl->ceil[i]; + impl->one_percent[p++] = lr_bucket_hi(i, impl->k_bits); if (p == 100) { if (nnn <= sub) { - int c = impl->ceil[i]; + int c = lr_bucket_hi(i, impl->k_bits); impl->one_percent[PER_INDEX_99_9] = c; p++; } } if (p == 101) { if (nnnn <= sub) { - int c = impl->ceil[i]; + int c = lr_bucket_hi(i, impl->k_bits); impl->one_percent[PER_INDEX_99_99] = c; p++; } @@ -216,35 +251,8 @@ void neper_histo_add(struct neper_histo *desi, const struct neper_histo *srci) desi->cur_buckets[i] += srci->all_buckets[i]; } -// binary search for the correct bucket index -static int histo_find_bucket_idx(struct neper_histo *impl, int ticks) -{ - int l_idx = 0; - int r_idx = impl->num_buckets - 1; - - if (ticks > impl->ceil[r_idx]) - return r_idx; - - while (l_idx <= r_idx) { - int idx = (l_idx + r_idx) / 2; - if (impl->ceil[idx] < ticks) { - l_idx = idx + 1; - } else { - if (idx == 0) - return idx; - else if (impl->ceil[idx -1] < ticks) - return idx; - else - r_idx = idx - 1; - } - } - - return -1; -} - void neper_histo_event(struct neper_histo *impl, double delta_s) { - int ticks = delta_s * TIME_RESOLUTION; int i; impl->cur_count++; @@ -254,13 +262,18 @@ void neper_histo_event(struct neper_histo *impl, double delta_s) impl->cur_min = MIN(impl->cur_min, delta_s); impl->cur_max = MAX(impl->cur_max, delta_s); - i = histo_find_bucket_idx(impl, ticks); - if (i == -1) { + delta_s *= TIME_RESOLUTION; /* convert to ticks, potential overflow */ + if (delta_s < 0 || delta_s > impl->sample_max_value) { LOG_ERROR(impl->thread->cb, - "%s(): not able to find bucket for ticks %d", - __func__, ticks); + "%s(): not able to find bucket for delta_s %g", + __func__, delta_s / TIME_RESOLUTION); + /* TODO: This will also cause an error in reporting 100% and + * high percentiles, because the sum of buckets will never + * reach the total count. + */ return; } + i = lr_bucket((uint64_t)delta_s, impl->k_bits); impl->cur_buckets[i]++; } @@ -280,7 +293,7 @@ void neper_histo_epoch(struct neper_histo *impl) impl->all_min = MIN(impl->all_min, impl->cur_min); impl->one_min = impl->cur_min; - impl->cur_min = DBL_MAX; + impl->cur_min = impl->sample_max_value;; impl->all_max = MAX(impl->all_max, impl->cur_max); impl->one_max = impl->cur_max; @@ -289,35 +302,6 @@ void neper_histo_epoch(struct neper_histo *impl) histo_one_finalize(impl); } -/* - * Returns the size of the hash table needed for the given parameters. - * If 'table' and 'ceil' are non-null then populate them as well. - * - * 'table' maps an incoming value to a bucket so we can do an O(1) lookup. - * 'ceils' tracks the maximum value stored in each bucket. - * - * The delta between each bucket increases exponentially and is stored as a - * double. However, it is rounded down to the nearest integer when used. So - * for example, with a growth rate of 1.02, the delta between the first and - * second buckets will be 1.02, rounded down to 1. The delta between the - * second and third buckets will be 1.02^2 ~= 1.04, which also rounds down to 1. - * Eventually the delta will climb above 2 and that will become the new value. - */ - -static void histo_hash(int num_buckets, double growth, int *ceils) -{ - double delta = 1.0; - int ceil = 1; - int hash = 0; - - while (hash < num_buckets) { - ceils[hash] = ceil; - delta *= growth; - ceil += (int)delta; - hash++; - } -} - void neper_histo_print(struct neper_histo *histo) { const struct thread *t = histo->thread; @@ -349,49 +333,29 @@ void neper_histo_delete(struct neper_histo *impl) if (impl) { free(impl->all_buckets); free(impl->cur_buckets); - free(impl->ceil); free(impl); } } -struct neper_histo *neper_histo_new( const struct neper_histo_factory *fimpl) +struct neper_histo *neper_histo_new(const struct thread *t, uint8_t k_bits) { struct neper_histo *impl = calloc(1, sizeof(*impl)); - impl->thread = fimpl->thread; - impl->num_buckets = fimpl->num_buckets; - impl->ceil = fimpl->ceil; + if (k_bits > 10) + k_bits = 10; + impl->thread = t; + impl->k_bits = k_bits; + impl->num_buckets = 65 * (1 << k_bits); + impl->sample_max_value = ~0ul; - impl->all_buckets = calloc(fimpl->num_buckets, sizeof(uint64_t)); - impl->cur_buckets = calloc(fimpl->num_buckets, sizeof(uint64_t)); + impl->all_buckets = calloc(impl->num_buckets, sizeof(uint64_t)); + impl->cur_buckets = calloc(impl->num_buckets, sizeof(uint64_t)); - impl->all_min = DBL_MAX; - impl->cur_min = DBL_MAX; + impl->all_min = impl->sample_max_value; + impl->cur_min = impl->sample_max_value; impl->first_all = true; return impl; } - -void neper_histo_factory_delete(struct neper_histo_factory *impl) -{ - if (impl) { - free(impl->ceil); - free(impl); - } -} - -struct neper_histo_factory *neper_histo_factory_new(const struct thread *t, - int num_buckets, double growth) -{ - struct neper_histo_factory *impl = calloc(1, sizeof(*impl)); - - impl->thread = t; - impl->num_buckets = num_buckets; - impl->ceil = calloc(impl->num_buckets, sizeof(int)); - - histo_hash(num_buckets, growth, impl->ceil); - - return impl; -} diff --git a/histo.h b/histo.h index 2b78d5e..d847bab 100644 --- a/histo.h +++ b/histo.h @@ -19,8 +19,6 @@ #include -struct thread; - /* * A simple histogram API for tracking a series of latency measurements: * @@ -31,11 +29,16 @@ struct thread; * neper_histo_epoch() perhaps every second or so. */ -struct neper_histo_factory; +/* Internally the collector allows 64-bit values in buckets with k_bits + * significant bits. 6 gives 1.5% error and about 4K buckets. + */ +#define DEFAULT_K_BITS 4 + +struct thread; struct neper_histo; /* Create a new collector */ -struct neper_histo *neper_histo_new(const struct neper_histo_factory *); +struct neper_histo *neper_histo_new(const struct thread *t, uint8_t k_bits); /* Returns the min of the previous sampling epoch. */ double neper_histo_min(const struct neper_histo *); @@ -70,15 +73,4 @@ void neper_histo_print(struct neper_histo *); /* Destroy the object */ void neper_histo_delete(struct neper_histo *); -/* - * We use a factory to create histo objects so they can all share one set of - * common lookup tables, saving a great deal of memory. - */ - -void neper_histo_factory_delete(struct neper_histo_factory *); - -struct neper_histo_factory *neper_histo_factory_new(const struct thread *, - int size, - double growth); - #endif diff --git a/rr.c b/rr.c index 00261e8..e163afa 100644 --- a/rr.c +++ b/rr.c @@ -144,7 +144,7 @@ static struct neper_stat *rr_latency_init(struct flow *f) if (t->opts->nostats) return NULL; - struct neper_histo *histo = neper_histo_new(t->histo_factory); + struct neper_histo *histo = neper_histo_new(t, DEFAULT_K_BITS); size = sizeof(struct rr_snap_opaque) + t->percentiles * sizeof(double); @@ -517,7 +517,7 @@ int rr_report_stats(struct thread *tinfo) int num_events = thread_stats_events(tinfo); PRINT(cb, "num_transactions", "%d", num_events); - struct neper_histo *sum = neper_histo_new(tinfo[0].histo_factory); + struct neper_histo *sum = neper_histo_new(tinfo, DEFAULT_K_BITS); for (i = 0; i < opts->num_threads; i++) tinfo[i].stats->sumforeach(tinfo[i].stats, fn_add, sum); neper_histo_epoch(sum); diff --git a/thread.c b/thread.c index 91c2a60..e3fee8a 100644 --- a/thread.c +++ b/thread.c @@ -36,11 +36,6 @@ #include #endif -// max value = 1.0025^8192 = 764278329 -// If TIME_RESOLUTION is 0.01 us, max latency in histogram = 7.642783298s -#define NEPER_HISTO_SIZE 8192 /* # of buckets in the histogram */ -#define NEPER_HISTO_GROWTH 1.0025 /* bucket growth rate */ - /* Callbacks for the neper_stats sumforeach() function. */ static int @@ -392,9 +387,6 @@ void start_worker_threads(struct options *opts, struct callbacks *cb, t[i].stats = neper_stats_init(cb); t[i].rusage = neper_rusage(opts->interval); t[i].data_pending = data_pending; - t[i].histo_factory = neper_histo_factory_new(&t[i], - NEPER_HISTO_SIZE, - NEPER_HISTO_GROWTH); t[i].loop_inited = loop_inited; t[i].loop_init_c = loop_init_c; t[i].loop_init_m = loop_init_m; diff --git a/thread.h b/thread.h index c3f1af4..2a4a893 100644 --- a/thread.h +++ b/thread.h @@ -24,7 +24,6 @@ struct addrinfo; struct neper_fn; -struct neper_histo_factory; struct neper_pq; struct neper_stats; @@ -103,7 +102,6 @@ struct thread { struct timespec *time_start; pthread_mutex_t *time_start_mutex; struct rusage *rusage_start; - struct neper_histo_factory *histo_factory; struct neper_stats *stats; struct neper_rusage *rusage; struct io_stats io_stats; From 8b2f9e1ee2814926daa37b470255ebaeba60c897 Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Thu, 4 Jan 2024 19:20:06 +0000 Subject: [PATCH 5/7] histograms: allow arbitrary percentiles Allow arbitrary percentiles to be specificed, instead of just integer and p99.9 and p99.99 This also makes the code faster because we can just compute the values requested instead of all 103 entries. Any floating point number between 0 and 100 is now accepted, with 999 and 9999 mapped to 99.9 and 99.99 for backward compatibility. Tested as usual with ./tcp_rr -c -H 127.0.0.1 -p1,2,10,50,90,999,9999,100 -A/tmp/x.csv and verifying the correct content of the csv file. --- define_all_flags.c | 2 +- histo.c | 162 +++++++++++++-------------------------------- histo.h | 4 +- percentiles.c | 105 ++++++++++++++--------------- percentiles.h | 12 +--- print.c | 17 +---- rr.c | 23 ++----- thread.c | 3 - thread.h | 1 - 9 files changed, 109 insertions(+), 220 deletions(-) diff --git a/define_all_flags.c b/define_all_flags.c index 0204262..4992c55 100644 --- a/define_all_flags.c +++ b/define_all_flags.c @@ -88,7 +88,7 @@ struct flags_parser *add_flags_rr(struct flags_parser *fp) /* Define flags common to all RR and CRR main programs */ DEFINE_FLAG(fp, int, request_size, 1, 'Q', "Number of bytes in a request from client to server"); DEFINE_FLAG(fp, int, response_size, 1, 'R', "Number of bytes in a response from server to client"); - DEFINE_FLAG(fp, struct percentiles, percentiles, { .chosen = { false } }, 'p', "Set reported latency percentiles (list)"); + DEFINE_FLAG(fp, struct percentiles, percentiles, {}, 'p', "Set reported latency percentiles (list)"); DEFINE_FLAG_PARSER(fp, percentiles, percentiles_parse); DEFINE_FLAG_PRINTER(fp, percentiles, percentiles_print); DEFINE_FLAG(fp, int, test_length, 10, 'l', "Test length, >0 seconds, <0 transactions"); diff --git a/histo.c b/histo.c index 1e3c326..cc697b0 100644 --- a/histo.c +++ b/histo.c @@ -22,6 +22,7 @@ // use 0.01 us time resolution static const int TIME_RESOLUTION = 100 * 1000000; +static const double TICKS_TO_SEC = 1.0 / TIME_RESOLUTION; struct neper_histo { const struct thread *thread; @@ -53,8 +54,8 @@ struct neper_histo { double one_max; double cur_max; - int all_percent[PER_INDEX_COUNT]; /* % across all completed epochs */ - int one_percent[PER_INDEX_COUNT]; /* % of the last completed epoch */ + double *all_p_values; /* % across all completed epochs */ + double *one_p_values; /* % of the last completed epoch */ bool first_all; /* Is this the first call to all_percent() */ }; @@ -127,12 +128,9 @@ double neper_histo_stddev(const struct neper_histo *histo) static void histo_all_finalize(struct neper_histo *impl) { + const struct percentiles *pc = &impl->thread->opts->percentiles; double cent = impl->all_count / 100.0; - double nnn = (impl->all_count * 99.9) / 100.0; - double nnnn = (impl->all_count * 99.99) / 100.0; - int sub = 0; - int p = 1; - int i; + int sub = 0, v = 0, i; if (!impl->first_all) return; @@ -140,96 +138,30 @@ static void histo_all_finalize(struct neper_histo *impl) for (i = 0; i < impl->num_buckets; i++) { sub += impl->all_buckets[i]; - while (p < 100 && p * cent <= sub) - impl->all_percent[p++] = lr_bucket_hi(i, impl->k_bits); - if (p == 100) { - if (nnn <= sub) { - int c = lr_bucket_hi(i, impl->k_bits); - impl->all_percent[PER_INDEX_99_9] = c; - p++; - } - } - if (p == 101) { - if (nnnn <= sub) { - int c = lr_bucket_hi(i, impl->k_bits); - impl->all_percent[PER_INDEX_99_99] = c; - p++; - } - } + while (v < pc->p_count && sub >= pc->p_th[v] * cent) + impl->all_p_values[v++] = lr_bucket_hi(i, impl->k_bits) * TICKS_TO_SEC; } } static void histo_one_finalize(struct neper_histo *impl) { + const struct percentiles *pc = &impl->thread->opts->percentiles; double cent = impl->one_count / 100.0; - double nnn = (impl->one_count * 99.9) / 100.0; - double nnnn = (impl->one_count * 99.99) / 100.0; - int sub = 0; - int p = 1; - int i; + int sub = 0, v = 0, i; for (i = 0; i < impl->num_buckets; i++) { int n = impl->cur_buckets[i]; sub += n; - while (p < 100 && p * cent <= sub) - impl->one_percent[p++] = lr_bucket_hi(i, impl->k_bits); - if (p == 100) { - if (nnn <= sub) { - int c = lr_bucket_hi(i, impl->k_bits); - impl->one_percent[PER_INDEX_99_9] = c; - p++; - } - } - if (p == 101) { - if (nnnn <= sub) { - int c = lr_bucket_hi(i, impl->k_bits); - impl->one_percent[PER_INDEX_99_99] = c; - p++; - } - } impl->all_buckets[i] += n; impl->cur_buckets[i] = 0; + while (v < pc->p_count && sub >= pc->p_th[v] * cent) + impl->one_p_values[v++] = lr_bucket_hi(i, impl->k_bits) * TICKS_TO_SEC; } } -static double histo_all_percent(struct neper_histo *impl, int percentage) +double neper_histo_percent(const struct neper_histo *impl, int index) { - histo_all_finalize(impl); - - switch (percentage) { - case 0: - return impl->all_min; - case 100: - return impl->all_max; - case 999: - return (double)impl->all_percent[PER_INDEX_99_9] / - TIME_RESOLUTION; - case 9999: - return (double)impl->all_percent[PER_INDEX_99_99] / - TIME_RESOLUTION; - default: - return (double)impl->all_percent[percentage] / - TIME_RESOLUTION; - } -} - -double neper_histo_percent(const struct neper_histo *impl, int percentage) -{ - switch (percentage) { - case 0: - return impl->one_min; - case 100: - return impl->one_max; - case 999: - return (double)impl->one_percent[PER_INDEX_99_9] / - TIME_RESOLUTION; - case 9999: - return (double)impl->one_percent[PER_INDEX_99_99] / - TIME_RESOLUTION; - default: - return (double)impl->one_percent[percentage] / - TIME_RESOLUTION; - } + return impl->one_p_values[index]; } uint64_t neper_histo_samples(const struct neper_histo *histo) @@ -293,7 +225,7 @@ void neper_histo_epoch(struct neper_histo *impl) impl->all_min = MIN(impl->all_min, impl->cur_min); impl->one_min = impl->cur_min; - impl->cur_min = impl->sample_max_value;; + impl->cur_min = impl->sample_max_value; impl->all_max = MAX(impl->all_max, impl->cur_max); impl->one_max = impl->cur_max; @@ -305,57 +237,55 @@ void neper_histo_epoch(struct neper_histo *impl) void neper_histo_print(struct neper_histo *histo) { const struct thread *t = histo->thread; - const struct options *opts = t->opts; + const struct percentiles *pc = &t->opts->percentiles; + histo_all_finalize(histo); PRINT(t->cb, "latency_min", "%.9f", histo->all_min); PRINT(t->cb, "latency_max", "%.9f", histo->all_max); PRINT(t->cb, "latency_mean", "%.9f", histo->all_sum / histo->all_count); PRINT(t->cb, "latency_stddev", "%.9f", histo_stddev(histo->all_count, histo->all_sum, histo->all_sum2)); - int i; - for (i = 0; i < 100; i++) - if (percentiles_chosen(&opts->percentiles, i)) { - char key[13]; - sprintf(key, "latency_p%d", i); - PRINT(t->cb, key, "%.9f", histo_all_percent(histo, i)); - } - if (percentiles_chosen(&opts->percentiles, PER_INDEX_99_9)) - PRINT(t->cb, "latency_p99.9", "%.9f", - histo_all_percent(histo, PER_INDEX_99_9)); - if (percentiles_chosen(&opts->percentiles, PER_INDEX_99_99)) - PRINT(t->cb, "latency_p99.99", "%.9f", - histo_all_percent(histo, PER_INDEX_99_99)); + for (int i = 0; i < pc->p_count; i++) { + char key[32]; + sprintf(key, "latency_p%g", pc->p_th[i]); + PRINT(t->cb, key, "%.9f", histo->all_p_values[i]); + } } void neper_histo_delete(struct neper_histo *impl) { - if (impl) { - free(impl->all_buckets); - free(impl->cur_buckets); + if (impl) free(impl); - } } struct neper_histo *neper_histo_new(const struct thread *t, uint8_t k_bits) { - - struct neper_histo *impl = calloc(1, sizeof(*impl)); + const uint16_t p_count = t->opts->percentiles.p_count; + struct neper_histo *ret, histo = {}; + size_t memsize = sizeof(histo); if (k_bits > 10) k_bits = 10; - impl->thread = t; - impl->k_bits = k_bits; - impl->num_buckets = 65 * (1 << k_bits); - impl->sample_max_value = ~0ul; - - impl->all_buckets = calloc(impl->num_buckets, sizeof(uint64_t)); - impl->cur_buckets = calloc(impl->num_buckets, sizeof(uint64_t)); - - impl->all_min = impl->sample_max_value; - impl->cur_min = impl->sample_max_value; - - impl->first_all = true; - - return impl; + histo.thread = t; + histo.k_bits = k_bits; + histo.num_buckets = 65 * (1 << k_bits); + histo.sample_max_value = ~0ul; + histo.first_all = true; + + /* Allocate memory in one chunk */ + memsize += histo.num_buckets * 2 * sizeof(histo.all_buckets[0]); + memsize += p_count * 2 * sizeof(histo.all_p_values[0]); + + ret = calloc(1, memsize); + *ret = histo; + ret->all_buckets = (void *)(ret + 1); + ret->cur_buckets = ret->all_buckets + ret->num_buckets; + ret->all_p_values = (void *)(ret->cur_buckets + ret->num_buckets); + ret->one_p_values = ret->all_p_values + p_count; + + ret->all_min = ret->sample_max_value; + ret->cur_min = ret->sample_max_value; + + return ret; } diff --git a/histo.h b/histo.h index d847bab..cb88a1c 100644 --- a/histo.h +++ b/histo.h @@ -52,8 +52,8 @@ double neper_histo_mean(const struct neper_histo *); /* Returns the stddev of the previous sampling epoch. */ double neper_histo_stddev(const struct neper_histo *); -/* Returns the percent of the previous sampling epoch. */ -double neper_histo_percent(const struct neper_histo *, int percentage); +/* Returns the index-th percent of the previous sampling epoch. */ +double neper_histo_percent(const struct neper_histo *, int index); /* Adds one histogram to the current epoch of another. */ void neper_histo_add(struct neper_histo *des, const struct neper_histo *src); diff --git a/percentiles.c b/percentiles.c index 89b757e..9f70746 100644 --- a/percentiles.c +++ b/percentiles.c @@ -23,77 +23,72 @@ #include "lib.h" #include "logging.h" +static int my_dsort(const void *p1, const void *p2) +{ + const double a = *(double *)p1, b = *(double *)p2; + + return a < b ? -1 : (a > b ? 1 : 0); +} + void percentiles_parse(const char *arg, void *out, struct callbacks *cb) { struct percentiles *p = out; char *endptr; - long val; + int sz = 0; + double d; - while (true) { + while (arg) { errno = 0; - val = strtol(arg, &endptr, 10); - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || - (errno != 0 && val == 0)) - PLOG_FATAL(cb, "strtol"); - if (endptr == arg) - break; - if ((val < 0 || val > 100) && (val != 999) && (val != 9999)) - LOG_FATAL(cb, "%ld percentile doesn't exist", val); - switch (val) { - case 999: - p->chosen[PER_INDEX_99_9] = true; - break; - case 9999: - p->chosen[PER_INDEX_99_99] = true; - break; - default: - p->chosen[val] = true; - break; + d = strtod(arg, &endptr); + /* backward compatibility */ + if (d == 999) + d = 99.9; + else if (d == 9999) + d = 99.99; + if (errno || d < 0 || d > 100 || endptr == arg) + LOG_FATAL(cb, "invalid -p argument %s", arg); + + if (p->p_count >= sz) { + sz = 2 * sz + 2; + p->p_th = realloc(p->p_th, sz * sizeof(double)); + if (!p->p_th) + LOG_FATAL(cb, "cannot allocate %d entries", sz); } - LOG_INFO(cb, "%ld percentile is chosen", val); + p->p_th[p->p_count++] = d; + LOG_INFO(cb, "%g percentile is chosen", d); if (*endptr == '\0') break; arg = endptr + 1; } + if (!p->p_count) + return; + qsort(p->p_th, p->p_count, sizeof(double), my_dsort); + /* remove duplicates */ + int i, cur = 0; + for (i = 1; i < p->p_count; i++) { + if (p->p_th[cur] == p->p_th[i]) + LOG_INFO(cb, "remove duplicate percentile %g", p->p_th[i]); + else + p->p_th[++cur] = p->p_th[i]; + } + p->p_count = cur; } void percentiles_print(const char *name, const void *var, struct callbacks *cb) { const struct percentiles *p = var; - char buf[10], s[400] = ""; - int i; - - for (i = 0; i <= 100; i++) { - if (p->chosen[i]) { - sprintf(buf, "%d,", i); - strcat(s, buf); - } - } - if (p->chosen[PER_INDEX_99_9]) - strcat(s, "99.9,"); - if (p->chosen[PER_INDEX_99_99]) - strcat(s, "99.99,"); - if (strlen(s) > 0) - s[strlen(s) - 1] = '\0'; /* remove trailing comma */ - PRINT(cb, name, "%s", s); -} + char buf, *s; + int i, len = 0; -bool percentiles_chosen(const struct percentiles *p, int percent) -{ - if (p) - return p->chosen[percent]; + /* first pass, compute length */ + for (i = 0; i < p->p_count; i++) + len += snprintf(&buf, 0, "%g,", p->p_th[i]); - return false; -} - -int percentiles_count(const struct percentiles *p) -{ - if (p) { - int i, sum = 0; - for (i = 0; i < PER_INDEX_COUNT; i++) - sum += p->chosen[i] ? 1 : 0; - return sum; - } - - return 0; + /* second pass, create string */ + s = calloc(1, len + 1); + len = 0; + for (i = 0; i < p->p_count; i++) + len += sprintf(s + len, "%g,", p->p_th[i]); + PRINT(cb, name, "%s", s); + free(s); } diff --git a/percentiles.h b/percentiles.h index bfb0440..8fb725a 100644 --- a/percentiles.h +++ b/percentiles.h @@ -17,22 +17,14 @@ #ifndef THIRD_PARTY_NEPER_PERCENTILES_H #define THIRD_PARTY_NEPER_PERCENTILES_H -#include - -#define PER_INDEX_99_9 101 -#define PER_INDEX_99_99 102 -#define PER_INDEX_COUNT 103 - struct callbacks; struct percentiles { - bool chosen[PER_INDEX_COUNT]; + int p_count; + double *p_th; /* indexes */ }; void percentiles_parse(const char *arg, void *out, struct callbacks *); void percentiles_print(const char *name, const void *var, struct callbacks *); -bool percentiles_chosen(const struct percentiles *, int percent); -int percentiles_count(const struct percentiles *); - #endif diff --git a/print.c b/print.c index ed858ca..fe9b213 100644 --- a/print.c +++ b/print.c @@ -43,23 +43,12 @@ FILE *print_header(const char *path, const char *things, const char *nl, return csv; } -void print_latency_header(FILE *csv, const struct percentiles *percentiles) +void print_latency_header(FILE *csv, const struct percentiles *pc) { fprintf(csv, ",latency_min,latency_mean,latency_max,latency_stddev"); - if (percentiles) { - int i; - for (i = 0; i < 100; i++) - if (percentiles_chosen(percentiles, i)) - fprintf(csv, ",latency_p%d", i); - if (percentiles_chosen(percentiles, PER_INDEX_99_9)) - fprintf(csv, ",latency_p99_9"); - if (percentiles_chosen(percentiles, PER_INDEX_99_99)) - fprintf(csv, ",latency_p99_99"); - if (percentiles_chosen(percentiles, 100)) - fprintf(csv, ",latency_p100"); - } - + for (int i = 0; i < pc->p_count; i++) + fprintf(csv, ",latency_p%g", pc->p_th[i]); fprintf(csv, "\n"); } diff --git a/rr.c b/rr.c index e163afa..4e08336 100644 --- a/rr.c +++ b/rr.c @@ -146,7 +146,7 @@ static struct neper_stat *rr_latency_init(struct flow *f) struct neper_histo *histo = neper_histo_new(t, DEFAULT_K_BITS); - size = sizeof(struct rr_snap_opaque) + t->percentiles * sizeof(double); + size = sizeof(struct rr_snap_opaque) + t->opts->percentiles.p_count * sizeof(double); return neper_stat_init(f, histo, size); } @@ -307,13 +307,8 @@ static void rr_snapshot(struct thread *t, struct neper_stat *stat, opaque->mean = neper_histo_mean(histo); opaque->stddev = neper_histo_stddev(histo); - if (t->percentiles) { - int i, j = 0; - for (i = 0; i < PER_INDEX_COUNT; i++) - if (percentiles_chosen(&t->opts->percentiles, i)) - opaque->percentile[j++] = - neper_histo_percent(histo, i); - } + for (int i = 0; i < t->opts->percentiles.p_count; i++) + opaque->percentile[i] = neper_histo_percent(histo, i); } static bool rr_do_compl(struct flow *f, @@ -480,16 +475,8 @@ static void rr_print_snap(struct thread *t, int flow_index, fprintf(csv, ",%f,%f,%f,%f", rso->min, rso->mean, rso->max, rso->stddev); - if (t->percentiles) { - const struct options *opts = t->opts; - int i, j = 0; - - for (i = 0; i < PER_INDEX_COUNT; i++) - if (percentiles_chosen(&opts->percentiles, i)) - fprintf(csv, ",%f", - rso->percentile[j++]); - } - + for (int i = 0; i < t->opts->percentiles.p_count; i++) + fprintf(csv, ",%f", rso->percentile[i]); fprintf(csv, "\n"); } } diff --git a/thread.c b/thread.c index e3fee8a..1abb6e7 100644 --- a/thread.c +++ b/thread.c @@ -360,8 +360,6 @@ void start_worker_threads(struct options *opts, struct callbacks *cb, allowed_cores = get_cpuset(cpuset, cb); LOG_INFO(cb, "Number of allowed_cores = %d", allowed_cores); - int percentiles = percentiles_count(&opts->percentiles); - for (i = 0; i < opts->num_threads; i++) { t[i].index = i; t[i].fn = fn; @@ -377,7 +375,6 @@ void start_worker_threads(struct options *opts, struct callbacks *cb, t[i].flow_first = first_flow_in_thread(&t[i]); t[i].flow_limit = flows_in_thread(&t[i]); t[i].flow_count = 0; - t[i].percentiles = percentiles; t[i].local_hosts = parse_local_hosts(opts, t[i].num_local_hosts, cb); t[i].ready = ready; diff --git a/thread.h b/thread.h index 2a4a893..e6d66b5 100644 --- a/thread.h +++ b/thread.h @@ -92,7 +92,6 @@ struct thread { int flow_first; /* global index of thread's first flow */ int flow_limit; /* number of flows to create on thread */ int flow_count; /* number of flows created on thread */ - int percentiles; /* number of requested percentiles */ int stop; void *f_mbuf; /* replaces per-flow buffers */ pthread_barrier_t *ready; From 7e3efb3ac0853bafe2b0e82bdedc379a60b80057 Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Thu, 4 Jan 2024 22:38:21 +0000 Subject: [PATCH 6/7] histo: only scan necessary buckets when computing percentiles Computing percentiles is expensive, as it requires scanning all the 4k-8k buckets used to store samples, and is done for each flow. Benchmarks show the original code took an average of 20us per flow, with frequent peaks in the 60-80us range. This patch eliminates the cost by not storing samples in buckets if no percentiles are requested, and otherwise achieves a ~5x reduction by tracking the range of buckets that contain values in each epoch. Also change the precision to 6 bits, which halves the cost without much impact on the results. This value may become a command line flag. Tested, as usual, by running tcp_rr and verifying the logs and csv --- histo.c | 38 +++++++++++++++++++++++++++++++------- histo.h | 2 +- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/histo.c b/histo.c index cc697b0..144e72d 100644 --- a/histo.c +++ b/histo.c @@ -29,6 +29,7 @@ struct neper_histo { uint32_t num_buckets; /* # of buckets allocated */ uint8_t k_bits; /* resolution */ + uint16_t p_count; /* percentiles, cached for convenience */ uint64_t sample_max_value; uint64_t *all_buckets; @@ -54,6 +55,14 @@ struct neper_histo { double one_max; double cur_max; + uint32_t all_max_index; + uint32_t one_max_index; + uint32_t cur_max_index; + + uint32_t all_min_index; + uint32_t one_min_index; + uint32_t cur_min_index; + double *all_p_values; /* % across all completed epochs */ double *one_p_values; /* % of the last completed epoch */ @@ -63,7 +72,6 @@ struct neper_histo { /* Conversion of a 64-bit value to an approximately logarithmic index * with k bits of resolution. * lr_bucket(n, k) computes the log2, followed by the k next significant bits. - * * lr_bucket_lo(b, k) returns the lower bound of bucket b. * Translate the index into the starting value for the corresponding interval. * Each power of 2 is mapped into N = 2**k intervals, each of size @@ -136,7 +144,7 @@ static void histo_all_finalize(struct neper_histo *impl) return; impl->first_all = false; - for (i = 0; i < impl->num_buckets; i++) { + for (i = impl->all_min_index; i <= impl->all_max_index; i++) { sub += impl->all_buckets[i]; while (v < pc->p_count && sub >= pc->p_th[v] * cent) impl->all_p_values[v++] = lr_bucket_hi(i, impl->k_bits) * TICKS_TO_SEC; @@ -149,7 +157,7 @@ static void histo_one_finalize(struct neper_histo *impl) double cent = impl->one_count / 100.0; int sub = 0, v = 0, i; - for (i = 0; i < impl->num_buckets; i++) { + for (i = impl->one_min_index; i <= impl->one_max_index; i++) { int n = impl->cur_buckets[i]; sub += n; impl->all_buckets[i] += n; @@ -177,9 +185,11 @@ void neper_histo_add(struct neper_histo *desi, const struct neper_histo *srci) desi->cur_min = MIN(desi->cur_min, srci->all_min); desi->cur_max = MAX(desi->cur_max, srci->all_max); + desi->cur_min_index = MIN(desi->cur_min_index, srci->all_min_index); + desi->cur_max_index = MAX(desi->cur_max_index, srci->all_max_index); int i; - for (i = 0; i < desi->num_buckets; i++) + for (i = srci->all_min_index; i <= srci->all_max_index; i++) desi->cur_buckets[i] += srci->all_buckets[i]; } @@ -205,7 +215,11 @@ void neper_histo_event(struct neper_histo *impl, double delta_s) */ return; } + if (!impl->p_count) + return; i = lr_bucket((uint64_t)delta_s, impl->k_bits); + impl->cur_min_index = MIN(impl->cur_min_index, i); + impl->cur_max_index = MAX(impl->cur_max_index, i); impl->cur_buckets[i]++; } @@ -231,6 +245,14 @@ void neper_histo_epoch(struct neper_histo *impl) impl->one_max = impl->cur_max; impl->cur_max = 0; + impl->all_min_index = MIN(impl->all_min_index, impl->cur_min_index); + impl->one_min_index = impl->cur_min_index; + impl->cur_min_index = impl->num_buckets - 1; + + impl->all_max_index = MAX(impl->all_max_index, impl->cur_max_index); + impl->one_max_index = impl->cur_max_index; + impl->cur_max_index = 0; + histo_one_finalize(impl); } @@ -261,7 +283,6 @@ void neper_histo_delete(struct neper_histo *impl) struct neper_histo *neper_histo_new(const struct thread *t, uint8_t k_bits) { - const uint16_t p_count = t->opts->percentiles.p_count; struct neper_histo *ret, histo = {}; size_t memsize = sizeof(histo); @@ -272,20 +293,23 @@ struct neper_histo *neper_histo_new(const struct thread *t, uint8_t k_bits) histo.num_buckets = 65 * (1 << k_bits); histo.sample_max_value = ~0ul; histo.first_all = true; + histo.p_count = t->opts->percentiles.p_count; /* Allocate memory in one chunk */ memsize += histo.num_buckets * 2 * sizeof(histo.all_buckets[0]); - memsize += p_count * 2 * sizeof(histo.all_p_values[0]); + memsize += histo.p_count * 2 * sizeof(histo.all_p_values[0]); ret = calloc(1, memsize); *ret = histo; ret->all_buckets = (void *)(ret + 1); ret->cur_buckets = ret->all_buckets + ret->num_buckets; ret->all_p_values = (void *)(ret->cur_buckets + ret->num_buckets); - ret->one_p_values = ret->all_p_values + p_count; + ret->one_p_values = ret->all_p_values + ret->p_count; ret->all_min = ret->sample_max_value; ret->cur_min = ret->sample_max_value; + ret->all_min_index = ret->num_buckets - 1; + ret->cur_min_index = ret->num_buckets - 1; return ret; } diff --git a/histo.h b/histo.h index cb88a1c..95ba9f3 100644 --- a/histo.h +++ b/histo.h @@ -32,7 +32,7 @@ /* Internally the collector allows 64-bit values in buckets with k_bits * significant bits. 6 gives 1.5% error and about 4K buckets. */ -#define DEFAULT_K_BITS 4 +#define DEFAULT_K_BITS 6 struct thread; struct neper_histo; From c412b1831484a1125e0a1cf2dd41b8b130d97e1f Mon Sep 17 00:00:00 2001 From: Luigi Rizzo Date: Fri, 5 Jan 2024 15:10:42 +0000 Subject: [PATCH 7/7] snaps: de-virtualize methods. No functional change. neper_snaps methods were implemented as virtual functions, but since there is only one possible implementation this was overkill. Simplify the code by exposing the actual methods. The implementation still remains opaque. No functional changes. Tested with ./tcp_rr -c -H 127.0.0.1 -p1,2,10,50,90,999.9999,100 -A/tmp/x.csv -l 4 and verified that the csv file has the correct data. (histograms are only exercised in rr tests) --- snaps.c | 80 +++++++++++++++++++++----------------------------------- snaps.h | 13 +++++---- stats.c | 10 +++---- thread.c | 2 +- 4 files changed, 42 insertions(+), 63 deletions(-) diff --git a/snaps.c b/snaps.c index 6168d66..81286a3 100644 --- a/snaps.c +++ b/snaps.c @@ -19,15 +19,13 @@ #include "print.h" #include "rusage.h" -struct snaps_impl { - struct neper_snaps snaps; - +struct neper_snaps { struct neper_rusage *rusage; int total; /* # of snap structs allocated */ int count; /* # of populated snap structs */ int iter; /* iterator bookmark */ int extent; /* extended size of the snap struct */ - void *ptr; /* storage */ + char *ptr; /* storage */ }; /* @@ -50,11 +48,8 @@ void neper_snap_print(const struct neper_snap *snap, FILE *csv, print_rusage(csv, snap->rusage, nl); } -static struct neper_snap *snaps_get(const struct neper_snaps *snaps, int i) +static struct neper_snap *snaps_get(const struct neper_snaps *impl, int i) { - struct snaps_impl *impl = (void *)snaps; - char *ptr = impl->ptr; - if (i < 0) return NULL; if (i >= impl->total) { @@ -66,74 +61,57 @@ static struct neper_snap *snaps_get(const struct neper_snaps *snaps, int i) if (!reported) { fprintf(stderr, "Test longer than expected (%d), " - "use -l to extend\n", + "use -l to extend\n", i + reported++); - } - return (void *)(ptr + impl->total * impl->extent); + } + i = impl->total; /* point to spare element */ } - return (void *)(ptr + i * impl->extent); + return (void *)(impl->ptr + i * impl->extent); } /* Compare two containers by comparing the current iterator objects for each. */ -double neper_snaps_cmp(const struct neper_snaps *aptr, - const struct neper_snaps *bptr) +double neper_snaps_cmp(const struct neper_snaps *a, + const struct neper_snaps *b) { - const struct snaps_impl *a = (void *)aptr; - const struct snaps_impl *b = (void *)bptr; - const struct neper_snap *sa = snaps_get(&a->snaps, a->iter); - const struct neper_snap *sb = snaps_get(&b->snaps, b->iter); + const struct neper_snap *sa = snaps_get(a, a->iter); + const struct neper_snap *sb = snaps_get(b, b->iter); return neper_snap_cmp(sa, sb); } -static struct neper_snap *snaps_add(struct neper_snaps *snaps, - const struct timespec *now, uint64_t things) +struct neper_snap *neper_snaps_add(struct neper_snaps *snaps, + const struct timespec *now, uint64_t things) { - struct snaps_impl *impl = (void *)snaps; - struct neper_snap *snap = snaps_get(snaps, impl->count++); + struct neper_snap *snap = snaps_get(snaps, snaps->count++); snap->timespec = *now; - snap->rusage = impl->rusage->get(impl->rusage, now); + snap->rusage = snaps->rusage->get(snaps->rusage, now); snap->things = things; return snap; } -static int snaps_count(const struct neper_snaps *snaps) +int neper_snaps_count(const struct neper_snaps *snaps) { - const struct snaps_impl *impl = (void *)snaps; - - return impl->count; + return snaps->count; } -static const struct neper_snap *snaps_iter_next(struct neper_snaps *snaps) +const struct neper_snap *neper_snaps_iter_next(struct neper_snaps *snaps) { - struct snaps_impl *impl = (void *)snaps; - - int i = impl->iter++; - return (i < impl->count) ? snaps_get(snaps, i) : NULL; + int i = snaps->iter++; + return (i < snaps->count) ? snaps_get(snaps, i) : NULL; } -static bool snaps_iter_done(const struct neper_snaps *snaps) +bool neper_snaps_iter_done(const struct neper_snaps *snaps) { - const struct snaps_impl *impl = (void *)snaps; - - return (impl->iter >= impl->count); + return snaps->iter >= snaps->count; } struct neper_snaps *neper_snaps_init(struct neper_rusage *rusage, int total, int extra) { - struct snaps_impl *impl = calloc(1, sizeof(struct snaps_impl)); - struct neper_snaps *snaps = &impl->snaps; - - impl->rusage = rusage; - impl->total = total; - impl->count = 0; - impl->iter = 0; - impl->extent = sizeof(struct neper_snap) + extra; /* * Allocate memory for all the samples at startup, based on the * known values for total_length and interval. The actual test @@ -144,12 +122,14 @@ struct neper_snaps *neper_snaps_init(struct neper_rusage *rusage, * See snaps_get(). * TODO(lrizzo) Design a proper mechanism for dynamic allocation. */ - impl->ptr = calloc(total + 1, impl->extent); - - snaps->add = snaps_add; - snaps->count = snaps_count; - snaps->iter_next = snaps_iter_next; - snaps->iter_done = snaps_iter_done; + const int extent = sizeof(struct neper_snap) + extra; + const int size = sizeof(struct neper_snaps) + extent * (total + 1); + struct neper_snaps *snaps = calloc(1, size); + + snaps->rusage = rusage; + snaps->total = total; + snaps->extent = extent; + snaps->ptr = (char *)(snaps + 1); return snaps; } diff --git a/snaps.h b/snaps.h index 63a1b81..e8d26c1 100644 --- a/snaps.h +++ b/snaps.h @@ -52,13 +52,12 @@ void neper_snap_print(const struct neper_snap *, FILE *, double raw_thruput, * iter_done() Returns true once the end of the iterator has been reached. */ -struct neper_snaps { - struct neper_snap *(*add)(struct neper_snaps *, const struct timespec *, - uint64_t); - int (*count)(const struct neper_snaps *); - const struct neper_snap *(*iter_next)(struct neper_snaps *); - bool (*iter_done)(const struct neper_snaps *); -}; +struct neper_snaps; + +struct neper_snap *neper_snaps_add(struct neper_snaps *, const struct timespec *, uint64_t); +int neper_snaps_count(const struct neper_snaps *); +const struct neper_snap *neper_snaps_iter_next(struct neper_snaps *); +bool neper_snaps_iter_done(const struct neper_snaps *); double neper_snaps_cmp(const struct neper_snaps *, const struct neper_snaps *); diff --git a/stats.c b/stats.c index 26e18d9..96cc600 100644 --- a/stats.c +++ b/stats.c @@ -93,13 +93,13 @@ static void stat_event(struct thread *t, struct neper_stat *stat, int things, impl->things += things; - int i = snaps->count(snaps); + int i = neper_snaps_count(snaps); double threshold = t->opts->interval * (i + 1); /* Always record the first event, to capture the start time. */ if (elapsed >= threshold || !i || force) { - struct neper_snap *snap = snaps->add(snaps, &now, impl->things); + struct neper_snap *snap = neper_snaps_add(snaps, &now, impl->things); if (fn) fn(t, stat, snap); @@ -125,7 +125,7 @@ struct neper_coef *neper_stat_print(struct thread *ts, FILE *csv, while ((stat = pq->deq(pq))) { struct stat_impl *impl = (void *)stat; struct neper_snaps *snaps = impl->snaps; - const struct neper_snap *snap = snaps->iter_next(snaps); + const struct neper_snap *snap = neper_snaps_iter_next(snaps); current_total += snap->things - impl->scratch; impl->scratch = snap->things; @@ -157,7 +157,7 @@ struct neper_coef *neper_stat_print(struct thread *ts, FILE *csv, coef->event(coef, &snap->timespec, current_total); - if (!snaps->iter_done(snaps)) + if (!neper_snaps_iter_done(snaps)) pq->enq(pq, stat); } @@ -221,7 +221,7 @@ static int fn_snaps(struct neper_stat *stat, void *unused) struct stat_impl *impl = (void *)stat; const struct neper_snaps *snaps = impl->snaps; - return snaps->count(snaps); + return neper_snaps_count(snaps); } static void stats_container_insert(struct neper_stats *stats, diff --git a/thread.c b/thread.c index 1abb6e7..db7435a 100644 --- a/thread.c +++ b/thread.c @@ -49,7 +49,7 @@ static int fn_count_snaps(struct neper_stat *stat, void *ptr) { const struct neper_snaps *snaps = stat->snaps(stat); - return snaps->count(snaps); + return neper_snaps_count(snaps); } static int