Debug ds_pool refs

src/engine/SConscript

@@ -32,7 +32,7 @@ def scons():
                'srv_cli.c', 'profile.c', 'rpc.c',
                'server_iv.c', 'srv.c', 'srv.pb-c.c',
                'sched.c', 'ult.c', 'event.pb-c.c',
-               'srv_metrics.c'] + libdaos_tgts
+               'srv_metrics.c', 'ref.c'] + libdaos_tgts
 
     if denv["STACK_MMAP"] == 1:
         denv.Append(CCFLAGS=['-DULT_MMAP_STACK'])

src/engine/ref.c

@@ -0,0 +1,55 @@
+/*
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
+ *
+ * SPDX-License-Identifier: BSD-2-Clause-Patent
+ */
+
+#define D_LOGFAC DD_FAC(server)
+
+#include <daos_srv/daos_engine.h>
+
+#include <execinfo.h>
+
+#ifdef DAOS_WITH_REF_TRACKER
+
+static void
+dss_ref_tracker_dumper_ult(void *arg)
+{
+	struct dss_ref_tracker_dumper *dumper = arg;
+	int                            n;
+
+	for (n = 0; !dss_ult_exiting(dumper->rftd_req); n++) {
+		if (n % 10 == 0)
+			d_ref_tracker_dump(dumper->rftd_tracker, dumper->rftd_func,
+					   dumper->rftd_line);
+		sched_req_sleep(dumper->rftd_req, 1000 /* ms */);
+	}
+}
+
+/** Use DSS_REF_TRACKER_INIT_DUMPER instead. */
+void
+dss_ref_tracker_init_dumper(struct dss_ref_tracker_dumper *dumper, struct d_ref_tracker *tracker,
+			     const char *func, int line)
+{
+	uuid_t                anonym_uuid;
+	struct sched_req_attr attr;
+
+	uuid_clear(anonym_uuid);
+	sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
+	dumper->rftd_req = sched_create_ult(&attr, dss_ref_tracker_dumper_ult, dumper, 0);
+	D_ASSERT(dumper->rftd_req != NULL);
+
+	dumper->rftd_tracker = tracker;
+	dumper->rftd_func    = func;
+	dumper->rftd_line    = line;
+}
+
+/** Use DSS_REF_TRACKER_FINI_DUMPER instead. */
+void
+dss_ref_tracker_fini_dumper(struct dss_ref_tracker_dumper *dumper)
+{
+	sched_req_wait(dumper->rftd_req, true);
+	sched_req_put(dumper->rftd_req);
+}
+
+#endif /* DAOS_WITH_REF_TRACKER */

src/gurt/misc.c

@@ -20,8 +20,7 @@
 #include <pthread.h>
 #include <limits.h>
 #include <stdint.h>
-
-#include <malloc.h>
+#include <execinfo.h>
 
 #include <gurt/common.h>
 #include <gurt/atomic.h>
@@ -1718,53 +1717,170 @@ d_stand_div(double *array, int nr)
 	return sqrt(std);
 }
 
-int
-d_vec_pointers_init(struct d_vec_pointers *pointers, uint32_t cap)
+D_VEC_DEFINE(d_vec_pointers, void *, elem, D_)
+
+#ifdef DAOS_WITH_REF_TRACKER
+
+/* Reference tracker record */
+struct d_ref_tracker_rec {
+	void       *rftr_addr;
+	uint64_t    rftr_time;
+	const char *rftr_func;
+	int         rftr_line;
+	int         rftr_user;
+	uint64_t    rftr_thread;
+	void       *rftr_trace[5];
+};
+
+D_VEC_DEFINE(d_vec_ref_tracker_rec, struct d_ref_tracker_rec, *elem, D_VEC_A_)
+
+/** Initialize \a tracker. */
+void
+d_ref_tracker_init(struct d_ref_tracker *tracker, d_ref_tracker_cb_get_time_t get_time,
+		   d_ref_tracker_cb_get_thread_t get_thread)
 {
-	void **buf = NULL;
+	int rc;
 
-	if (cap > 0) {
-		D_ALLOC_ARRAY(buf, cap);
-		if (buf == NULL)
-			return -DER_NOMEM;
+	tracker->rft_get_time   = get_time;
+	tracker->rft_get_thread = get_thread;
+
+	rc = d_vec_ref_tracker_rec_init(&tracker->rft_vec, 0);
+	D_ASSERTF(rc == 0, "d_vec_ref_tracker_rec_init(0): " DF_RC "\n", DP_RC(rc));
+}
+
+/** Finalize \a tracker. */
+void
+d_ref_tracker_fini(struct d_ref_tracker *tracker)
+{
+	D_REF_TRACKER_DUMP(tracker);
+	D_ASSERT(tracker->rft_vec.p_len == 0);
+	d_vec_ref_tracker_rec_fini(&tracker->rft_vec);
+}
+
+/** Use D_REF_TRACKER_DUMP instead. */
+void
+d_ref_tracker_dump(struct d_ref_tracker *tracker, const char *func, int line)
+{
+	int i;
+
+	if (tracker->rft_vec.p_len > 0)
+		D_INFO("%s@%d: dumping %d references:\n", func, line, tracker->rft_vec.p_len);
+	for (i = 0; i < tracker->rft_vec.p_len; i++) {
+		struct d_ref_tracker_rec *ref = &tracker->rft_vec.p_buf[i];
+		char **symbols;
+		int j;
+
+		D_INFO("  %p: %s@%d at " DF_X64 " (%d) by " DF_U64 "\n", ref->rftr_addr, ref->rftr_func,
+		       ref->rftr_line, ref->rftr_time, ref->rftr_user, ref->rftr_thread);
+		symbols = backtrace_symbols(ref->rftr_trace, ARRAY_SIZE(ref->rftr_trace));
+		D_ASSERTF(symbols != NULL, "backtrace_symbols failed\n");
+		for (j = 0; j < ARRAY_SIZE(ref->rftr_trace); j++)
+			D_INFO("    %s\n", symbols[j]);
+		free(symbols);
 	}
+}
 
-	pointers->p_buf = buf;
-	pointers->p_cap = cap;
-	pointers->p_len = 0;
-	return 0;
+static int
+d_ref_tracker_find(struct d_ref_tracker *tracker, void *addr)
+{
+	int i;
+
+	for (i = 0; i < tracker->rft_vec.p_len; i++)
+		if (tracker->rft_vec.p_buf[i].rftr_addr == addr)
+			return i;
+	return -1;
 }
 
+/** Use D_REF_TRACKER_TRACK instead. */
 void
-d_vec_pointers_fini(struct d_vec_pointers *pointers)
+d_ref_tracker_track(struct d_ref_tracker *tracker, void *addr, const char *func, int line)
 {
-	D_FREE(pointers->p_buf);
-	pointers->p_cap = 0;
-	pointers->p_len = 0;
+	struct d_ref_tracker_rec ref = {.rftr_addr = addr, .rftr_func = func, .rftr_line = line};
+	int                      i;
+	int                      rc;
+
+	D_ASSERT(addr != NULL);
+
+	ref.rftr_time   = tracker->rft_get_time();
+	ref.rftr_thread = tracker->rft_get_thread();
+	backtrace(ref.rftr_trace, ARRAY_SIZE(ref.rftr_trace));
+
+	/*
+	 * If the address is already used by an existing reference, then the
+	 * usual reason is that the variable storing the reference was
+	 * destructed without a d_ref_tracker_untrack call---we have very likely leaked
+	 * that reference.
+	 */
+	i = d_ref_tracker_find(tracker, addr);
+	D_ASSERTF(i == -1, "addr %p already used by %s@%d\n", addr,
+		  tracker->rft_vec.p_buf[i].rftr_func, tracker->rft_vec.p_buf[i].rftr_line);
+
+	rc = d_vec_ref_tracker_rec_append(&tracker->rft_vec, &ref);
+	D_ASSERTF(rc == 0, "d_vec_ref_tracker_rec_append: " DF_RC "\n", DP_RC(rc));
 }
 
-int
-d_vec_pointers_append(struct d_vec_pointers *pointers, void *pointer)
+/** Use D_REF_TRACKER_UNTRACK instead. */
+void
+d_ref_tracker_untrack(struct d_ref_tracker *tracker, void *addr)
 {
-	if (pointers->p_len == pointers->p_cap) {
-		void		**buf;
-		uint32_t	  cap;
+	int i;
+	int rc;
 
-		if (pointers->p_cap == 0)
-			cap = 1;
-		else
-			cap = 2 * pointers->p_cap;
+	D_ASSERT(addr != NULL);
 
-		D_REALLOC_ARRAY(buf, pointers->p_buf, pointers->p_cap, cap);
-		if (buf == NULL)
-			return -DER_NOMEM;
+	/*
+	 * If the address is not found, then the usual reason is that we have
+	 * copied a reference from the original variable to a new one without a
+	 * d_ref_tracker_retrack call.
+	 */
+	i = d_ref_tracker_find(tracker, addr);
+	D_ASSERTF(i != -1, "addr not found\n");
 
-		pointers->p_buf = buf;
-		pointers->p_cap = cap;
-	}
+	rc = d_vec_ref_tracker_rec_delete_at(&tracker->rft_vec, i,
+					     D_VEC_F_REORDER | D_VEC_F_SHRINK);
+	D_ASSERTF(rc == 0, "d_vec_ref_tracker_rec_delete_at: " DF_RC "\n", DP_RC(rc));
+}
 
-	D_ASSERTF(pointers->p_len < pointers->p_cap, "%u < %u\n", pointers->p_len, pointers->p_cap);
-	pointers->p_buf[pointers->p_len] = pointer;
-	pointers->p_len++;
-	return 0;
+/** Use D_REF_TRACKER_MOVE instead. */
+void
+d_ref_tracker_retrack(struct d_ref_tracker *tracker, void *new_addr, void *old_addr,
+		      const char *func, int line)
+{
+	int                       i;
+	int                       j;
+	struct d_ref_tracker_rec *ref;
+	int rc;
+
+	D_ASSERT(old_addr != NULL);
+	D_ASSERT(new_addr != NULL);
+	D_ASSERT(old_addr != new_addr);
+
+	i = d_ref_tracker_find(tracker, old_addr);
+	D_ASSERTF(i != -1, "old_addr not found\n");
+
+	j = d_ref_tracker_find(tracker, new_addr);
+	D_ASSERTF(j == -1, "new_addr %p already used by %s@%d\n", new_addr,
+		  tracker->rft_vec.p_buf[j].rftr_func, tracker->rft_vec.p_buf[j].rftr_line);
+
+	ref            = &tracker->rft_vec.p_buf[i];
+	ref->rftr_addr = new_addr;
+	ref->rftr_time   = tracker->rft_get_time();
+	ref->rftr_func = func;
+	ref->rftr_line = line;
+	ref->rftr_thread = tracker->rft_get_thread();
+	rc = backtrace(ref->rftr_trace, ARRAY_SIZE(ref->rftr_trace));
+	for (i = rc; i < ARRAY_SIZE(ref->rftr_trace); i++)
+		ref->rftr_trace[i] = NULL;
 }
+
+void
+d_ref_tracker_set_user(struct d_ref_tracker *tracker, void *addr, int user)
+{
+	int i;
+
+	i = d_ref_tracker_find(tracker, addr);
+	D_ASSERTF(i != -1, "addr not found\n");
+	tracker->rft_vec.p_buf[i].rftr_user = user;
+}
+
+#endif /* DAOS_WITH_REF_TRACKER */