DAOS-18857 build: Support of Gperftools Heap Profiler

ci/unit/required_packages.sh

@@ -36,6 +36,7 @@ pkgs="boost-python3$PY_MINOR_VER-devel                               \
       $(utils/rpms/package_version.sh pmdk debug pmem)               \
       fuse3                                                          \
       gotestsum                                                      \
+      gperftools-devel                                               \
       hwloc-devel                                                    \
       libasan                                                        \
       libipmctl-devel                                                \

docs/dev/development.md

@@ -173,6 +173,32 @@ ASan support in DAOS is still experimental and has the following known issues:
    ASan is not fully integrated with the DAOS regression testing framework. Some tests, such as
    those using Valgrind, may fail due to false positives.
 
+### Heap Profiler
+
+To debug memory leaks which are properly deleted when the application stop, the
+[Gperftools Heap Profiler](https://gperftools.github.io/gperftools/heapprofile.html) can be used.
+This heap profiler mostly rely on the [TCMalloc](https://github.com/google/tcmalloc) memory
+allocator.  This memory allocator can either be linked to DAOS executables or loaded at runtime
+thanks to `LD_PRELOAD`.
+
+To profile heap allocation of the `daos_engine` with `LD_PRELOAD`, the following entries can be
+added to the `env_vars` section of the `daos_server.yml` configuration file:
+```yaml
+engines:
+- ...
+  env_vars:
+  ...
+  - LD_PRELOAD=/usr/lib64/libtcmalloc.so.4
+  - HEAPPROFILE=/var/daos/hprof/daos_engine.1.hprof
+  - HEAP_PROFILE_INUSE_INTERVAL=1073741824
+  - HEAP_PROFILE_ALLOCATION_INTERVAL=0
+  - HEAP_PROFILE_TIME_INTERVAL=0
+```
+
+To link DAOS executabls with the TCMalloc memory allocator, use the `HEAP_PROFILER=true` flag with
+the `scons` command.  To profile the daos engien heap allocation, the same `daos_server.yml`
+configuration can be used without setting the `LD_PRELOAD` environment variable.
+
 ## Go dependencies
 
 Developers contributing Go code may need to change the external dependencies

site_scons/prereq_tools/base.py

@@ -517,6 +517,8 @@ def __init__(self, env, opts):
                               ['warning', 'warn', 'error'], ignorecase=2))
         opts.Add(('SANITIZERS', 'Instrument C code with google sanitizers', None))
         opts.Add(BoolVariable('CMOCKA_FILTER_SUPPORTED', 'Allows to filter cmocka tests', False))
+        opts.Add(BoolVariable('HEAP_PROFILER', 'Instrument C code with Gperftools Heap Profiler',
+                              False))
 
         opts.Update(self.__env)
 

site_scons/site_tools/compiler_setup.py

@@ -59,6 +59,11 @@ def _base_setup(env):
     env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS)
 
     if 'SANITIZERS' in env and env['SANITIZERS'] != "":
+
+        if "HEAP_PROFILER" in env and env['HEAP_PROFILER']:
+            print('Google Sanitizers and Gperftools.Heap.Profiler can not be mixed')
+            Exit(2)
+
         cc = 'gcc'
         if 'COMPILER' in env:
             cc = env['COMPILER']
@@ -87,6 +92,10 @@ def _base_setup(env):
                 env.AppendUnique(LINKFLAGS=flag)
                 print(f"Enabling {flag.split('=')[1]} sanitizer for C code")
 
+    if 'HEAP_PROFILER' in env and env['HEAP_PROFILER']:
+        env.AppendUnique(LINKFLAGS="-ltcmalloc")
+        print("Enabling Gperftools Heap Profiler")
+
     if '-Wmismatched-dealloc' in env['CCFLAGS']:
         env.AppendUnique(CPPDEFINES={'HAVE_DEALLOC': '1'})
 
@@ -222,6 +231,10 @@ def _check_func(env, func_name):
             denv["CCFLAGS"].remove(flag)
             denv["LINKFLAGS"].remove(flag)
 
+    # NOTE Remove Heap Profiler to not scramble the test output
+    if 'HEAP_PROFILER' in denv and denv['HEAP_PROFILER']:
+        denv["LINKFLAGS"].remove("-ltcmalloc")
+
     config = Configure(denv)
     res = config.CheckFunc(func_name)
     config.Finish()

utils/rpms/daos.changelog

@@ -1,4 +1,7 @@
 %changelog
+* Mon May 04 2026 Cedric Koch-Hofer <cedric.koch-hofer@hpe.com> 2.9.100-4
+- Add gperftools-devel as a build dependency of daos-devel
+
 * Fri Apr 24 2026 Tomasz Gromadzki <tomasz.gromadzki@hpe.com> 2.9.100-3
 - Update PMDK to version 2.1.3-2
 

utils/rpms/daos.sh

@@ -399,7 +399,7 @@ TARGET_PATH="${daoshome}/python"
 list_files files "${SL_PREFIX}/lib/daos/python/*"
 append_install_list "${files[@]}"
 
-EXTERNAL_DEPENDS=("${uuid_lib}")
+EXTERNAL_DEPENDS=("${uuid_lib}" "${gperftools_dev}")
 DEPENDS=("daos-client = ${VERSION}-${RELEASE}")
 build_package "${daos_dev}"
 

utils/rpms/daos.spec

@@ -24,7 +24,7 @@
 
 Name:          daos
 Version:       2.9.100
-Release:       3%{?relval}%{?dist}
+Release:       4%{?relval}%{?dist}
 Summary:       DAOS Storage Engine
 
 License:       BSD-2-Clause-Patent

utils/rpms/package_info.sh

@@ -129,6 +129,9 @@ export daos_dev
 set_lib_name uuid lib libuuid libuuid1 libuuid1
 export uuid_lib
 
+set_lib_name gperftools dev gperftools gperftools libgoogle-perftools
+export gperftools_dev
+
 set_lib_name hdf5 lib hdf5 hdf5 hdf5
 export hdf5_lib
 

utils/scripts/install-el8.sh

@@ -37,6 +37,7 @@ dnf --nodocs install ${dnf_install_args} \
     git \
     glibc-langpack-en \
     golang \
+    gperftools-devel \
     graphviz \
     help2man \
     hdf5-devel \

utils/scripts/install-el9.sh

@@ -35,6 +35,7 @@ dnf --nodocs install ${dnf_install_args} \
     git \
     glibc-langpack-en \
     golang \
+    gperftools-devel \
     graphviz \
     help2man \
     hdf5-devel \

utils/scripts/install-leap15.sh

@@ -32,6 +32,7 @@ dnf --nodocs install ${dnf_install_args} \
     git \
     go \
     go-race \
+    gperftools-devel \
     graphviz \
     gzip \
     hdf5-devel \

utils/scripts/install-ubuntu.sh

@@ -37,6 +37,7 @@ apt-get install ${apt_get_install_args} \
     libcunit1-dev \
     libdaxctl-dev \
     libfuse3-dev \
+    libgoogle-perftools-dev \
     libhwloc-dev \
     libibverbs-dev \
     libjson-c-dev \