feat: Add NVIDIA DCGM installation

README.md

@@ -115,6 +115,18 @@ Default: `true`
 
 Type: `bool`
 
+### hpc_install_nvidia_dcgm
+
+Whether to install the NVIDIA datacenter GPU manager(DCGM) and  enable its nvidia-dcgm service.
+
+NVIDIA DCGM is a GPU monitoring and management toolkit for large-scale GPU deployments, install DCGM on all GPU nodes in an HPC cluster to maintain reliability and monitor GPU health.
+
+Run `dcgmi` in the GPU nodes, e.g. `dcgmi discovery -l` to list GPUs on the node.
+
+Default: `true`
+
+Type: `bool`
+
 ### hpc_install_rdma
 
 Whether to install the NVIDIA RDMA package.

defaults/main.yml

@@ -21,6 +21,7 @@ hpc_install_cuda_toolkit: true
 hpc_install_hpc_nvidia_nccl: true
 hpc_install_nvidia_fabric_manager: true
 hpc_install_nvidia_imex: true
+hpc_install_nvidia_dcgm: true
 hpc_install_rdma: true
 hpc_enable_azure_persistent_rdma_naming: true
 hpc_azure_disable_predictable_net_names: true

tasks/main.yml

@@ -593,6 +593,23 @@
                 value: "0"
                 state: present
 
+- name: Install NVIDIA datacenter GPU manager enable service
+  when: hpc_install_nvidia_dcgm
+  block:
+    - name: Install NVIDIA datacenter GPU manager
+      package:
+        name: "{{ __hpc_nvidia_dcgm }}"
+        state: present
+        use: "{{ (__hpc_server_is_ostree | d(false)) |
+          ternary('ansible.posix.rhel_rpm_ostree', omit) }}"
+      register: __hpc_nvidia_dcgm_install
+      until: __hpc_nvidia_dcgm_install is success
+
+    - name: Ensure dcgm service is enabled
+      service:
+        name: nvidia-dcgm
+        enabled: true
+
 - name: Install RDMA packages
   when: hpc_install_rdma
   block:

vars/RedHat_9.yml

@@ -34,6 +34,8 @@
 __hpc_docker_packages:
   - moby-engine-29.1.4-1.el9
   - moby-cli-29.1.4-1.el9
+__hpc_nvidia_dcgm:
+  - datacenter-gpu-manager-4-cuda12
 
 # Vars related to building packages from source
 __hpc_gdrcopy_info:
@@ -75,4 +77,4 @@
 __hpc_kvp_client_info:
   name: kvp_client.c
   sha256: 0f7fa598c40994e9a18fa2982ee6ca9a591bd2ebc419f1ec526ea9bf19aae1eb
   url: https://raw.githubusercontent.com/microsoft/lis-test/master/WS2012R2/lisa/tools/KVP/kvp_client.c