initial HCU DCGM

.gitignore

@@ -3,7 +3,6 @@
 /dcgmi
 /dcgmi-*
 /hcu-dcgm
-/dcgm-dcu
 samples/*/test
 pkg/cmd/dcgmi
 pkg/service/dcgm

deployment/hcu-dcgm-docker.sh

@@ -1,18 +1,15 @@
-#!/bin/bash
-# Copyright (c) 2026 Hygon Information Technology Co., Ltd.
-# SPDX-License-Identifier: Apache-2.0
-
-mkdir -p /etc/vdev
-
-docker run --name hcu-dcgm -d --privileged \
-  --device=/dev/kfd \
-  --device=/dev/mkfd \
-  --device=/dev/dri \
-  -v /etc/vdev:/etc/vdev \
-  -v /etc/hostname:/etc/hostname \
-  -v /etc/vdev:/etc/vdev \
-  -v /opt/hyhal:/opt/hyhal \
-  -v /home/chengdm/config:/home/dcgm/config \
-  -p 16081:16081 \
-  -e LD_LIBRARY_PATH="/opt/hyhal/lib" \
-  hcu-dcgm:v2.0.0
+#!/bin/bash
+# Copyright (c) 2026 Hygon Information Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+mkdir -p /etc/vdev
+
+docker run --name hcu-dcgm -d --privileged \
+  --device=/dev/kfd \
+  --device=/dev/mkfd \
+  --device=/dev/dri \
+  -v /etc/vdev:/etc/vdev \
+  -v /opt/hyhal:/opt/hyhal \
+  -p 16081:16081 \
+  image.sourcefind.cn:5000/dcu/admin/base/dcu-dcgm:v2.1.0 \
+  /bin/bash -c "/usr/local/bin/start-dcgm.sh"

deployment/hcu-dcgm-k8s.yaml

@@ -1,92 +1,75 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: hcu-dcgm
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      app: hcu-dcgm
-  template:
-    metadata:
-      labels:
-        app: hcu-dcgm
-    spec:
-      hostNetwork: true
-      containers:
-        - name: hcu-dcgm
-          image: hcu-dcgm:v1.0.0
-          imagePullPolicy: IfNotPresent  # 如果节点上已有镜像就不会再拉取
-          securityContext:
-            privileged: true # 必须开启特权模式以允许容器访问设备
-          ports:
-            - name: dcgm-port
-              containerPort: 16081
-              hostPort: 16081
-          volumeMounts:
-            - name: kfd
-              mountPath: /dev/kfd
-            - name: mkfd
-              mountPath: /dev/mkfd
-            - name: dri
-              mountPath: /dev/dri
-            - name: vdev
-              mountPath: /etc/vdev
-            - name: hostname
-              mountPath: /etc/hostname
-            - name: hyhal
-              mountPath: /opt/hyhal
-            - name: dtk
-              mountPath: /opt/dtk
-            - name: usr-hyhal
-              mountPath: /usr/local/hyhal
-            - name: home-config
-              mountPath: /home/dcgm/config
-          env:
-            - name: LD_LIBRARY_PATH
-              value: "/opt/hyhal/lib"
-      volumes:
-        - name: kfd
-          hostPath:
-            path: /dev/kfd
-        - name: mkfd
-          hostPath:
-            path: /dev/mkfd
-        - name: dri
-          hostPath:
-            path: /dev/dri
-        - name: vdev
-          hostPath:
-            path: /etc/vdev
-        - name: hostname
-          hostPath:
-            path: /etc/hostname
-        - name: hyhal
-          hostPath:
-            path: /opt/hyhal
-        - name: dtk
-          hostPath:
-            path: /opt/dtk
-        - name: usr-hyhal
-          hostPath:
-            path: /usr/local/hyhal
-        - name: home-config
-          hostPath:
-            path: /home/dcgm/config # 根据宿主机实际路径配置
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: hcu-dcgm
-  namespace: kube-system
-spec:
-  selector:
-    app: hcu-dcgm
-  ports:
-    - name: dcgm-port
-      port: 16081
-      targetPort: 16081
-      protocol: TCP
-  type: NodePort
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: hcu-dcgm
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: hcu-dcgm
+  template:
+    metadata:
+      labels:
+        app: hcu-dcgm
+    spec:
+      hostNetwork: true
+      containers:
+        - name: hcu-dcgm
+          image: image.sourcefind.cn:5000/dcu/admin/base/dcu-dcgm:v2.1.0
+          imagePullPolicy: IfNotPresent  # 如果节点上已有镜像就不会再拉取
+          command: ["/bin/bash", "-c"]
+          args:
+            - /usr/local/bin/start-dcgm.sh
+          securityContext:
+            privileged: true # 必须开启特权模式以允许容器访问设备
+          ports:
+            - name: dcgm-port
+              containerPort: 16081
+              hostPort: 16081
+          volumeMounts:
+            - name: kfd
+              mountPath: /dev/kfd
+            - name: mkfd
+              mountPath: /dev/mkfd
+            - name: dri
+              mountPath: /dev/dri
+            - name: vdev
+              mountPath: /etc/vdev
+            - name: hyhal
+              mountPath: /opt/hyhal
+          env:
+            - name: LD_LIBRARY_PATH
+              value: "/opt/hyhal/lib"
+      volumes:
+        - name: kfd
+          hostPath:
+            path: /dev/kfd
+        - name: mkfd
+          hostPath:
+            path: /dev/mkfd
+        - name: dri
+          hostPath:
+            path: /dev/dri
+        - name: vdev
+          hostPath:
+            path: /etc/vdev
+        - name: hyhal
+          hostPath:
+            path: /opt/hyhal
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: hcu-dcgm
+  namespace: kube-system
+spec:
+  selector:
+    app: hcu-dcgm
+  ports:
+    - name: dcgm-port
+      port: 16081
+      targetPort: 16081
+      protocol: TCP
+  type: NodePort

pkg/cmd/cli/diag.go

@@ -92,7 +92,11 @@ func formatGEMMSummary(result dcgm.TargetStressResult) string {
 		metrics := make([]string, 0, len(gemmResults))
 		for _, gemm := range gemmResults {
 			if gemm.Failed {
-				metrics = append(metrics, fmt.Sprintf("%s=FAIL", gemm.GemmName))
+				if gemm.Mean > 0 {
+					metrics = append(metrics, fmt.Sprintf("%s=%.2f(FAIL)", gemm.GemmName, gemm.Mean))
+				} else {
+					metrics = append(metrics, fmt.Sprintf("%s=FAIL", gemm.GemmName))
+				}
 				continue
 			}
 			passed++
@@ -104,6 +108,11 @@ func formatGEMMSummary(result dcgm.TargetStressResult) string {
 			status = "WARN"
 		}
 		fmt.Fprintf(&output, "HCU %d  %s %d/%d  mean: %s\n", hcuID, status, passed, len(gemmResults), strings.Join(metrics, ", "))
+		for _, gemm := range gemmResults {
+			if gemm.Error != "" {
+				fmt.Fprintf(&output, "  [%s] error: %s\n", gemm.GemmName, gemm.Error)
+			}
+		}
 	}
 	return output.String()
 }

pkg/cmd/cli/diag_test.go

@@ -101,6 +101,25 @@ func TestFormatGEMMSummary(t *testing.T) {
 	}
 }
 
+func TestFormatGEMMSummary_FailWithMeanAndError(t *testing.T) {
+	output := formatGEMMSummary(dcgm.TargetStressResult{
+		Results: []dcgm.GemmTestResult{
+			{HCUId: 0, GemmName: "hgemm", Mean: 95.50, Failed: true, Error: "子进程异常退出 exit status 1"},
+			{HCUId: 0, GemmName: "sgemm", Mean: 40.10},
+		},
+	})
+
+	for _, want := range []string{
+		"hgemm=95.50(FAIL)",
+		"sgemm=40.10",
+		"[hgemm] error: 子进程异常退出 exit status 1",
+	} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("output %q does not contain %q", output, want)
+		}
+	}
+}
+
 func TestDiagnosticStatusLabel(t *testing.T) {
 	if got := diagnosticStatusLabel(dcgm.DiagResultWarn); got != "Warning" {
 		t.Fatalf("warning label = %q, want Warning", got)

pkg/dcgm/admin.go

@@ -6,7 +6,7 @@ package dcgm
 
 /*
 #cgo CFLAGS: -Wall -I./include
-#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdint.h>
 #include <kfd_ioctl.h>
 #include <rocm_smi64Config.h>

pkg/dcgm/api.go

@@ -2783,10 +2783,6 @@ func ShowPcieReplayCount(dvIdList []int) (pcieReplayCountInfos []PcieReplayCount
 	return
 }
 
-// 获取指定进程的进程信息
-func ProcessInfo(pid int) (proc ProcessInfos, err error) {
-	return rsmiComputeProcessInfoByPidGet(pid)
-}
 
 // 获取指定设备上的进程信息
 func ProcessInfoByDevice(pid int, dvInd int) (proc ProcessInfos, err error) {
@@ -4925,10 +4921,17 @@ func ComputeProcessInfoGet() (processInfo []ProcessInfos, numItems int, err erro
 	return rsmiComputeProcessInfoGet()
 }
 
+// ProcessInfoByPid 根据PID获取进程信息
 func ProcessInfoByPid(pid uint32) (RsmiProcessInfoV2, error) {
 	return getProcessInfoByPID(pid)
 }
 
+// 获取指定进程的进程信息
+func ProcessInfo(pid int) (proc ProcessInfos, err error) {
+	return rsmiComputeProcessInfoByPidGet(pid)
+}
+
+
 func DeviceGetCount() (deviceCount int, err error) {
 	return nvmlDeviceGetCount()
 }

pkg/dcgm/device_info.go

@@ -6,7 +6,7 @@ package dcgm
 
 /*
 #cgo CFLAGS: -Wall -I./include
-#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>

pkg/dcgm/device_status.go

@@ -6,7 +6,7 @@ package dcgm
 
 /*
 #cgo CFLAGS: -Wall -I./include
-#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdint.h>
 #include <kfd_ioctl.h>
 #include <rocm_smi64Config.h>

pkg/dcgm/diag.go

@@ -512,11 +512,16 @@ func runStressTests(level int) (DiagResults, error) {
 				if r.Failed || r.Mean <= 0 {
 					status = DiagResultWarn
 				}
+				errCode := 0
+				if r.Error != "" {
+					errCode = -1
+				}
 				dr := DiagResult{
-					Status:     status,
-					TestName:   fmt.Sprintf("TargetStress GEMM %s", r.GemmName),
-					TestOutput: fmt.Sprintf("GEMM=%s, Mean=%.3f", r.GemmName, r.Mean),
-					ErrorCode:  0,
+					Status:       status,
+					TestName:     fmt.Sprintf("TargetStress GEMM %s", r.GemmName),
+					TestOutput:   fmt.Sprintf("GEMM=%s, Mean=%.3f", r.GemmName, r.Mean),
+					ErrorCode:    errCode,
+					ErrorMessage: r.Error,
 				}
 				merged := false
 				for i := range aggregated.PerHCU {

pkg/dcgm/edpp.go

@@ -59,12 +59,11 @@ const (
 	edppStressLoopCount = "1900000000"
 	edppStressDuration  = 10 * time.Second
 
-	// BW1000 空闲功耗约 86-105W，满载约 1000W+；设 300W 为最低门槛，
-	// 压测启动后功耗应迅速超过该值，空闲状态绝对无法触及。
-	edppBMZMinAvgPowerW = 300
-	// BW1100/NMZ 实测空闲约 170W，EDPp 压力约 800W且不超过 1000W；
-	// 以约 400W 作为进入计算状态的保守门槛，避免依赖 loader 不输出的 TFLOPS 文本。
-	edppNMZMinAvgPowerW = 400
+	// BMZ 系列统一使用 200W 作为最低门槛；BW100 实测压测平均功耗约 231W，
+	// 该阈值可避免将已通过 GEMM CPU CHECK 的有效计算误判为空载。
+	edppBMZMinAvgPowerW = 200
+	// NMZ 系列统一使用 300W 作为进入计算状态的最低门槛。
+	edppNMZMinAvgPowerW = 300
 )
 
 type edppBackendSpec struct {

pkg/dcgm/edpp_test.go

@@ -71,6 +71,14 @@ func TestEdppBackendSpecsUseStressMode(t *testing.T) {
 	if bmz.coName != "fp16_bmz_edpp.co" || bmz.minAvgPowerW != edppBMZMinAvgPowerW {
 		t.Fatalf("bmz spec = %+v, want fp16 co and bmz threshold", bmz)
 	}
+
+	nmz, err = edppBackendSpecFor(edppBackendNMZGFX938)
+	if err != nil {
+		t.Fatalf("NMZ spec error: %v", err)
+	}
+	if nmz.minAvgPowerW != edppNMZMinAvgPowerW {
+		t.Fatalf("NMZ threshold = %d, want %d", nmz.minAvgPowerW, edppNMZMinAvgPowerW)
+	}
 }
 
 func TestFinalizeStressEdppResult(t *testing.T) {
@@ -97,8 +105,8 @@ func TestFinalizeStressEdppResult(t *testing.T) {
 	}
 
 	result = GemmPowerEdppResult{HCU: 2, Backend: spec.name}
-	err = finalizeStressEdppResult(&result, spec, edppStressSamples{totalPower: 740, peakPower: 370, powerSamples: 2})
-	if err == nil || !strings.Contains(result.Error, "avg power 370W < threshold 400W") {
+	err = finalizeStressEdppResult(&result, spec, edppStressSamples{totalPower: 580, peakPower: 290, powerSamples: 2})
+	if err == nil || !strings.Contains(result.Error, "avg power 290W < threshold 300W") {
 		t.Fatalf("low power err = %v, result = %+v", err, result)
 	}
 }