diff --git a/.gitignore b/.gitignore index 22960a6..32e45aa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ /dcgmi /dcgmi-* /hcu-dcgm -/dcgm-dcu samples/*/test pkg/cmd/dcgmi pkg/service/dcgm diff --git a/deployment/hcu-dcgm-docker.sh b/deployment/hcu-dcgm-docker.sh index 7a38522..0d63a66 100644 --- a/deployment/hcu-dcgm-docker.sh +++ b/deployment/hcu-dcgm-docker.sh @@ -1,18 +1,15 @@ -#!/bin/bash -# Copyright (c) 2026 Hygon Information Technology Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -mkdir -p /etc/vdev - -docker run --name hcu-dcgm -d --privileged \ - --device=/dev/kfd \ - --device=/dev/mkfd \ - --device=/dev/dri \ - -v /etc/vdev:/etc/vdev \ - -v /etc/hostname:/etc/hostname \ - -v /etc/vdev:/etc/vdev \ - -v /opt/hyhal:/opt/hyhal \ - -v /home/chengdm/config:/home/dcgm/config \ - -p 16081:16081 \ - -e LD_LIBRARY_PATH="/opt/hyhal/lib" \ - hcu-dcgm:v2.0.0 +#!/bin/bash +# Copyright (c) 2026 Hygon Information Technology Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +mkdir -p /etc/vdev + +docker run --name hcu-dcgm -d --privileged \ + --device=/dev/kfd \ + --device=/dev/mkfd \ + --device=/dev/dri \ + -v /etc/vdev:/etc/vdev \ + -v /opt/hyhal:/opt/hyhal \ + -p 16081:16081 \ + image.sourcefind.cn:5000/dcu/admin/base/dcu-dcgm:v2.1.0 \ + /bin/bash -c "/usr/local/bin/start-dcgm.sh" diff --git a/deployment/hcu-dcgm-k8s.yaml b/deployment/hcu-dcgm-k8s.yaml index 6e5f5de..12a8587 100644 --- a/deployment/hcu-dcgm-k8s.yaml +++ b/deployment/hcu-dcgm-k8s.yaml @@ -1,92 +1,75 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: hcu-dcgm - namespace: kube-system -spec: - selector: - matchLabels: - app: hcu-dcgm - template: - metadata: - labels: - app: hcu-dcgm - spec: - hostNetwork: true - containers: - - name: hcu-dcgm - image: hcu-dcgm:v1.0.0 - imagePullPolicy: IfNotPresent # 如果节点上已有镜像就不会再拉取 - securityContext: - privileged: true # 必须开启特权模式以允许容器访问设备 - ports: - - name: dcgm-port - containerPort: 16081 - hostPort: 16081 - volumeMounts: - - name: kfd - mountPath: /dev/kfd - - name: mkfd - mountPath: /dev/mkfd - - name: dri - mountPath: /dev/dri - - name: vdev - mountPath: /etc/vdev - - name: hostname - mountPath: /etc/hostname - - name: hyhal - mountPath: /opt/hyhal - - name: dtk - mountPath: /opt/dtk - - name: usr-hyhal - mountPath: /usr/local/hyhal - - name: home-config - mountPath: /home/dcgm/config - env: - - name: LD_LIBRARY_PATH - value: "/opt/hyhal/lib" - volumes: - - name: kfd - hostPath: - path: /dev/kfd - - name: mkfd - hostPath: - path: /dev/mkfd - - name: dri - hostPath: - path: /dev/dri - - name: vdev - hostPath: - path: /etc/vdev - - name: hostname - hostPath: - path: /etc/hostname - - name: hyhal - hostPath: - path: /opt/hyhal - - name: dtk - hostPath: - path: /opt/dtk - - name: usr-hyhal - hostPath: - path: /usr/local/hyhal - - name: home-config - hostPath: - path: /home/dcgm/config # 根据宿主机实际路径配置 - ---- - -apiVersion: v1 -kind: Service -metadata: - name: hcu-dcgm - namespace: kube-system -spec: - selector: - app: hcu-dcgm - ports: - - name: dcgm-port - port: 16081 - targetPort: 16081 - protocol: TCP - type: NodePort +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hcu-dcgm + namespace: kube-system +spec: + selector: + matchLabels: + app: hcu-dcgm + template: + metadata: + labels: + app: hcu-dcgm + spec: + hostNetwork: true + containers: + - name: hcu-dcgm + image: image.sourcefind.cn:5000/dcu/admin/base/dcu-dcgm:v2.1.0 + imagePullPolicy: IfNotPresent # 如果节点上已有镜像就不会再拉取 + command: ["/bin/bash", "-c"] + args: + - /usr/local/bin/start-dcgm.sh + securityContext: + privileged: true # 必须开启特权模式以允许容器访问设备 + ports: + - name: dcgm-port + containerPort: 16081 + hostPort: 16081 + volumeMounts: + - name: kfd + mountPath: /dev/kfd + - name: mkfd + mountPath: /dev/mkfd + - name: dri + mountPath: /dev/dri + - name: vdev + mountPath: /etc/vdev + - name: hyhal + mountPath: /opt/hyhal + env: + - name: LD_LIBRARY_PATH + value: "/opt/hyhal/lib" + volumes: + - name: kfd + hostPath: + path: /dev/kfd + - name: mkfd + hostPath: + path: /dev/mkfd + - name: dri + hostPath: + path: /dev/dri + - name: vdev + hostPath: + path: /etc/vdev + - name: hyhal + hostPath: + path: /opt/hyhal + +--- + +apiVersion: v1 +kind: Service +metadata: + name: hcu-dcgm + namespace: kube-system +spec: + selector: + app: hcu-dcgm + ports: + - name: dcgm-port + port: 16081 + targetPort: 16081 + protocol: TCP + type: NodePort diff --git a/pkg/cmd/cli/diag.go b/pkg/cmd/cli/diag.go index 215ea0e..f510e61 100644 --- a/pkg/cmd/cli/diag.go +++ b/pkg/cmd/cli/diag.go @@ -92,7 +92,11 @@ func formatGEMMSummary(result dcgm.TargetStressResult) string { metrics := make([]string, 0, len(gemmResults)) for _, gemm := range gemmResults { if gemm.Failed { - metrics = append(metrics, fmt.Sprintf("%s=FAIL", gemm.GemmName)) + if gemm.Mean > 0 { + metrics = append(metrics, fmt.Sprintf("%s=%.2f(FAIL)", gemm.GemmName, gemm.Mean)) + } else { + metrics = append(metrics, fmt.Sprintf("%s=FAIL", gemm.GemmName)) + } continue } passed++ @@ -104,6 +108,11 @@ func formatGEMMSummary(result dcgm.TargetStressResult) string { status = "WARN" } fmt.Fprintf(&output, "HCU %d %s %d/%d mean: %s\n", hcuID, status, passed, len(gemmResults), strings.Join(metrics, ", ")) + for _, gemm := range gemmResults { + if gemm.Error != "" { + fmt.Fprintf(&output, " [%s] error: %s\n", gemm.GemmName, gemm.Error) + } + } } return output.String() } diff --git a/pkg/cmd/cli/diag_test.go b/pkg/cmd/cli/diag_test.go index 2a11fe1..b84aa03 100644 --- a/pkg/cmd/cli/diag_test.go +++ b/pkg/cmd/cli/diag_test.go @@ -101,6 +101,25 @@ func TestFormatGEMMSummary(t *testing.T) { } } +func TestFormatGEMMSummary_FailWithMeanAndError(t *testing.T) { + output := formatGEMMSummary(dcgm.TargetStressResult{ + Results: []dcgm.GemmTestResult{ + {HCUId: 0, GemmName: "hgemm", Mean: 95.50, Failed: true, Error: "子进程异常退出 exit status 1"}, + {HCUId: 0, GemmName: "sgemm", Mean: 40.10}, + }, + }) + + for _, want := range []string{ + "hgemm=95.50(FAIL)", + "sgemm=40.10", + "[hgemm] error: 子进程异常退出 exit status 1", + } { + if !strings.Contains(output, want) { + t.Fatalf("output %q does not contain %q", output, want) + } + } +} + func TestDiagnosticStatusLabel(t *testing.T) { if got := diagnosticStatusLabel(dcgm.DiagResultWarn); got != "Warning" { t.Fatalf("warning label = %q, want Warning", got) diff --git a/pkg/dcgm/admin.go b/pkg/dcgm/admin.go index da37144..cddcdb5 100644 --- a/pkg/dcgm/admin.go +++ b/pkg/dcgm/admin.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/api.go b/pkg/dcgm/api.go index 97b56ef..3cd2076 100644 --- a/pkg/dcgm/api.go +++ b/pkg/dcgm/api.go @@ -2783,10 +2783,6 @@ func ShowPcieReplayCount(dvIdList []int) (pcieReplayCountInfos []PcieReplayCount return } -// 获取指定进程的进程信息 -func ProcessInfo(pid int) (proc ProcessInfos, err error) { - return rsmiComputeProcessInfoByPidGet(pid) -} // 获取指定设备上的进程信息 func ProcessInfoByDevice(pid int, dvInd int) (proc ProcessInfos, err error) { @@ -4925,10 +4921,17 @@ func ComputeProcessInfoGet() (processInfo []ProcessInfos, numItems int, err erro return rsmiComputeProcessInfoGet() } +// ProcessInfoByPid 根据PID获取进程信息 func ProcessInfoByPid(pid uint32) (RsmiProcessInfoV2, error) { return getProcessInfoByPID(pid) } +// 获取指定进程的进程信息 +func ProcessInfo(pid int) (proc ProcessInfos, err error) { + return rsmiComputeProcessInfoByPidGet(pid) +} + + func DeviceGetCount() (deviceCount int, err error) { return nvmlDeviceGetCount() } diff --git a/pkg/dcgm/device_info.go b/pkg/dcgm/device_info.go index 3a89973..1c0d66a 100644 --- a/pkg/dcgm/device_info.go +++ b/pkg/dcgm/device_info.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/device_status.go b/pkg/dcgm/device_status.go index b3ec901..518bfc1 100644 --- a/pkg/dcgm/device_status.go +++ b/pkg/dcgm/device_status.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/diag.go b/pkg/dcgm/diag.go index bdc2da0..a465bcc 100644 --- a/pkg/dcgm/diag.go +++ b/pkg/dcgm/diag.go @@ -512,11 +512,16 @@ func runStressTests(level int) (DiagResults, error) { if r.Failed || r.Mean <= 0 { status = DiagResultWarn } + errCode := 0 + if r.Error != "" { + errCode = -1 + } dr := DiagResult{ - Status: status, - TestName: fmt.Sprintf("TargetStress GEMM %s", r.GemmName), - TestOutput: fmt.Sprintf("GEMM=%s, Mean=%.3f", r.GemmName, r.Mean), - ErrorCode: 0, + Status: status, + TestName: fmt.Sprintf("TargetStress GEMM %s", r.GemmName), + TestOutput: fmt.Sprintf("GEMM=%s, Mean=%.3f", r.GemmName, r.Mean), + ErrorCode: errCode, + ErrorMessage: r.Error, } merged := false for i := range aggregated.PerHCU { diff --git a/pkg/dcgm/edpp.go b/pkg/dcgm/edpp.go index 2a97575..dce4aff 100644 --- a/pkg/dcgm/edpp.go +++ b/pkg/dcgm/edpp.go @@ -59,12 +59,11 @@ const ( edppStressLoopCount = "1900000000" edppStressDuration = 10 * time.Second - // BW1000 空闲功耗约 86-105W，满载约 1000W+；设 300W 为最低门槛， - // 压测启动后功耗应迅速超过该值，空闲状态绝对无法触及。 - edppBMZMinAvgPowerW = 300 - // BW1100/NMZ 实测空闲约 170W，EDPp 压力约 800W且不超过 1000W； - // 以约 400W 作为进入计算状态的保守门槛，避免依赖 loader 不输出的 TFLOPS 文本。 - edppNMZMinAvgPowerW = 400 + // BMZ 系列统一使用 200W 作为最低门槛；BW100 实测压测平均功耗约 231W， + // 该阈值可避免将已通过 GEMM CPU CHECK 的有效计算误判为空载。 + edppBMZMinAvgPowerW = 200 + // NMZ 系列统一使用 300W 作为进入计算状态的最低门槛。 + edppNMZMinAvgPowerW = 300 ) type edppBackendSpec struct { diff --git a/pkg/dcgm/edpp_test.go b/pkg/dcgm/edpp_test.go index db33600..aa89da8 100644 --- a/pkg/dcgm/edpp_test.go +++ b/pkg/dcgm/edpp_test.go @@ -71,6 +71,14 @@ func TestEdppBackendSpecsUseStressMode(t *testing.T) { if bmz.coName != "fp16_bmz_edpp.co" || bmz.minAvgPowerW != edppBMZMinAvgPowerW { t.Fatalf("bmz spec = %+v, want fp16 co and bmz threshold", bmz) } + + nmz, err = edppBackendSpecFor(edppBackendNMZGFX938) + if err != nil { + t.Fatalf("NMZ spec error: %v", err) + } + if nmz.minAvgPowerW != edppNMZMinAvgPowerW { + t.Fatalf("NMZ threshold = %d, want %d", nmz.minAvgPowerW, edppNMZMinAvgPowerW) + } } func TestFinalizeStressEdppResult(t *testing.T) { @@ -97,8 +105,8 @@ func TestFinalizeStressEdppResult(t *testing.T) { } result = GemmPowerEdppResult{HCU: 2, Backend: spec.name} - err = finalizeStressEdppResult(&result, spec, edppStressSamples{totalPower: 740, peakPower: 370, powerSamples: 2}) - if err == nil || !strings.Contains(result.Error, "avg power 370W < threshold 400W") { + err = finalizeStressEdppResult(&result, spec, edppStressSamples{totalPower: 580, peakPower: 290, powerSamples: 2}) + if err == nil || !strings.Contains(result.Error, "avg power 290W < threshold 300W") { t.Fatalf("low power err = %v, result = %+v", err, result) } } diff --git a/pkg/dcgm/gemmperf.go b/pkg/dcgm/gemmperf.go index fb7de03..eea5af4 100644 --- a/pkg/dcgm/gemmperf.go +++ b/pkg/dcgm/gemmperf.go @@ -84,7 +84,7 @@ func checkDependencies(binaryPath string) error { } // ---------------- 启动单个 GEMM 压测（增强版） ---------------- -func runGemmTest(gemmPerfPath string, devInd int, gemmIdx int, iterations int, logfile string, mValue int) error { +func runGemmTest(gemmPerfPath string, gemmIdx int, iterations int, logfile string, mValue int) error { args := []string{ "-m", strconv.Itoa(mValue), "-n", "4096", @@ -126,6 +126,7 @@ func parseGemmLog(logfile string, devInd int) (mean float64, fail bool, err erro defer f.Close() re := regexp.MustCompile(`HCU` + strconv.Itoa(devInd) + `:.*mean:\s*([0-9]*\.?[0-9]+)`) + found := false scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() @@ -134,6 +135,7 @@ func parseGemmLog(logfile string, devInd int) (mean float64, fail bool, err erro if err != nil { return 0, false, fmt.Errorf("解析 mean 失败: %w", err) } + found = true } if strings.Contains(line, "FAIL") { fail = true @@ -142,6 +144,9 @@ func parseGemmLog(logfile string, devInd int) (mean float64, fail bool, err erro if err := scanner.Err(); err != nil { return 0, false, fmt.Errorf("扫描日志失败: %w", err) } + if !found { + return 0, fail, fmt.Errorf("日志中未找到 HCU%d 的 mean 值，日志: %s", devInd, logfile) + } return mean, fail, nil } @@ -178,22 +183,23 @@ func targetStressTest() { return } - for devInd := 0; devInd < totalHCU; devInd++ { - for _, gemm := range gemmList { - logfile := filepath.Join(GEMMLogDir, fmt.Sprintf("%s_hcu%d.log", gemm.Name, devInd)) - mValue := 5632 + // gemmPerf 每次调用都在所有设备上运行，因此按 gemm 类型循环一次即可， + // 再从同一份日志里解析各设备的结果。 + for _, gemm := range gemmList { + logfile := filepath.Join(GEMMLogDir, fmt.Sprintf("%s.log", gemm.Name)) + mValue := 5632 - if err := runGemmTest(gemmPerfPath, devInd, gemm.Idx, iterations, logfile, mValue); err != nil { - fmt.Printf("[gemmperf] device %d, gemm %s failed: %v\n", devInd, gemm.Name, err) - continue - } + runErr := runGemmTest(gemmPerfPath, gemm.Idx, iterations, logfile, mValue) + if runErr != nil { + fmt.Printf("[gemmperf] gemm %s failed: %v\n", gemm.Name, runErr) + } - mean, _, err := parseGemmLog(logfile, devInd) - if err != nil { - fmt.Printf("[gemmperf parse] device %d, gemm %s failed: %v\n", devInd, gemm.Name, err) + for devInd := 0; devInd < totalHCU; devInd++ { + mean, _, parseErr := parseGemmLog(logfile, devInd) + if parseErr != nil { + fmt.Printf("[gemmperf parse] device %d, gemm %s failed: %v\n", devInd, gemm.Name, parseErr) continue } - fmt.Printf("TargetStress: HCU%d, %s mean %.2f\n", devInd, gemm.Name, mean) } } @@ -208,6 +214,7 @@ type GemmTestResult struct { GemmName string // GEMM 测试名称（如 "hgemm", "sgemm"） Mean float64 // 测得的平均性能值（单位同你日志/工具，比如 GFLOPS 或 MB/s） Failed bool // 测试是否失败（true=失败/出错，false=成功） + Error string // 具体错误信息；Failed=true 时非空 } // TargetStressResult 汇总整个测试的结构化结果 @@ -253,38 +260,35 @@ func runTargetStressTestWithResult() (TargetStressResult, error) { results := make([]GemmTestResult, 0, totalHCU*len(gemmList)) - for devInd := 0; devInd < totalHCU; devInd++ { - for _, gemm := range gemmList { - logfile := filepath.Join(GEMMLogDir, fmt.Sprintf("%s_hcu%d.log", gemm.Name, devInd)) - mValue := 5632 - - if err := runGemmTest(gemmPerfPath, devInd, gemm.Idx, iterations, logfile, mValue); err != nil { - results = append(results, GemmTestResult{ - HCUId: devInd, - GemmName: gemm.Name, - Mean: 0, - Failed: true, - }) - continue - } + // gemmPerf 每次调用都在所有设备上运行，因此按 gemm 类型循环一次即可， + // 再从同一份日志里解析各设备的结果。 + for _, gemm := range gemmList { + logfile := filepath.Join(GEMMLogDir, fmt.Sprintf("%s.log", gemm.Name)) + mValue := 5632 - mean, fail, err := parseGemmLog(logfile, devInd) - if err != nil { - results = append(results, GemmTestResult{ - HCUId: devInd, - GemmName: gemm.Name, - Mean: 0, - Failed: true, - }) - continue - } + runErr := runGemmTest(gemmPerfPath, gemm.Idx, iterations, logfile, mValue) + + for devInd := 0; devInd < totalHCU; devInd++ { + mean, fail, parseErr := parseGemmLog(logfile, devInd) - results = append(results, GemmTestResult{ + r := GemmTestResult{ HCUId: devInd, GemmName: gemm.Name, Mean: mean, - Failed: fail, - }) + Failed: runErr != nil || fail, + } + if runErr != nil { + r.Error = runErr.Error() + } + if parseErr != nil { + if r.Error != "" { + r.Error += "; " + parseErr.Error() + } else { + r.Error = parseErr.Error() + } + r.Failed = true + } + results = append(results, r) } } diff --git a/pkg/dcgm/gemmperf_test.go b/pkg/dcgm/gemmperf_test.go new file mode 100644 index 0000000..898f64b --- /dev/null +++ b/pkg/dcgm/gemmperf_test.go @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 Hygon Information Technology Co., Ltd. + */ +package dcgm + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseGemmLog_ValidMean(t *testing.T) { + dir := t.TempDir() + logfile := filepath.Join(dir, "hgemm.log") + content := "fp16(h)-HCU0: min: 90.0\tmax: 99.0\tmean: 98.500\tstdev: 1.0\n" + if err := os.WriteFile(logfile, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + mean, fail, err := parseGemmLog(logfile, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fail { + t.Fatal("expected fail=false") + } + if mean != 98.500 { + t.Fatalf("mean = %v, want 98.500", mean) + } +} + +func TestParseGemmLog_MissingMean(t *testing.T) { + dir := t.TempDir() + logfile := filepath.Join(dir, "hgemm.log") + // 只有 HCU0，没有 HCU1 + content := "fp16(h)-HCU0: min: 90.0\tmax: 99.0\tmean: 98.500\tstdev: 1.0\n" + if err := os.WriteFile(logfile, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + _, _, err := parseGemmLog(logfile, 1) + if err == nil { + t.Fatal("expected error for missing mean, got nil") + } +} + +func TestParseGemmLog_FailFlag(t *testing.T) { + dir := t.TempDir() + logfile := filepath.Join(dir, "hgemm.log") + content := "fp16(h)-HCU0: min: 40.0\tmax: 55.0\tmean: 50.000\tstdev: 2.0\nFAIL: verification error\n" + if err := os.WriteFile(logfile, []byte(content), 0644); err != nil { + t.Fatal(err) + } + + mean, fail, err := parseGemmLog(logfile, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !fail { + t.Fatal("expected fail=true") + } + if mean != 50.000 { + t.Fatalf("mean = %v, want 50.000", mean) + } +} diff --git a/pkg/dcgm/mig.go b/pkg/dcgm/mig.go index d1ea698..86cca3a 100644 --- a/pkg/dcgm/mig.go +++ b/pkg/dcgm/mig.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -lhydmi_mig -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -lhydmi_mig -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/policy.go b/pkg/dcgm/policy.go index da65f0f..ebe9105 100644 --- a/pkg/dcgm/policy.go +++ b/pkg/dcgm/policy.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/process_info.go b/pkg/dcgm/process_info.go index 91fee4f..3ef9986 100644 --- a/pkg/dcgm/process_info.go +++ b/pkg/dcgm/process_info.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/structs.go b/pkg/dcgm/structs.go index 371d0af..9447a6a 100644 --- a/pkg/dcgm/structs.go +++ b/pkg/dcgm/structs.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/topology.go b/pkg/dcgm/topology.go index a7efb6f..881b705 100644 --- a/pkg/dcgm/topology.go +++ b/pkg/dcgm/topology.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/dcgm/util.go b/pkg/dcgm/util.go index 02a40c5..d5547e9 100644 --- a/pkg/dcgm/util.go +++ b/pkg/dcgm/util.go @@ -6,7 +6,7 @@ package dcgm /* #cgo CFLAGS: -Wall -I./include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/pkg/service/router/api.go b/pkg/service/router/api.go index e727098..935d541 100644 --- a/pkg/service/router/api.go +++ b/pkg/service/router/api.go @@ -4377,12 +4377,16 @@ func startTargetStress(c *gin.Context) { if r.Failed || r.Mean <= 0 { status = DiagResultWarn } + errCode := 0 + if r.Error != "" { + errCode = -1 + } dr := dcgm.DiagResult{ Status: status, TestName: fmt.Sprintf("TargetStress GEMM %s", r.GemmName), TestOutput: fmt.Sprintf("GEMM=%s, Mean=%.3f", r.GemmName, r.Mean), - ErrorCode: 0, - ErrorMessage: "", + ErrorCode: errCode, + ErrorMessage: r.Error, } perMap[r.HCUId] = append(perMap[r.HCUId], dr) } diff --git a/pkg/service/router/router.go b/pkg/service/router/router.go index 7cbd750..63c9aaf 100644 --- a/pkg/service/router/router.go +++ b/pkg/service/router/router.go @@ -250,8 +250,8 @@ func InitRouter() *gin.Engine { // 仪电近似指标：vDevice compute busy（≈MIG_ENC_UTIL/DEC_UTIL） router.GET("/vdevice/busypercent/:vDvInd", GetVDevBusyPercent) - // 仪电近似指标：进程 GPU compute 占用率（≈PROC_ENC_UTIL） - router.GET("/process/gpuusage/:pid", GetProcessGpuUsage) + // // 仪电近似指标：进程 GPU compute 占用率（≈PROC_ENC_UTIL） + // router.GET("/process/gpuusage/:pid", GetProcessGpuUsage) // 功率上限设置与复位（对应 hy-smi --setpoweroverdrive / --resetpoweroverdrive） router.POST("/powercap/:dvInd", SetPowerCap) diff --git a/pkg/service/router/structs.go b/pkg/service/router/structs.go index 681e4a0..ec90611 100644 --- a/pkg/service/router/structs.go +++ b/pkg/service/router/structs.go @@ -6,7 +6,7 @@ package router /* #cgo CFLAGS: -Wall -I../../dcgm/include -#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files +#cgo LDFLAGS: -L/opt/hyhal/lib -Wl,--enable-new-dtags -Wl,-rpath,/opt/hyhal/lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files #include #include #include diff --git a/samples/device_info/main.go b/samples/device_info/main.go index 55f5e7c..9a14c89 100644 --- a/samples/device_info/main.go +++ b/samples/device_info/main.go @@ -37,113 +37,119 @@ func main() { dcgm.Init() defer dcgm.ShutDown() //HCU物理设备总数量 - dcgm.NumMonitorDevices() - //HCU名称 - dcgm.DevName(0) - //获取指定HCU设备可用的PCIE宽带列表 - dcgm.DevPciBandwidth(0) - //设置HCU的PowerPlay性能级别 - dcgm.DevPerfLevelSet(0, dcgm.RSMI_DEV_PERF_LEVEL_LOW) - //HCU设备内存使用百分比 - dcgm.MemoryPercent(0) - //获取HCU设备名称 - dcgm.DevName(1) - //获取设备的sku - dcgm.DevSku(1) - //获取设备品牌名称 - dcgm.DevBrand(1) - //获取设备的供应商的名称 - dcgm.DevVendorName(1) - //获取设备显存供应商名称 - dcgm.DevVramVendor(1) - //获取指定HCU的度量信息 - dcgm.DevGpuMetricsInfo(0) - //获取物理设备的监控指标 - dcgm.CollectDeviceMetrics() - //HCU设备指定类型的内存使用情况 [vram|vis_vram|gtt - dcgm.MemInfo(0, "vram") - //dcgm.MemInfo(0, "vis_vram") - //dcgm.MemInfo(0, "gtt") - //获取设备当前的性能水平 - dcgm.PerfLevel(0) - //获取设备的平均功率 - dcgm.Power(1) - //设备的VBIOS版本信息 - dcgm.VbiosVersion(1) - //获取系统的驱动程序版本 - dcgm.Version(dcgm.RSMISwCompFirst) - dcgm.Version(dcgm.RSMISwCompDriver) - dcgm.Version(dcgm.RSMISwCompLast) - // 调用方法打印事件列表 - //获取设备的HSL hive id - dcgm.HSLHiveIdGet(1) - - //批量展示显示设备硬件信息 - dcgm.ShowAllConciseHw([]int{0, 1, 2}) - //批量展示显示时钟信息 - dcgm.ShowClocks([]int{0, 1, 2}) - //展示风扇转速和风扇级别 - dcgm.ShowCurrentFans([]int{0, 1, 2}, false) - //显示设备的所有可用温度传感器的温度 - dcgm.ShowCurrentTemps([]int{0, 1, 2}) - //显示设备中指定固件类型的固件版本信息 - dcgm.ShowFwInfo([]int{0, 1, 2}, []string{"all"}) - - //获取HCU设备的的粗粒度利用率 - dcgm.GetCoarseGrainUtil(0, "all") - //批量获取HCU的使用率 - dcgm.ShowHCUUse([]int{0, 1, 2}) - //批量获取设备消耗的能量 - dcgm.ShowEnergy([]int{0, 1, 2}) - //HCU设备的ID（十六进制表示） - //设备的最大功率值 - dcgm.MaxPower(0) - //设备的不同类型的内存使用情况 memType:[vram|vis_vram|gtt] - dcgm.MemInfo(0, "vram") - //批量获取设备内存的信息 - dcgm.ShowMemInfo([]int{0, 1, 2}, []string{"VRAM", "VIS_VRAM"}) - //批量获取设备内存使用情况 - dcgm.ShowMemUse([]int{0, 1, 2}) - //批量获取备供应商信息 - dcgm.ShowMemVendor([]int{0, 1, 2}) - //批量获取设备的PCIe带宽使用情况 - dcgm.ShowPcieBw([]int{0, 1, 2}) - //批量获取设备PCIe重放计数 - dcgm.ShowPcieReplayCount([]int{0, 1, 2}) - - //批量获取设备的平均功率 - dcgm.ShowPower([]int{0, 1, 2}) - - //当前设备内存时钟频率和电压（K100_AI卡不支持该操作） - dcgm.ShowPowerPlayTable([]int{0, 1, 2}) - //可用电源配置文件 - dcgm.ShowProfile([]int{0, 1, 2}) - - //电流或电压范围（K100_AI卡不支持该操作） - devices := []int{0, 1, 2} - dcgm.ShowRange(devices, "sclk") - dcgm.ShowRange(devices, "mclk") - dcgm.ShowRange(devices, "voltage") - - //显示设备中指定类型的退役页 - dcgm.ShowRetiredPages([]int{0, 1, 2}, "all") - //设备序列号 - dcgm.ShowSerialNumber([]int{0, 1, 2}) - //设备的唯一设备ID - dcgm.ShowUId([]int{0, 1, 2}) - //设备的VBIOS版本信息（格式化打印并返回设备的VBIOS版本信息） - dcgm.ShowVbiosVersion([]int{0, 1, 2}) - - // 示例设备列表和事件类型 - deviceList := []int{0, 1, 2} - eventTypes := []string{"VM_FAULT", "THERMAL_THROTTLE"} - dcgm.ShowEvents(deviceList, eventTypes) - //指定设备的当前电压信息 - dcgm.ShowVoltage([]int{0, 1, 2}) - - //获取指定设备的电压曲线点（K100_AI卡不支持该操作） - dcgm.ShowVoltageCurve([]int{0, 1, 2}) - //指定设备的HSL错误状态（K100_AI卡不支持该操作） - dcgm.ShowHSLErr([]int{0, 1, 2}, true) + // dcgm.NumMonitorDevices() + // //HCU名称 + // dcgm.DevName(0) + // //获取指定HCU设备可用的PCIE宽带列表 + // dcgm.DevPciBandwidth(0) + // //设置HCU的PowerPlay性能级别 + // dcgm.DevPerfLevelSet(0, dcgm.RSMI_DEV_PERF_LEVEL_LOW) + // //HCU设备内存使用百分比 + // dcgm.MemoryPercent(0) + // //获取HCU设备名称 + // dcgm.DevName(1) + // //获取设备的sku + // dcgm.DevSku(1) + // //获取设备品牌名称 + // dcgm.DevBrand(1) + // //获取设备的供应商的名称 + // dcgm.DevVendorName(1) + // //获取设备显存供应商名称 + // dcgm.DevVramVendor(1) + // //获取指定HCU的度量信息 + // dcgm.DevGpuMetricsInfo(0) + // //获取物理设备的监控指标 + // dcgm.CollectDeviceMetrics() + // //HCU设备指定类型的内存使用情况 [vram|vis_vram|gtt + // dcgm.MemInfo(0, "vram") + // //dcgm.MemInfo(0, "vis_vram") + // //dcgm.MemInfo(0, "gtt") + // //获取设备当前的性能水平 + // dcgm.PerfLevel(0) + // //获取设备的平均功率 + // dcgm.Power(1) + // //设备的VBIOS版本信息 + // dcgm.VbiosVersion(1) + // //获取系统的驱动程序版本 + // dcgm.Version(dcgm.RSMISwCompFirst) + // dcgm.Version(dcgm.RSMISwCompDriver) + // dcgm.Version(dcgm.RSMISwCompLast) + // // 调用方法打印事件列表 + // //获取设备的HSL hive id + // dcgm.HSLHiveIdGet(1) + + // //批量展示显示设备硬件信息 + // dcgm.ShowAllConciseHw([]int{0, 1, 2}) + // //批量展示显示时钟信息 + // dcgm.ShowClocks([]int{0, 1, 2}) + // //展示风扇转速和风扇级别 + // dcgm.ShowCurrentFans([]int{0, 1, 2}, false) + // //显示设备的所有可用温度传感器的温度 + // dcgm.ShowCurrentTemps([]int{0, 1, 2}) + // //显示设备中指定固件类型的固件版本信息 + // dcgm.ShowFwInfo([]int{0, 1, 2}, []string{"all"}) + + // //获取HCU设备的的粗粒度利用率 + // dcgm.GetCoarseGrainUtil(0, "all") + // //批量获取HCU的使用率 + // dcgm.ShowHCUUse([]int{0, 1, 2}) + // //批量获取设备消耗的能量 + // dcgm.ShowEnergy([]int{0, 1, 2}) + // //HCU设备的ID（十六进制表示） + // //设备的最大功率值 + // dcgm.MaxPower(0) + // //设备的不同类型的内存使用情况 memType:[vram|vis_vram|gtt] + // dcgm.MemInfo(0, "vram") + // //批量获取设备内存的信息 + // dcgm.ShowMemInfo([]int{0, 1, 2}, []string{"VRAM", "VIS_VRAM"}) + // //批量获取设备内存使用情况 + // dcgm.ShowMemUse([]int{0, 1, 2}) + // //批量获取备供应商信息 + // dcgm.ShowMemVendor([]int{0, 1, 2}) + // //批量获取设备的PCIe带宽使用情况 + // dcgm.ShowPcieBw([]int{0, 1, 2}) + // //批量获取设备PCIe重放计数 + // dcgm.ShowPcieReplayCount([]int{0, 1, 2}) + + // //批量获取设备的平均功率 + // dcgm.ShowPower([]int{0, 1, 2}) + + // //当前设备内存时钟频率和电压（K100_AI卡不支持该操作） + // dcgm.ShowPowerPlayTable([]int{0, 1, 2}) + // //可用电源配置文件 + // dcgm.ShowProfile([]int{0, 1, 2}) + + // //电流或电压范围（K100_AI卡不支持该操作） + // devices := []int{0, 1, 2} + // dcgm.ShowRange(devices, "sclk") + // dcgm.ShowRange(devices, "mclk") + // dcgm.ShowRange(devices, "voltage") + + // //显示设备中指定类型的退役页 + // dcgm.ShowRetiredPages([]int{0, 1, 2}, "all") + // //设备序列号 + // dcgm.ShowSerialNumber([]int{0, 1, 2}) + // //设备的唯一设备ID + // dcgm.ShowUId([]int{0, 1, 2}) + // //设备的VBIOS版本信息（格式化打印并返回设备的VBIOS版本信息） + // dcgm.ShowVbiosVersion([]int{0, 1, 2}) + + // // 示例设备列表和事件类型 + // deviceList := []int{0, 1, 2} + // eventTypes := []string{"VM_FAULT", "THERMAL_THROTTLE"} + // dcgm.ShowEvents(deviceList, eventTypes) + // //指定设备的当前电压信息 + // dcgm.ShowVoltage([]int{0, 1, 2}) + + // //获取指定设备的电压曲线点（K100_AI卡不支持该操作） + // dcgm.ShowVoltageCurve([]int{0, 1, 2}) + // //指定设备的HSL错误状态（K100_AI卡不支持该操作） + // dcgm.ShowHSLErr([]int{0, 1, 2}, true) + use, err := dcgm.HCUUse(0) + if err != nil { + glog.Error(err) + return + } + glog.Info(use) } diff --git a/samples/process/main.go b/samples/process/main.go index c9629f5..15eaf50 100644 --- a/samples/process/main.go +++ b/samples/process/main.go @@ -41,35 +41,4 @@ func main() { fmt.Printf(" CU Occupancy : %d\n", process.CuOccupancy) fmt.Printf(" HCU Device IDs : %v\n", process.MinorNumbers) } - - for _, pid := range []uint32{61756, 61527} { - printProcessInfoByPID(pid) - } -} - -func printProcessInfoByPID(pid uint32) { - info, err := dcgm.ProcessInfoByPid(pid) - if err != nil { - fmt.Printf("\n[ProcessInfoByPid: PID %d]\n Query Error : %v\n", pid, err) - return - } - - fmt.Printf("\n[ProcessInfoByPid: PID %d]\n", pid) - fmt.Printf(" Process ID : %d\n", info.ProcessID) - fmt.Printf(" VRAM Usage : %d MiB\n", info.VRAMUsageSize) - fmt.Printf(" VRAM Usage Rate : %.2f%%\n", info.VRAMUsageRate) - fmt.Printf(" HCU Device Count: %d\n", info.GPUCount) - if info.GPUCount == 0 { - fmt.Println(" HCU Devices : none") - return - } - - fmt.Println(" HCU Devices:") - for index, deviceID := range info.GPUIndex { - usageRate := float32(0) - if index < len(info.GPUUsageRate) { - usageRate = info.GPUUsageRate[index] - } - fmt.Printf(" - Device ID %d, Usage Rate %.2f%%\n", deviceID, usageRate) - } } diff --git a/samples/test/main.go b/samples/test/main.go index b3f3fa5..42e00c9 100644 --- a/samples/test/main.go +++ b/samples/test/main.go @@ -394,8 +394,11 @@ func main() { //demoGetDeviceId() //demoHCUHealthCheck() - demoDeviceTemperatureInfo(numDevices) - demoDeviceUtilizationInfo(numDevices) + //demoGetTempByMetric(numDevices) + demoCommonTemperatures(numDevices) + //demoDeviceTemperatureInfo(numDevices) + //demoDeviceUtilizationInfo(numDevices) + //demoExporterErrorInterfaces(numDevices) // fmt.Println("==== HCU Interconnect Topology Demo ====") // @@ -689,6 +692,93 @@ func dataToJson(data any) string { return string(jsonData) } +func demoCommonTemperatures(numDevices int) { + const ( + edgeSensor = 0 + junctionSensor = 1 + memorySensor = 2 + coreSensor = 11 + ) + + sensors := []struct { + name string + typeID int + }{ + {"EDGE", edgeSensor}, + {"JUNCTION", junctionSensor}, + {"MEMORY", memorySensor}, + {"CORE", coreSensor}, + } + + fmt.Println() + fmt.Println("============================================================") + fmt.Println(" Common Temperature Sensors Demo") + fmt.Println(" metric: RSMI_TEMP_CURRENT") + fmt.Println("============================================================") + fmt.Printf("Detected %d HCU(s)\n\n", numDevices) + + for dvInd := 0; dvInd < numDevices; dvInd++ { + fmt.Printf("-------------------- HCU [%d] --------------------\n", dvInd) + for _, sensor := range sensors { + temp, err := dcgm.GetTempBySensor(dvInd, sensor.typeID, dcgm.RSMI_TEMP_CURRENT) + if err != nil { + fmt.Printf(" %-8s (sensorType=%2d): ERROR: %v\n", sensor.name, sensor.typeID, err) + glog.Warningf("HCU %d GetTempBySensor(%s) failed: %v", dvInd, sensor.name, err) + continue + } + fmt.Printf(" %-8s (sensorType=%2d): %.2f°C\n", sensor.name, sensor.typeID, temp) + glog.V(5).Infof("HCU %d %s temperature = %.2f°C", dvInd, sensor.name, temp) + } + fmt.Println() + } + + fmt.Println("============================================================") +} + +func demoGetTempByMetric(numDevices int) { + metrics := []struct { + name string + metric dcgm.RSMITemperatureMetric + }{ + {"RSMI_TEMP_CURRENT", dcgm.RSMI_TEMP_CURRENT}, + {"RSMI_TEMP_MAX", dcgm.RSMI_TEMP_MAX}, + {"RSMI_TEMP_MIN", dcgm.RSMI_TEMP_MIN}, + {"RSMI_TEMP_MAX_HYST", dcgm.RSMI_TEMP_MAX_HYST}, + {"RSMI_TEMP_MIN_HYST", dcgm.RSMI_TEMP_MIN_HYST}, + {"RSMI_TEMP_CRITICAL", dcgm.RSMI_TEMP_CRITICAL}, + {"RSMI_TEMP_CRITICAL_HYST", dcgm.RSMI_TEMP_CRITICAL_HYST}, + {"RSMI_TEMP_EMERGENCY", dcgm.RSMI_TEMP_EMERGENCY}, + {"RSMI_TEMP_EMERGENCY_HYST", dcgm.RSMI_TEMP_EMERGENCY_HYST}, + {"RSMI_TEMP_CRIT_MIN", dcgm.RSMI_TEMP_CRIT_MIN}, + {"RSMI_TEMP_CRIT_MIN_HYST", dcgm.RSMI_TEMP_CRIT_MIN_HYST}, + {"RSMI_TEMP_OFFSET", dcgm.RSMI_TEMP_OFFSET}, + {"RSMI_TEMP_LOWEST", dcgm.RSMI_TEMP_LOWEST}, + {"RSMI_TEMP_HIGHEST", dcgm.RSMI_TEMP_HIGHEST}, + } + + fmt.Println() + fmt.Println("============================================================") + fmt.Println(" GetTempByMetric Demo (EDGE sensor)") + fmt.Println("============================================================") + + for dvInd := 0; dvInd < numDevices; dvInd++ { + fmt.Printf("HCU[%d]\n", dvInd) + for _, item := range metrics { + temp, err := dcgm.GetTempByMetric(dvInd, item.metric) + if err != nil { + fmt.Printf(" %-28s (%2d): ERROR: %v\n", item.name, item.metric, err) + glog.Warningf("HCU %d GetTempByMetric(%s) failed: %v", dvInd, item.name, err) + continue + } + fmt.Printf(" %-28s (%2d): %.2f°C\n", item.name, item.metric, temp) + glog.V(5).Infof("HCU %d GetTempByMetric(%s) = %.2f°C", dvInd, item.name, temp) + } + fmt.Println() + } + + fmt.Println("============================================================") +} + func demoDeviceTemperatureInfo(numDevices int) { fmt.Println() fmt.Println("============================================================") @@ -757,3 +847,110 @@ func demoDeviceUtilizationInfo(numDevices int) { fmt.Println("============================================================") } + +// demoExporterErrorInterfaces 逐设备测试 exporter 日志中对应的 DCGM 接口。 +// 该 demo 保留原始返回值和错误，便于在不同版本驱动上对比接口支持情况。 +func demoExporterErrorInterfaces(numDevices int) { + const umcDelay = 10 + + fmt.Println() + fmt.Println("============================================================") + fmt.Println(" Exporter Error Interfaces Demo") + fmt.Println("============================================================") + fmt.Println("The following calls correspond to the unsupported-interface logs from hcu-exporter.") + fmt.Printf("Detected %d HCU(s), sample window = %d ms, UMC delay = %d\n\n", + numDevices, defaultSampleDurationMs, umcDelay) + + for dvInd := 0; dvInd < numDevices; dvInd++ { + fmt.Printf("-------------------- HCU [%d] --------------------\n", dvInd) + + fmt.Println("[1] HCUSEUsage -> rsmi_dev_se_util_get") + seUsage, err := dcgm.HCUSEUsage(dvInd) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %s\n", dataToJson(seUsage)) + } + + fmt.Println("[2] HSLErrorStatus -> rsmi_dev_xgmi_error_status") + status, err := dcgm.HSLErrorStatus(dvInd) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: status=%d\n", status) + } + + fmt.Printf("[3] HCUSampledUsage -> rsmi_dev_hcu_util_get (duration=%d)\n", defaultSampleDurationMs) + hcuUtil, err := dcgm.HCUSampledUsage(dvInd, defaultSampleDurationMs) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %.6f\n", hcuUtil) + } + + fmt.Printf("[4] HCUCUSampledUsage -> rsmi_dev_cu_util_get (duration=%d)\n", defaultSampleDurationMs) + cuUtil, err := dcgm.HCUCUSampledUsage(dvInd, defaultSampleDurationMs) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %.6f\n", cuUtil) + } + + fmt.Printf("[5] HCUWaveSampledUsage -> rsmi_dev_wave_util_get (duration=%d)\n", defaultSampleDurationMs) + waveUtil, err := dcgm.HCUWaveSampledUsage(dvInd, defaultSampleDurationMs) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %.6f\n", waveUtil) + } + + fmt.Printf("[6] DevCuUtil -> rsmi_dev_cu_util_get (duration=%d)\n", defaultSampleDurationMs) + devCuUtil, err := dcgm.DevCuUtil(dvInd, defaultSampleDurationMs) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %.6f\n", devCuUtil) + } + + fmt.Printf("[7] DevWaveUtil -> rsmi_dev_wave_util_get (duration=%d)\n", defaultSampleDurationMs) + devWaveUtil, err := dcgm.DevWaveUtil(dvInd, defaultSampleDurationMs) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Printf(" RESULT: %.6f\n", devWaveUtil) + } + + fmt.Printf("[8] UMCBandwidth -> rsmi_dev_umc_bandwidth_get (all channels, delay=%d)\n", umcDelay) + umcInfo, err := dcgm.UMCBandwidth(dvInd, dcgm.MAX_UMC_CHAN_NUM, umcDelay) + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + var read, write, readWrite float64 + for i := range dcgm.MAX_UMC_CHAN_NUM { + read += umcInfo.ReadBW[i] + write += umcInfo.WriteBW[i] + readWrite += umcInfo.ReadWriteBW[i] + } + fmt.Printf(" RESULT: read=%.2f write=%.2f readWrite=%.2f\n", read, write, readWrite) + } + + fmt.Println() + } + + fmt.Println("[9] GetHyLinkStatus -> rsmi_dev_xhcl_bandwidth_get (directions 0 and 1)") + linkStatus, err := dcgm.GetHyLinkStatus() + if err != nil { + fmt.Printf(" ERROR: %v\n", err) + } else { + fmt.Println(" RESULT: query succeeded") + } + for _, item := range linkStatus { + fmt.Printf(" HCU[%d]: recv=%.2f send=%.2f", item.DvInd, item.Recv, item.Send) + if item.Err != "" { + fmt.Printf(" err=%s", item.Err) + } + fmt.Println() + } + + fmt.Println("============================================================") +}