Skip to content

Commit 79aba98

Browse files
committed
fix: isolate GPU monitoring failures and scope device warnings
1 parent 806de63 commit 79aba98

20 files changed

Lines changed: 410 additions & 234 deletions

File tree

‎agent/app/dto/ai.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,23 @@ package dto
33
import "time"
44

55
type ModelGPUDevice struct {
6-
Index int `json:"index"`
7-
MemoryUsed string `json:"memoryUsed"`
8-
MemoryTotal string `json:"memoryTotal"`
9-
ID string `json:"id"`
10-
Name string `json:"name"`
11-
Type string `json:"type"`
12-
UUID string `json:"uuid,omitempty"`
13-
Available bool `json:"available"`
14-
NUMANode *int `json:"numaNode,omitempty"`
6+
Warnings []string `json:"warnings,omitempty"`
7+
Index int `json:"index"`
8+
MemoryUsed string `json:"memoryUsed"`
9+
MemoryTotal string `json:"memoryTotal"`
10+
ID string `json:"id"`
11+
Name string `json:"name"`
12+
Type string `json:"type"`
13+
UUID string `json:"uuid,omitempty"`
14+
Available bool `json:"available"`
15+
NUMANode *int `json:"numaNode,omitempty"`
1516
}
1617

1718
type ModelGPUDevices struct {
18-
Devices []ModelGPUDevice `json:"devices"`
19-
Warnings []string `json:"warnings"`
20-
NUMANodes []ModelNUMANode `json:"numaNodes"`
19+
VendorWarnings map[string][]string `json:"vendorWarnings,omitempty"`
20+
Devices []ModelGPUDevice `json:"devices"`
21+
Warnings []string `json:"warnings"`
22+
NUMANodes []ModelNUMANode `json:"numaNodes"`
2123
}
2224

2325
type ModelNUMANode struct {

‎agent/app/service/gpu.go‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -52,19 +52,18 @@ func LoadGPUInfo(ctx context.Context, refresh bool) (*dto.GPURealtimeInfo, error
5252
collectErr = err
5353
if collected != nil {
5454
snapshot = collected
55+
} else if err != nil {
56+
snapshot.Info.Warnings = append(snapshot.Info.Warnings, err.Error())
5557
}
5658
}
5759
if err := ctx.Err(); err != nil {
5860
return nil, err
5961
}
60-
if collectErr != nil {
61-
snapshot.Info.Warnings = append(snapshot.Info.Warnings, collectErr.Error())
62-
}
6362
if snapshot.Info.CollectedAt.IsZero() {
6463
snapshot.Info.CollectedAt = time.Now()
6564
}
6665
if global.LOG != nil {
67-
for _, warning := range snapshot.Info.Warnings {
66+
for _, warning := range snapshot.Warnings {
6867
global.LOG.Warnf("load realtime accelerator data: %s", warning)
6968
}
7069
}
@@ -167,7 +166,7 @@ func assembleGPUEnvironment(probe gpuEnvironmentProbe, summaries []dto.GPUSummar
167166
}
168167
for _, vendor := range gpuVendors {
169168
index := slices.IndexFunc(summaries, func(item dto.GPUSummary) bool { return item.Vendor == vendor })
170-
if index < 0 && probe.Hardware[vendor] == 0 && !(probe.HardwareError != "" && probe.Commands[vendor] && collectErr != nil) {
169+
if index < 0 && probe.Hardware[vendor] == 0 && len(info.VendorWarnings[vendor]) == 0 && !(probe.HardwareError != "" && probe.Commands[vendor] && collectErr != nil) {
171170
continue
172171
}
173172
check := dto.GPUEnvironmentCheck{Vendor: vendor, Item: "driver", Status: "unknown"}
@@ -176,32 +175,23 @@ func assembleGPUEnvironment(probe gpuEnvironmentProbe, summaries []dto.GPUSummar
176175
check.Status = "passed"
177176
check.Version = summaries[index].DriverVersion
178177
if vendor == "intel" {
179-
var readErrors []string
178+
unreadable := 0
180179
for _, device := range info.XPUs {
181-
if device.ReadError != "" {
182-
readErrors = append(readErrors, device.ReadError)
180+
if device.ReadError != "" && device.Stats == (xpu.Stats{}) && !slices.ContainsFunc(device.Tiles, func(tile xpu.TileStats) bool { return tile.Stats != (xpu.Stats{}) }) {
181+
unreadable++
183182
}
184183
}
185-
if len(readErrors) > 0 {
186-
check.Status = "unknown"
187-
if len(readErrors) == len(info.XPUs) {
188-
check.Status = "failed"
189-
}
190-
check.Reason = strings.Join(readErrors, "; ")
184+
if unreadable > 0 && unreadable == len(info.XPUs) {
185+
check.Status = "failed"
191186
}
192187
}
188+
case len(info.VendorWarnings[vendor]) > 0:
189+
check.Status = "failed"
193190
case !probe.Commands[vendor]:
194191
check.Status = "missing"
195192
check.Reason = "The device management utility was not found; check the driver and management tool installation."
196-
case collectErr != nil:
197-
check.Status = "failed"
198-
check.Reason = collectErr.Error()
199193
default:
200194
check.Reason = "The management utility returned no readable devices."
201-
if len(info.Warnings) > 0 {
202-
check.Status = "failed"
203-
check.Reason = strings.Join(info.Warnings, "; ")
204-
}
205195
}
206196
result.Checks = append(result.Checks, check)
207197
}

‎agent/app/service/model_gpu.go‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,14 @@ func LoadModelGPUDevices() dto.ModelGPUDevices {
3131
return result
3232
}
3333
snapshot, err := client.Collect(context.Background())
34-
if err != nil {
34+
if err != nil && snapshot == nil {
3535
result.Warnings = append(result.Warnings, err.Error())
36+
}
37+
if snapshot == nil {
3638
return result
3739
}
3840
result.Warnings = append(result.Warnings, snapshot.Info.Warnings...)
41+
result.VendorWarnings = snapshot.Info.VendorWarnings
3942
for _, monitored := range snapshot.Devices {
4043
if monitored.ParentID != "" {
4144
continue
@@ -44,39 +47,46 @@ func LoadModelGPUDevices() dto.ModelGPUDevices {
4447
devicePath := ""
4548
switch {
4649
case monitored.GPU != nil:
50+
device.Warnings = append(device.Warnings, monitored.GPU.Warnings...)
4751
if monitored.Vendor == "nvidia" && !re.GetRegex(re.NvidiaGPUUUIDPattern).MatchString(monitored.GPU.UUID) {
48-
result.Warnings = append(result.Warnings, fmt.Sprintf("invalid NVIDIA GPU UUID from nvidia-smi: %s", monitored.GPU.UUID))
49-
continue
52+
device.Available = false
53+
device.Warnings = append(device.Warnings, fmt.Sprintf("invalid NVIDIA GPU UUID from nvidia-smi: %s", monitored.GPU.UUID))
5054
}
5155
device.ID = monitored.ID
5256
device.Name = fmt.Sprintf("GPU %d · %s", monitored.Index, monitored.Name)
5357
device.Type = monitored.Vendor
5458
device.UUID = monitored.GPU.UUID
5559
case monitored.XPU != nil:
60+
device.Warnings = append(device.Warnings, monitored.XPU.Warnings...)
61+
device.ID = monitored.ID
62+
device.Name = fmt.Sprintf("XPU %d · %s", monitored.Index, monitored.Name)
63+
device.Type = "intel"
5664
devicePath = monitored.XPU.Basic.DrmDevice
5765
if !re.GetRegex(re.IntelGPUDevicePathPattern).MatchString(devicePath) {
66+
devicePath = ""
5867
directory := filepath.Join("/sys/bus/pci/devices", filepath.Base(monitored.BusID), "drm")
5968
entries, err := os.ReadDir(directory)
6069
if err != nil {
61-
result.Warnings = append(result.Warnings, fmt.Sprintf("resolve Intel GPU %d render device: %v", monitored.Index, err))
62-
continue
63-
}
64-
devicePath = ""
65-
for _, entry := range entries {
66-
if re.GetRegex(re.IntelGPURenderNamePattern).MatchString(entry.Name()) {
67-
devicePath = filepath.Join("/dev/dri", entry.Name())
68-
break
70+
device.Warnings = append(device.Warnings, fmt.Sprintf("resolve Intel GPU %d render device: %v", monitored.Index, err))
71+
} else {
72+
for _, entry := range entries {
73+
if re.GetRegex(re.IntelGPURenderNamePattern).MatchString(entry.Name()) {
74+
devicePath = filepath.Join("/dev/dri", entry.Name())
75+
break
76+
}
77+
}
78+
if devicePath == "" {
79+
device.Warnings = append(device.Warnings, fmt.Sprintf("Intel GPU %d has no render device", monitored.Index))
6980
}
70-
}
71-
if devicePath == "" {
72-
result.Warnings = append(result.Warnings, fmt.Sprintf("Intel GPU %d has no render device", monitored.Index))
73-
continue
7481
}
7582
}
76-
device.ID = "intel:" + devicePath
77-
device.Name = fmt.Sprintf("XPU %d · %s", monitored.Index, monitored.Name)
78-
device.Type = "intel"
83+
if devicePath == "" {
84+
device.Available = false
85+
} else {
86+
device.ID = "intel:" + devicePath
87+
}
7988
case monitored.NPU != nil:
89+
device.Warnings = append(device.Warnings, monitored.NPU.Warnings...)
8090
device.ID = fmt.Sprintf("ascend:%d", monitored.Index)
8191
device.Name = fmt.Sprintf("NPU %d · Chip %d · %s", monitored.NPUIndex, monitored.ChipIndex, monitored.Name)
8292
device.Type = "ascend"
@@ -88,7 +98,7 @@ func LoadModelGPUDevices() dto.ModelGPUDevices {
8898
info, err := os.Stat(devicePath)
8999
device.Available = err == nil && info.Mode()&os.ModeCharDevice != 0
90100
if err != nil {
91-
result.Warnings = append(result.Warnings, err.Error())
101+
device.Warnings = append(device.Warnings, err.Error())
92102
}
93103
}
94104
device.NUMANode = loadModelGPUNode("/sys", monitored.BusID)

‎agent/app/service/model_gpu_usage.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
func LoadModelGPUUsage(ctx context.Context, req dto.ModelGPUUsageRequest) (dto.ModelGPUUsage, error) {
2020
topology := LoadModelGPUDevices()
2121
result := dto.ModelGPUUsage{Devices: []dto.ModelGPUUsageDevice{}, Warnings: topology.Warnings}
22+
result.Warnings = append(result.Warnings, topology.VendorWarnings[req.ImageType]...)
2223
values, err := godotenv.Unmarshal(req.Env)
2324
if err != nil {
2425
return result, err

‎agent/utils/ai_tools/accelerator/accelerator.go‎

Lines changed: 53 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ type Client struct {
1818
}
1919

2020
type providerResult struct {
21+
kind Kind
2122
snapshot *ProviderSnapshot
2223
err error
2324
}
@@ -48,40 +49,60 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
4849
if err := ctx.Err(); err != nil {
4950
return nil, err
5051
}
51-
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
52+
collectCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
5253
defer cancel()
5354
results := make([]providerResult, len(c.providers))
5455
var wg sync.WaitGroup
5556
for index, item := range c.providers {
5657
wg.Add(1)
5758
go func() {
5859
defer wg.Done()
59-
providerSnapshot, err := item.Collect(ctx)
60+
providerSnapshot, err := item.Collect(collectCtx)
6061
if err != nil {
6162
err = fmt.Errorf("%s provider failed: %w", item.Kind(), err)
6263
}
63-
results[index] = providerResult{snapshot: providerSnapshot, err: err}
64+
results[index] = providerResult{kind: item.Kind(), snapshot: providerSnapshot, err: err}
6465
}()
6566
}
6667
wg.Wait()
6768
if err := ctx.Err(); err != nil {
6869
return nil, err
6970
}
7071

71-
snapshot := &Snapshot{}
72+
snapshot := &Snapshot{Info: Info{VendorWarnings: map[string][]string{}}}
7273
var (
7374
errs []error
7475
active []*ProviderSnapshot
7576
xpuVersion string
7677
)
78+
if err := collectCtx.Err(); err != nil {
79+
snapshot.Info.Warnings = append(snapshot.Info.Warnings, err.Error())
80+
}
7781
for _, result := range results {
78-
if result.err != nil {
79-
errs = append(errs, result.err)
80-
continue
82+
vendor := ""
83+
switch result.kind {
84+
case KindXPU:
85+
vendor = "intel"
86+
case KindNPU:
87+
vendor = "ascend"
88+
}
89+
if result.err != nil && (result.snapshot == nil || len(result.snapshot.VendorWarnings) == 0) {
90+
if vendor == "" {
91+
snapshot.Info.Warnings = append(snapshot.Info.Warnings, result.err.Error())
92+
} else {
93+
snapshot.Info.VendorWarnings[vendor] = append(snapshot.Info.VendorWarnings[vendor], result.err.Error())
94+
}
8195
}
8296
if result.snapshot != nil {
83-
for _, warning := range result.snapshot.Warnings {
84-
errs = append(errs, errors.New(warning))
97+
for key, warnings := range result.snapshot.VendorWarnings {
98+
if len(warnings) > 0 {
99+
snapshot.Info.VendorWarnings[key] = append(snapshot.Info.VendorWarnings[key], warnings...)
100+
}
101+
}
102+
if vendor != "" && len(result.snapshot.Warnings) > 0 {
103+
snapshot.Info.VendorWarnings[vendor] = append(snapshot.Info.VendorWarnings[vendor], result.snapshot.Warnings...)
104+
} else {
105+
snapshot.Info.Warnings = append(snapshot.Info.Warnings, result.snapshot.Warnings...)
85106
}
86107
}
87108
if result.snapshot == nil || len(result.snapshot.Devices) == 0 {
@@ -100,17 +121,33 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
100121
xpuVersion = item.DriverVersion
101122
}
102123
}
103-
snapshot.Warnings = append(snapshot.Warnings, errs...)
104-
if len(snapshot.Devices) == 0 && len(errs) > 0 {
105-
return nil, fmt.Errorf("calling accelerator monitoring tools failed: %w", errors.Join(errs...))
124+
warnings := append([]string{}, snapshot.Info.Warnings...)
125+
for _, vendorWarnings := range snapshot.Info.VendorWarnings {
126+
warnings = append(warnings, vendorWarnings...)
106127
}
107-
108-
snapshot.Info.CollectedAt = time.Now()
109-
for _, err := range errs {
110-
snapshot.Info.Warnings = append(snapshot.Info.Warnings, err.Error())
128+
for _, device := range snapshot.Devices {
129+
if device.ParentID != "" {
130+
continue
131+
}
132+
switch {
133+
case device.GPU != nil:
134+
warnings = append(warnings, device.GPU.Warnings...)
135+
case device.NPU != nil:
136+
warnings = append(warnings, device.NPU.Warnings...)
137+
case device.XPU != nil:
138+
warnings = append(warnings, device.XPU.Warnings...)
139+
}
111140
}
141+
for _, warning := range warnings {
142+
errs = append(errs, errors.New(warning))
143+
}
144+
snapshot.Warnings = errs
145+
snapshot.Info.CollectedAt = time.Now()
112146
snapshot.Info.XPUDriverVersion = xpuVersion
113147
snapshot.Info.Type, snapshot.Info.DriverVersion = mergeProviderMetadata(active)
148+
if len(snapshot.Devices) == 0 && len(errs) > 0 {
149+
return snapshot, fmt.Errorf("calling accelerator monitoring tools failed: %w", errors.Join(errs...))
150+
}
114151
return snapshot, nil
115152
}
116153

‎agent/utils/ai_tools/accelerator/provider.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,21 @@ func (p gpuProvider) Kind() Kind { return KindGPU }
2222

2323
func (p gpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) {
2424
info, err := p.client.LoadInfoContext(ctx)
25-
if err != nil {
25+
if info == nil {
2626
return nil, err
2727
}
2828
result := &ProviderSnapshot{
29-
Type: info.Type,
30-
Warnings: info.Warnings,
31-
DriverVersion: info.DriverVersion,
32-
CudaVersion: info.CudaVersion,
33-
GPUs: info.Devices,
29+
VendorWarnings: info.VendorWarnings,
30+
Type: info.Type,
31+
Warnings: info.Warnings,
32+
DriverVersion: info.DriverVersion,
33+
CudaVersion: info.CudaVersion,
34+
GPUs: info.Devices,
3435
}
3536
for index := range result.GPUs {
3637
result.Devices = append(result.Devices, normalizeGPU(&result.GPUs[index]))
3738
}
38-
return result, nil
39+
return result, err
3940
}
4041

4142
type npuProvider struct {

‎agent/utils/ai_tools/accelerator/types.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ const (
1818
)
1919

2020
type Info struct {
21-
CollectedAt time.Time `json:"collectedAt"`
22-
Warnings []string `json:"warnings"`
21+
VendorWarnings map[string][]string `json:"vendorWarnings,omitempty"`
22+
CollectedAt time.Time `json:"collectedAt"`
23+
Warnings []string `json:"warnings"`
2324

2425
Type string `json:"type"`
2526
CudaVersion string `json:"cudaVersion"`
@@ -124,12 +125,13 @@ func (s Snapshot) Warning() error {
124125
}
125126

126127
type ProviderSnapshot struct {
127-
Warnings []string
128-
Type string
129-
DriverVersion string
130-
CudaVersion string
131-
GPUs []gpu.Device
132-
NPUs []npu.Device
133-
XPUs []xpu.Device
134-
Devices []Device
128+
VendorWarnings map[string][]string
129+
Warnings []string
130+
Type string
131+
DriverVersion string
132+
CudaVersion string
133+
GPUs []gpu.Device
134+
NPUs []npu.Device
135+
XPUs []xpu.Device
136+
Devices []Device
135137
}

0 commit comments

Comments
 (0)