diff --git a/cmd/cadvisor.go b/cmd/cadvisor.go index fc34d967bd..83614d5008 100644 --- a/cmd/cadvisor.go +++ b/cmd/cadvisor.go @@ -83,6 +83,7 @@ var ( container.NetworkTcpUsageMetrics: struct{}{}, container.NetworkUdpUsageMetrics: struct{}{}, container.NetworkAdvancedTcpUsageMetrics: struct{}{}, + container.NetworkAdvancedUdpUsageMetrics: struct{}{}, container.ProcessSchedulerMetrics: struct{}{}, container.ProcessMetrics: struct{}{}, container.HugetlbUsageMetrics: struct{}{}, diff --git a/cmd/cadvisor_test.go b/cmd/cadvisor_test.go index 58461ae182..3df5b993e9 100644 --- a/cmd/cadvisor_test.go +++ b/cmd/cadvisor_test.go @@ -41,6 +41,12 @@ func TestUdpMetricsAreDisabledByDefault(t *testing.T) { assert.True(t, ignoreMetrics.Has(container.NetworkUdpUsageMetrics)) } +func TestAdvancedUdpMetricsAreDisabledByDefault(t *testing.T) { + assert.True(t, ignoreMetrics.Has(container.NetworkAdvancedUdpUsageMetrics)) + flag.Parse() + assert.True(t, ignoreMetrics.Has(container.NetworkAdvancedUdpUsageMetrics)) +} + func TestReferencedMemoryMetricsIsDisabledByDefault(t *testing.T) { assert.True(t, ignoreMetrics.Has(container.ReferencedMemoryMetrics)) flag.Parse() @@ -103,6 +109,7 @@ func TestToIncludedMetrics(t *testing.T) { container.NetworkTcpUsageMetrics: struct{}{}, container.NetworkAdvancedTcpUsageMetrics: struct{}{}, container.NetworkUdpUsageMetrics: struct{}{}, + container.NetworkAdvancedUdpUsageMetrics: struct{}{}, container.ProcessMetrics: struct{}{}, container.AppMetrics: struct{}{}, container.HugetlbUsageMetrics: struct{}{}, diff --git a/container/factory.go b/container/factory.go index c48a64e163..0451b48cb7 100644 --- a/container/factory.go +++ b/container/factory.go @@ -57,6 +57,7 @@ const ( NetworkTcpUsageMetrics MetricKind = "tcp" NetworkAdvancedTcpUsageMetrics MetricKind = "advtcp" NetworkUdpUsageMetrics MetricKind = "udp" + NetworkAdvancedUdpUsageMetrics MetricKind = "advudp" AppMetrics MetricKind = "app" ProcessMetrics MetricKind = "process" HugetlbUsageMetrics MetricKind = "hugetlb" @@ -82,6 +83,7 @@ var AllMetrics = MetricSet{ NetworkTcpUsageMetrics: struct{}{}, NetworkAdvancedTcpUsageMetrics: struct{}{}, NetworkUdpUsageMetrics: struct{}{}, + NetworkAdvancedUdpUsageMetrics: struct{}{}, ProcessMetrics: struct{}{}, AppMetrics: struct{}{}, HugetlbUsageMetrics: struct{}{}, @@ -99,6 +101,7 @@ var AllNetworkMetrics = MetricSet{ NetworkTcpUsageMetrics: struct{}{}, NetworkAdvancedTcpUsageMetrics: struct{}{}, NetworkUdpUsageMetrics: struct{}{}, + NetworkAdvancedUdpUsageMetrics: struct{}{}, } func (mk MetricKind) String() string { diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index 5bf1a4f997..63824425f0 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -159,6 +159,14 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { stats.Network.Udp6 = u6 } } + if h.includedMetrics.Has(container.NetworkAdvancedUdpUsageMetrics) { + ua, err := advancedUDPStatsFromProc(h.rootFs, h.pid, "net/snmp") + if err != nil { + klog.V(4).Infof("Unable to get advanced udp stats from pid %d: %v", h.pid, err) + } else { + stats.Network.UdpAdvanced = ua + } + } } // some process metrics are per container ( number of processes, number of // file descriptors etc.) and not required a proper container's @@ -580,48 +588,55 @@ func scanAdvancedTCPStats(advancedStats *info.TcpAdvancedStat, advancedTCPStatsF scanner := bufio.NewScanner(reader) scanner.Split(bufio.ScanLines) - advancedTCPStats := make(map[string]interface{}) + advancedTCPStats, err := ParseStatsFile(scanner, map[string]struct{}{"Tcp": {}, "TcpExt": {}}) + if err != nil { + return fmt.Errorf("failure parsing stats file %s: %v", advancedTCPStatsFile, err) + } + + b, err := json.Marshal(advancedTCPStats) + if err != nil { + return err + } + + err = json.Unmarshal(b, advancedStats) + if err != nil { + return err + } + + return scanner.Err() +} + +func ParseStatsFile(scanner *bufio.Scanner, protocols map[string]struct{}) (map[string]interface{}, error) { + stats := make(map[string]interface{}) for scanner.Scan() { nameParts := strings.Split(scanner.Text(), " ") scanner.Scan() valueParts := strings.Split(scanner.Text(), " ") - // Remove trailing :. and ignore non-tcp + // Remove trailing :. and ignore other protocols protocol := nameParts[0][:len(nameParts[0])-1] - if protocol != "TcpExt" && protocol != "Tcp" { + if _, ok := protocols[protocol]; !ok { continue } if len(nameParts) != len(valueParts) { - return fmt.Errorf("mismatch field count mismatch in %s: %s", - advancedTCPStatsFile, protocol) + return nil, fmt.Errorf("mismatch field count mismatch in %s", protocol) } for i := 1; i < len(nameParts); i++ { if strings.Contains(valueParts[i], "-") { vInt64, err := strconv.ParseInt(valueParts[i], 10, 64) if err != nil { - return fmt.Errorf("decode value: %s to int64 error: %s", valueParts[i], err) + return nil, fmt.Errorf("decode value: %s to int64 error: %s", valueParts[i], err) } - advancedTCPStats[nameParts[i]] = vInt64 + stats[nameParts[i]] = vInt64 } else { vUint64, err := strconv.ParseUint(valueParts[i], 10, 64) if err != nil { - return fmt.Errorf("decode value: %s to uint64 error: %s", valueParts[i], err) + return nil, fmt.Errorf("decode value: %s to uint64 error: %s", valueParts[i], err) } - advancedTCPStats[nameParts[i]] = vUint64 + stats[nameParts[i]] = vUint64 } } } - - b, err := json.Marshal(advancedTCPStats) - if err != nil { - return err - } - - err = json.Unmarshal(b, advancedStats) - if err != nil { - return err - } - - return scanner.Err() + return stats, nil } func scanTCPStats(tcpStatsFile string) (info.TcpStat, error) { @@ -706,6 +721,47 @@ func udpStatsFromProc(rootFs string, pid int, file string) (info.UdpStat, error) return udpStats, nil } +func advancedUDPStatsFromProc(rootFs string, pid int, file string) (info.UdpAdvancedStat, error) { + var advancedStats info.UdpAdvancedStat + var err error + + netstatFile := path.Join(rootFs, "proc", strconv.Itoa(pid), file) + err = scanAdvancedUDPStats(&advancedStats, netstatFile) + if err != nil { + return advancedStats, err + } + + return advancedStats, nil +} + +func scanAdvancedUDPStats(advancedStats *info.UdpAdvancedStat, advancedUDPStatsFile string) error { + data, err := os.ReadFile(advancedUDPStatsFile) + if err != nil { + return fmt.Errorf("failure opening %s: %v", advancedUDPStatsFile, err) + } + + reader := strings.NewReader(string(data)) + scanner := bufio.NewScanner(reader) + scanner.Split(bufio.ScanLines) + + advancedUDPStats, err := ParseStatsFile(scanner, map[string]struct{}{"Udp": {}}) + if err != nil { + return fmt.Errorf("failure parsing stats file %s: %v", advancedUDPStats, err) + } + + b, err := json.Marshal(advancedUDPStats) + if err != nil { + return err + } + + err = json.Unmarshal(b, advancedStats) + if err != nil { + return err + } + + return scanner.Err() +} + func scanUDPStats(r io.Reader) (info.UdpStat, error) { var stats info.UdpStat diff --git a/container/libcontainer/handler_test.go b/container/libcontainer/handler_test.go index 82da0b3e67..d59b648819 100644 --- a/container/libcontainer/handler_test.go +++ b/container/libcontainer/handler_test.go @@ -15,8 +15,10 @@ package libcontainer import ( + "bufio" "os" "reflect" + "strings" "testing" "github.com/opencontainers/runc/libcontainer/cgroups" @@ -91,6 +93,72 @@ func TestScanUDPStats(t *testing.T) { } } +func TestParseStatsFile(t *testing.T) { + snmpFile := "testdata/snmp" + data, err := os.ReadFile(snmpFile) + if err != nil { + t.Errorf("failure opening %s: %v", snmpFile, err) + } + + reader := strings.NewReader(string(data)) + scanner := bufio.NewScanner(reader) + scanner.Split(bufio.ScanLines) + + stats, err := ParseStatsFile(scanner, map[string]struct{}{"Tcp": {}}) + + if err != nil { + t.Error(err) + } + + if RtoAlgorithm, ok := stats["RtoAlgorithm"].(uint64); ok { + if RtoAlgorithm != 1 { + t.Errorf("Expected Tcp stats RtoAlgorithm 1, got %d", RtoAlgorithm) + } + } else { + t.Errorf("Expected Tcp stats RtoAlgorithm, got nil") + } + + if _, ok := stats["Forwarding"]; ok { + t.Errorf("Expected Ip stats Forwarding to be nil, got %v", stats["Forwarding"]) + } + if _, ok := stats["InDatagrams"]; ok { + t.Errorf("Expected Udp stats InDatagrams to be nil, got %v", stats["InDatagrams"]) + } + if _, ok := stats["InType0"]; ok { + t.Errorf("Expected Icmp stats InType0 to be nil, got %v", stats["InType0"]) + } +} + +func TestScanAdvancedTCPStats(t *testing.T) { + snmpFile := "testdata/snmp" + advancedStats := info.TcpAdvancedStat{} + err := scanAdvancedTCPStats(&advancedStats, snmpFile) + if err != nil { + t.Error(err) + } + if advancedStats.RtoAlgorithm != 1 { + t.Errorf("Expected RtoAlgorithm 1, got %d", advancedStats.RtoAlgorithm) + } + if advancedStats.MaxConn != -1 { + t.Errorf("Expected MaxConn -1, got %d", advancedStats.RtoMin) + } +} + +func TestScanAdvanceUDPStats(t *testing.T) { + snmpFile := "testdata/snmp" + advancedStats := info.UdpAdvancedStat{} + err := scanAdvancedUDPStats(&advancedStats, snmpFile) + if err != nil { + t.Error(err) + } + if advancedStats.InErrors != 8 { + t.Errorf("Expected InErrors 0, got %d", advancedStats.InErrors) + } + if advancedStats.NoPorts != 8996 { + t.Errorf("Expected NoPorts 0, got %d", advancedStats.NoPorts) + } +} + // https://github.com/docker/libcontainer/blob/v2.2.1/cgroups/fs/cpuacct.go#L19 const nanosecondsInSeconds = 1000000000 diff --git a/container/libcontainer/testdata/snmp b/container/libcontainer/testdata/snmp new file mode 100644 index 0000000000..70e83c8b09 --- /dev/null +++ b/container/libcontainer/testdata/snmp @@ -0,0 +1,12 @@ +Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates +Ip: 1 64 22010002886 126 0 3897 0 0 21992054949 20118212517 1 3835921 465 396834 198179 465 250500 54 501915 +Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps +Icmp: 44109161 84392 2 344441 2262297 0 0 79 25614079 15888259 1 0 2 0 54115640 0 4100211 227 0 0 0 24401136 25614065 0 1 0 0 +IcmpMsg: InType0 InType3 InType5 InType8 InType11 InType13 InType17 InType165 OutType0 OutType3 OutType8 OutType11 OutType14 +IcmpMsg: 15888259 344441 79 25614079 2262297 1 2 1 25614065 4100211 24401136 227 1 +Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors +Tcp: 1 200 120000 -1 241161083 2999339 892122 2075279 493 21773277356 28027757848 82377554 39722 274330587 0 +Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti +Udp: 174675874 8996 8 174697410 8 0 0 0 +UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti +UdpLite: 2 1 0 3 0 0 0 0 \ No newline at end of file diff --git a/info/v1/container.go b/info/v1/container.go index ae1d9caecc..93840d3c29 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -463,6 +463,8 @@ type NetworkStats struct { Udp6 UdpStat `json:"udp6"` // TCP advanced stats TcpAdvanced TcpAdvancedStat `json:"tcp_advanced"` + // UDP advanced stats + UdpAdvanced UdpAdvancedStat `json:"udp_advanced"` } type TcpStat struct { @@ -743,6 +745,34 @@ type UdpStat struct { TxQueued uint64 } +type UdpAdvancedStat struct { + // The total number of datagrams successfully received, excluding those + // discarded due to no ports, errors, etc. + InDatagrams uint64 + + // The number of datagrams discarded because no listening ports were available. + NoPorts uint64 + + // The number of datagrams discarded due to errors, including checksum errors, + // receive buffer errors, and other related issues. + InErrors uint64 + + // The total number of datagrams successfully transmitted. + OutDatagrams uint64 + + // The number of datagrams dropped due to insufficient memory in the received buffer. + RcvbufErrors uint64 + + // The number of datagrams dropped due to insufficient memory in the send buffer. + SndbufErrors uint64 + + // The number of datagrams discarded due to invalid checksums. + InCsumErrors uint64 + + // The number of datagrams discarded because multicast packets were ignored. + IgnoredMulti uint64 +} + type FsStats struct { // The block device name associated with the filesystem. Device string `json:"device,omitempty"` diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 86064819d3..3de98ff8a7 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -1515,6 +1515,60 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }...) } + if includedMetrics.Has(container.NetworkAdvancedUdpUsageMetrics) { + c.containerMetrics = append(c.containerMetrics, []containerMetric{ + { + name: "container_network_advance_udp_stats_total", + help: "advance udp statistic for container", + valueType: prometheus.CounterValue, + extraLabels: []string{"udp_state"}, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{ + { + value: float64(s.Network.UdpAdvanced.InDatagrams), + labels: []string{"indatagrams"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.NoPorts), + labels: []string{"noports"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.InErrors), + labels: []string{"inerrors"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.OutDatagrams), + labels: []string{"outdatagrams"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.RcvbufErrors), + labels: []string{"rcvbuferrors"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.SndbufErrors), + labels: []string{"sndbuferrors"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.InCsumErrors), + labels: []string{"incsumerrors"}, + timestamp: s.Timestamp, + }, + { + value: float64(s.Network.UdpAdvanced.IgnoredMulti), + labels: []string{"ignoredmulti"}, + timestamp: s.Timestamp, + }, + } + }, + }, + }...) + } if includedMetrics.Has(container.ProcessMetrics) { c.containerMetrics = append(c.containerMetrics, []containerMetric{ { diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index fd43b78148..9910e2bbbb 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -535,6 +535,16 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req RxQueued: 0, TxQueued: 0, }, + UdpAdvanced: info.UdpAdvancedStat{ + InDatagrams: 174675874, + NoPorts: 8996, + InErrors: 8, + OutDatagrams: 174697410, + RcvbufErrors: 8, + SndbufErrors: 0, + InCsumErrors: 0, + IgnoredMulti: 0, + }, }, DiskIo: info.DiskIoStats{ IoServiceBytes: []info.PerDiskStats{{ diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index a385e50689..6b6382ca4d 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -296,6 +296,16 @@ container_network_advance_tcp_stats_total{container_env_foo_env="prod",container container_network_advance_tcp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",tcp_state="tw",zone_name="hello"} 1.0436427e+07 1395066363000 container_network_advance_tcp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",tcp_state="twkilled",zone_name="hello"} 0 1395066363000 container_network_advance_tcp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",tcp_state="twrecycled",zone_name="hello"} 0 1395066363000 +# HELP container_network_advance_udp_stats_total advance udp statistic for container +# TYPE container_network_advance_udp_stats_total counter +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="ignoredmulti",zone_name="hello"} 0 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="incsumerrors",zone_name="hello"} 0 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="indatagrams",zone_name="hello"} 1.74675874e+08 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="inerrors",zone_name="hello"} 8 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="noports",zone_name="hello"} 8996 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="outdatagrams",zone_name="hello"} 1.7469741e+08 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="rcvbuferrors",zone_name="hello"} 8 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",udp_state="sndbuferrors",zone_name="hello"} 0 1395066363000 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter container_network_receive_bytes_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 1395066363000 diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index 921b2e1106..8863f0f1b4 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -296,6 +296,16 @@ container_network_advance_tcp_stats_total{container_env_foo_env="prod",id="testc container_network_advance_tcp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",tcp_state="tw",zone_name="hello"} 1.0436427e+07 1395066363000 container_network_advance_tcp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",tcp_state="twkilled",zone_name="hello"} 0 1395066363000 container_network_advance_tcp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",tcp_state="twrecycled",zone_name="hello"} 0 1395066363000 +# HELP container_network_advance_udp_stats_total advance udp statistic for container +# TYPE container_network_advance_udp_stats_total counter +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="ignoredmulti",zone_name="hello"} 0 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="incsumerrors",zone_name="hello"} 0 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="indatagrams",zone_name="hello"} 1.74675874e+08 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="inerrors",zone_name="hello"} 8 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="noports",zone_name="hello"} 8996 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="outdatagrams",zone_name="hello"} 1.7469741e+08 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="rcvbuferrors",zone_name="hello"} 8 1395066363000 +container_network_advance_udp_stats_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",udp_state="sndbuferrors",zone_name="hello"} 0 1395066363000 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter container_network_receive_bytes_total{container_env_foo_env="prod",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 1395066363000