Add Outdated Metric (#71)

Add separate metric for when a chart has a newer version

.gitignore

@@ -1 +1,2 @@
 helm/helm-exporter*.tgz
+.idea/

README.md

@@ -64,9 +64,24 @@ config:
 
 # Format
 ```
+# HELP helm_chart_info Information on helm releases
+# TYPE helm_chart_info gauge
 helm_chart_info{chart="ark",release="ark",version="1.2.1",latestVersion="1.2.3",appVersion="1.2.3",updated="1553201431",namespace="test"} 1
 helm_chart_info{chart="cluster-autoscaler",release="cluster-autoscaler",version="0.7.0",latestVersion=7.3.2,appVersion="",updated="1553201431",namespace="other"} 4
 helm_chart_info{chart="dex",release="dex",version="0.1.0",latestVersion="3.4.0",appVersion="1.2.3",updated="1553201431",namespace="test"} 1
+
+# HELP helm_chart_outdated Outdated helm versions of helm releases
+# TYPE helm_chart_outdated gauge
+helm_chart_outdated{chart="ark",latestVersion="1.2.3",namespace="test",release="ark",version="1.2.1"} 1
+helm_chart_outdated{chart="cluster-autoscaler",latestVersion="7.3.2",namespace="other",release="cluster-autoscaler",version="0.7.0"} 1
+helm_chart_outdated{chart="external-secrets",latestVersion="3.4.0",namespace="test",release="dex",version="0.1.0"} 1
+
+# HELP helm_chart_timestamp Timestamps of helm releases
+# TYPE helm_chart_timestamp gauge
+helm_chart_timestamp{chart="ark",release="ark",version="1.2.1",latestVersion="1.2.3",appVersion="1.2.3",updated="1553201431",namespace="test"} 1.617197959e+12
+helm_chart_timestamp{chart="cluster-autoscaler",release="cluster-autoscaler",version="0.7.0",latestVersion=7.3.2,appVersion="",updated="1553201431",namespace="other"} 1.617196128e+12
+helm_chart_timestamp{chart="dex",release="dex",version="0.1.0",latestVersion="3.4.0",appVersion="1.2.3",updated="1553201431",namespace="test"} 1.62245881e+12
+
 ```
 
 The metric value is the helm status code.  These status codes indexes do not map up directly to helm.  This is so I can make the bad cases negative values.

go.mod

@@ -3,6 +3,17 @@ module github.com/sstarcher/helm-exporter
 go 1.17
 
 require (
+	github.com/Masterminds/semver v1.5.0
+	github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456
+	github.com/knadh/koanf v1.2.4
+	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
+	github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc
+	github.com/prometheus/client_golang v1.11.0
+	github.com/sirupsen/logrus v1.8.1
+	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.7.0
+	k8s.io/apimachinery v0.22.2
+	k8s.io/client-go v0.22.2
 	cloud.google.com/go v0.97.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -40,7 +51,6 @@ require (
 	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
-	github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/fatih/color v1.13.0 // indirect
@@ -73,7 +83,6 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/knadh/koanf v1.2.4
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.3 // indirect
@@ -83,7 +92,6 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.2 // indirect
@@ -97,19 +105,16 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.31.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
-	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.2.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -135,15 +140,11 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	helm.sh/helm/v3 v3.7.0
 	k8s.io/api v0.22.2 // indirect
 	k8s.io/apiextensions-apiserver v0.22.2 // indirect
-	k8s.io/apimachinery v0.22.2
 	k8s.io/apiserver v0.22.2 // indirect
 	k8s.io/cli-runtime v0.22.2 // indirect
-	k8s.io/client-go v0.22.2
 	k8s.io/component-base v0.22.2 // indirect
 	k8s.io/klog/v2 v2.20.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210929172449-94abcedd1aa4 // indirect

main.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	semver "github.com/Masterminds/semver"
 
 	"github.com/sstarcher/helm-exporter/config"
 
@@ -41,6 +42,7 @@ var (
 
 	statsInfo      *prometheus.GaugeVec
 	statsTimestamp *prometheus.GaugeVec
+	statsOutdated  *prometheus.GaugeVec
 
 	namespaces = flag.String("namespaces", "", "namespaces to monitor.  Defaults to all")
 	configFile = flag.String("config", "", "Configfile to load for helm overwrite registries.  Default is empty")
@@ -49,6 +51,7 @@ var (
 
 	infoMetric      = flag.Bool("info-metric", true, "Generate info metric.  Defaults to true")
 	timestampMetric = flag.Bool("timestamp-metric", true, "Generate timestamps metric.  Defaults to true")
+	outdatedMetric = flag.Bool("outdated-metric", true, "Generate version outdated metric.  Defaults to true")
 
 	fetchLatest = flag.Bool("latest-chart-version", true, "Attempt to fetch the latest chart version from registries. Defaults to true")
 
@@ -69,7 +72,7 @@ var (
 	prometheusHandler = promhttp.Handler()
 )
 
-func configureMetrics() (info *prometheus.GaugeVec, timestamp *prometheus.GaugeVec) {
+func configureMetrics() (info *prometheus.GaugeVec, timestamp *prometheus.GaugeVec, outdated *prometheus.GaugeVec) {
 	if *infoMetric == true {
 		info = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "helm_chart_info",
@@ -81,8 +84,7 @@ func configureMetrics() (info *prometheus.GaugeVec, timestamp *prometheus.GaugeV
 			"appVersion",
 			"updated",
 			"namespace",
-			"latestVersion",
-		})
+			"latestVersion"})
 	}
 
 	if *timestampMetric == true {
@@ -96,21 +98,36 @@ func configureMetrics() (info *prometheus.GaugeVec, timestamp *prometheus.GaugeV
 			"appVersion",
 			"updated",
 			"namespace",
-			"latestVersion",
-		})
+			"latestVersion"})
+	}
+
+	if *outdatedMetric == true {
+		outdated = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "helm_chart_outdated",
+			Help: "Outdated helm versions of helm releases",
+		}, []string{
+			"chart",
+			"release",
+			"version",
+			"namespace",
+			"latestVersion"})
 	}
 
 	return
 }
 
-func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *prometheus.GaugeVec) {
+func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *prometheus.GaugeVec, outdated *prometheus.GaugeVec) {
 	if info != nil {
 		info.Reset()
 	}
 	if timestamp != nil {
 		timestamp.Reset()
 	}
 
+	if outdated != nil {
+		outdated.Reset()
+	}
+
 	for _, client := range clients.Items() {
 		list := action.NewList(client.(*action.Configuration))
 		items, err := list.Run()
@@ -133,6 +150,22 @@ func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *promet
 				latestVersion = config.HelmRegistries.GetLatestVersionFromHelm(item.Chart.Name())
 			}
 
+			lv, err := semver.NewVersion(latestVersion)
+			if err == nil {
+				log.WithField("chart", chart).WithField("version", version).WithField("latest", latestVersion).Debug("Comparing versions")
+				lc, err := semver.NewConstraint(">" + version)
+				if err == nil {
+					a := lc.Check(lv)
+					if a {
+						if outdated != nil {
+							outdated.WithLabelValues(chart, releaseName, version, namespace, latestVersion).Set(1)
+						}
+					}
+				} else {
+					log.WithField("chart", chart).WithField("version", version).WithField("latest", latestVersion).Error("%s", err)
+				}
+            }
+
 			if info != nil {
 				info.WithLabelValues(chart, releaseName, version, appVersion, strconv.FormatInt(updated, 10), namespace, latestVersion).Set(status)
 			}
@@ -145,14 +178,14 @@ func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *promet
 
 func runStatsPeriodically(interval time.Duration, config config.Config) {
 	for {
-		info, timestamp := configureMetrics()
-		runStats(config, info, timestamp)
-		registerMetrics(prometheus.DefaultRegisterer, info, timestamp)
+		info, timestamp, outdated := configureMetrics()
+		runStats(config, info, timestamp, outdated)
+		registerMetrics(prometheus.DefaultRegisterer, info, timestamp, outdated)
 		time.Sleep(interval)
 	}
 }
 
-func registerMetrics(register prometheus.Registerer, info, timestamp *prometheus.GaugeVec) {
+func registerMetrics(register prometheus.Registerer, info, timestamp *prometheus.GaugeVec, outdated *prometheus.GaugeVec) {
 	mutex.Lock()
 	defer mutex.Unlock()
 
@@ -167,12 +200,18 @@ func registerMetrics(register prometheus.Registerer, info, timestamp *prometheus
 	}
 	register.MustRegister(timestamp)
 	statsTimestamp = timestamp
+
+	if statsOutdated != nil {
+		register.Unregister(statsOutdated)
+	}
+	register.MustRegister(outdated)
+	statsOutdated = outdated
 }
 
 func newHelmStatsHandler(config config.Config, synchrone bool) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if synchrone {
-			runStats(config, statsInfo, statsTimestamp)
+			runStats(config, statsInfo, statsTimestamp,statsOutdated)
 		} else {
 			mutex.RLock()
 			defer mutex.RUnlock()
@@ -256,8 +295,8 @@ func main() {
 	if runIntervalDuration != 0 {
 		go runStatsPeriodically(runIntervalDuration, config)
 	} else {
-		info, timestamp := configureMetrics()
-		registerMetrics(prometheus.DefaultRegisterer, info, timestamp)
+		info, timestamp, outdated := configureMetrics()
+		registerMetrics(prometheus.DefaultRegisterer, info, timestamp, outdated)
 	}
 
 	http.HandleFunc("/metrics", newHelmStatsHandler(config, runIntervalDuration == 0))

versioning/versioning.go

@@ -103,4 +103,4 @@ func DetermineLifeCycleStatus(latestVersion string, currentVersion string) strin
 	}
 
 	return Unknown
-}
+}