Add metric for total number of shards (#891)

Currently, we only record the number of compound shards. This PR adds a metric
for the total number of index shards. This metric is helpful for alerting, as
it's important to catch a significant decrease in the number of indexed data.

cmd/zoekt-sourcegraph-indexserver/cleanup.go

@@ -379,6 +379,11 @@ var metricVacuumRunning = promauto.NewGauge(prometheus.GaugeOpts{
 	Help: "Set to 1 if indexserver's vacuum job is running.",
 })
 
+var metricNumberShards = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "index_number_shards",
+	Help: "The number of total shards.",
+})
+
 var metricNumberCompoundShards = promauto.NewGauge(prometheus.GaugeOpts{
 	Name: "index_number_compound_shards",
 	Help: "The number of compound shards.",

cmd/zoekt-sourcegraph-indexserver/main.go

@@ -376,7 +376,7 @@ func (s *Server) Run() {
 			missing := s.queue.Bump(repos.IDs)
 			s.Sourcegraph.ForceIterateIndexOptions(s.queue.AddOrUpdate, func(uint32, error) {}, missing...)
 
-			setCompoundShardCounter(s.IndexDir)
+			setShardsCounter(s.IndexDir)
 
 			<-cleanupDone
 		}
@@ -1208,12 +1208,20 @@ func joinStringSet(set map[string]struct{}, sep string) string {
 	return strings.Join(xs, sep)
 }
 
-func setCompoundShardCounter(indexDir string) {
-	fns, err := filepath.Glob(filepath.Join(indexDir, "compound-*.zoekt"))
+func setShardsCounter(indexDir string) {
+	fns, err := filepath.Glob(filepath.Join(indexDir, "*.zoekt"))
 	if err != nil {
-		errorLog.Printf("setCompoundShardCounter: %s\n", err)
+		errorLog.Printf("setShardsCounter: %s\n", err)
 		return
 	}
+	metricNumberShards.Set(float64(len(fns)))
+
+	compoundFns := make([]string, 0, len(fns))
+	for _, fn := range fns {
+		if strings.HasPrefix(filepath.Base(fn), "compound-") {
+			compoundFns = append(compoundFns, fn)
+		}
+	}
 	metricNumberCompoundShards.Set(float64(len(fns)))
 }
 
@@ -1288,7 +1296,7 @@ func startServer(conf rootConfig) error {
 	}
 
 	profiler.Init("zoekt-sourcegraph-indexserver")
-	setCompoundShardCounter(s.IndexDir)
+	setShardsCounter(s.IndexDir)
 
 	if conf.listen != "" {