Cortex kv panels: narrow to one instance

Distributor uses multiple kv stores - for global limits and ha-tracker,
as well as reading from the ingester ring - so we need to narrow the
panel to just the one it says it is showing.

For consistency, do the same on the ingester panel, although currently
ingesters only have one kv store.

Note that the renamed recording rule will mean that dashboards show
no data for latency prior to the change.

Author: bboreham

cortex-mixin/dashboards/writes.libsonnet

@@ -104,11 +104,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Key-value store for high-availability (HA) deduplication')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.distributor))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s,kv_name="distributor-hatracker"}' % $.jobMatcher($._config.job_names.distributor))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
+      )
       )
     )
     .addRow(
@@ -133,11 +134,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Key-value store for the ingesters ring')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.ingester))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s,kv_name="ingester-lifecycler"}' % $.jobMatcher($._config.job_names.ingester))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+      )
       )
     )
     .addRowIf(

cortex-mixin/recording_rules.libsonnet

@@ -42,7 +42,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           utils.histogramRules('cortex_chunk_store_chunks_per_query', ['cluster', 'job']) +
           utils.histogramRules('cortex_database_request_duration_seconds', ['cluster', 'job', 'method']) +
           utils.histogramRules('cortex_gcs_request_duration_seconds', ['cluster', 'job', 'operation']) +
-          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job']),
+          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job', 'kv_name']),
       },
       {
         name: 'cortex_queries',