diff --git a/assets/walrus_storage_node.png b/assets/walrus_storage_node.png index 513c20c..a0a8925 100644 Binary files a/assets/walrus_storage_node.png and b/assets/walrus_storage_node.png differ diff --git a/grafana/dashboards/walrus_storage_node.json b/grafana/dashboards/walrus_storage_node.json index 77231ec..99c6d5b 100644 --- a/grafana/dashboards/walrus_storage_node.json +++ b/grafana/dashboards/walrus_storage_node.json @@ -1,3671 +1,5005 @@ { "annotations": { - "list": [{ - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - }] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "description": "This dashboard provides key insights into the performance and health of Storage Operators within the Walrus decentralized storage network. Storage Operators are the backbone of the system, responsible for storing and serving data slivers that collectively form the decentralized storage layer.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 4, "links": [], - "panels": [{ - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the current epoch number in the Walrus network, indicating the network's operational state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] }, - "id": 11, - "panels": [], - "title": "Overview", - "type": "row" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the current build version of the Walrus Storage Node.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(walrus_current_epoch{service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Current Epoch", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Number of shards owned by this storage node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 1 + { + "color": "green", + "value": 1 + } + ] }, - "id": 44, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 1 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 72, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_shards_owned{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Shards Owned", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Number of checkpoints the node is behind according to runtime monitoring (used for health). This is the lag observed by the runtime monitor, which may differ slightly from the checkpoint downloader lag because it reflects a different point in the pipeline (e.g. after event processing) and evaluation time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "walrus_build_info{service=\"storage_node\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - }], - "title": "Build Version", - "transformations": [{ - "id": "labelsToFields", - "options": { - "keepLabels": [ - "version" - ], - "valueLabel": "version" + { + "color": "yellow", + "value": 10 } - }], - "type": "stat" + ] + }, + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 73, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the chain identifier of the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_runtime_lag_current{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Runtime Lag", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Total checkpoint parsing errors (indicates incompatibility of checkpoint data).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 5, - "y": 1 + { + "color": "red", + "value": 1 + } + ] }, - "id": 45, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 1 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 74, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_checkpoint_parsing_errors_count{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Parsing Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Shows the current build version of the Walrus Storage Node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "editorMode": "code", - "expr": "uptime{service=\"storage_node\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }], - "title": "Chain ID", - "transformations": [{ - "id": "labelsToFields", - "options": { - "keepLabels": [ - "chain_identifier" - ], - "valueLabel": "chain_identifier" + { + "color": "green", + "value": 1 } - }], - "type": "stat" + ] + } + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 1 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total uptime of the Walrus Storage Node in days, indicating its operational status.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 0.1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 10, - "y": 1 - }, - "id": 2, + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "walrus_build_info{service=\"storage_node\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Build Version", + "transformations": [ + { + "id": "labelsToFields", "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "keepLabels": [ + "version" + ], + "valueLabel": "version" + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the chain identifier of the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "editorMode": "code", - "expr": "uptime{service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Uptime", - "type": "stat" + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 45, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 1 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "uid": "ctibdrobbprrc6ulkutg" - }, - "description": "Displays the current status of the Walrus storage node.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [{ - "options": { - "Active": { - "color": "green", - "index": 1, - "text": "active" - }, - "Standby": { - "color": "yellow", - "index": 0, - "text": "standby" - } - }, - "type": "value" - }], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 15, - "y": 1 - }, - "id": 63, + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "uptime{service=\"storage_node\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Chain ID", + "transformations": [ + { + "id": "labelsToFields", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^Status$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "columns": [{ - "selector": "data.nodeStatus", - "text": "Status", - "type": "string" - }], - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true + "keepLabels": [ + "chain_identifier" + ], + "valueLabel": "chain_identifier" + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total uptime of the Walrus Storage Node in days, indicating its operational status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "refId": "A", - "root_selector": "$.*", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "data": "", - "method": "GET" + { + "color": "green", + "value": 0.1 } - }], - "title": "Status", - "type": "stat" + ] + }, + "unit": "s" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the current epoch number in the Walrus network, indicating the network's operational state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "uptime{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "uid": "ctibdrobbprrc6ulkutg" + }, + "description": "Displays the current status of the Walrus storage node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Active": { + "color": "green", + "index": 1, + "text": "active" + }, + "Standby": { + "color": "yellow", + "index": 0, + "text": "standby" + } }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 43, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 63, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Status$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "columns": [ + { + "selector": "data.nodeStatus", + "text": "Status", + "type": "string" + } + ], + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "columnar": false, + "root_is_not_array": true + }, + "refId": "A", + "root_selector": "$.*", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET" + } + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the timeline of Walrus Storage Node epoch states, highlighting transitions between inactive (0) and active (1) statuses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 1, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "0": { + "index": 1 + }, + "1": { + "index": 0, + "text": "State" + } }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "editorMode": "code", - "expr": "max(walrus_current_epoch{service=\"storage_node\"})", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Current Epoch", - "type": "stat" + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "fieldOptions": { + "calcs": [], + "values": false + }, + "graphMode": "line", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "showPoints": "auto", + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the timeline of Walrus Storage Node epoch states, highlighting transitions between inactive (0) and active (1) statuses.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 1, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "symlog" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [{ - "options": { - "0": { - "index": 1 - }, - "1": { - "index": 0, - "text": "State" - } - }, - "type": "value" - }], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 4, - "options": { - "fieldOptions": { - "calcs": [], - "values": false + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_current_epoch_state{service=\"storage_node\"}", + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Epoch State Timeline", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the uptime trend status of the Walrus Storage Node. Green indicates uptime is increasing, and red indicates it is not.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 1, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ + { + "options": { + "0": { + "text": "Down" + }, + "1": { + "text": "UP" + } }, - "graphMode": "line", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "type": "value" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "showPoints": "auto", - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + { + "color": "green", + "value": 1 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "rate(uptime{service=\"storage_node\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{process}}", + "range": true, + "refId": "A" + } + ], + "title": "Uptime Timeline", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 68, + "panels": [], + "title": "Balances", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the current SUI address used by the Walrus Storage Node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "editorMode": "code", - "expr": "walrus_current_epoch_state{service=\"storage_node\"}", - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - }], - "title": "Epoch State Timeline", - "type": "timeseries" + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] }, - { + "gridPos": { + "h": 2, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 70, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 1 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the uptime trend status of the Walrus Storage Node. Green indicates uptime is increasing, and red indicates it is not.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 1, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "symlog" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [{ - "options": { - "0": { - "text": "Down" - }, - "1": { - "text": "UP" - } - }, - "type": "value" - }], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 51, + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "walrus_sui_balance_mist{service=\"storage_node\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Address", + "transformations": [ + { + "id": "labelsToFields", "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "keepLabels": [ + "sui_address" + ], + "valueLabel": "sui_address" + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Shows the historical Sui token balance of the Walrus storage node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Tokens", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + { + "color": "green", + "value": 1 } - }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "exemplar": false, - "expr": "rate(uptime{service=\"storage_node\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{process}}", - "range": true, - "refId": "A" - }], - "title": "Uptime Timeline", - "type": "timeseries" + ] + } + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 68, - "panels": [], - "title": "Balances", - "type": "row" + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 15 }, - { + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the current SUI address used by the Walrus Storage Node.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 10, - "x": 0, - "y": 11 - }, - "id": 70, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 1 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_sui_balance_mist{service=\"storage_node\"} / 1e9", + "legendFormat": "SUI Balance", + "range": true, + "refId": "A" + } + ], + "title": "Sui Balance Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the current SUI token balance (in SUI) for the storage node in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 9, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "walrus_sui_balance_mist{service=\"storage_node\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - }], - "title": "Address", - "transformations": [{ - "id": "labelsToFields", - "options": { - "keepLabels": [ - "sui_address" - ], - "valueLabel": "sui_address" + { + "color": "green", + "value": 1 } - }], - "type": "stat" + ] + } + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 69, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the historical Sui token balance of the Walrus storage node.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Tokens", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "dark-red" - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 14, - "x": 10, - "y": 11 - }, - "id": 71, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(walrus_sui_balance_mist{service=\"storage_node\"}) / 1e9", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Sui Balance", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 14, + "panels": [], + "title": "Checkpoints", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Number of checkpoints the checkpoint downloader is behind the Sui full node: the gap between the latest checkpoint the download pipeline has fetched and the latest available on the full node. This measures how far behind the ingestion pipeline is; it does not include processing delay after download.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + { + "color": "yellow", + "value": 10 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_sui_balance_mist{service=\"storage_node\"} / 1e9", - "legendFormat": "SUI Balance", - "range": true, - "refId": "A" - }], - "title": "Sui Balance Over Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 54, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the current SUI token balance (in SUI) for the storage node in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 9, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(checkpoint_downloader_checkpoint_lag{service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Lag", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the number of active workers in the checkpoint downloader's worker pool for the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 10, - "x": 0, - "y": 13 + { + "color": "green", + "value": 1 + } + ] }, - "id": 69, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "max(walrus_sui_balance_mist{service=\"storage_node\"}) / 1e9", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Sui Balance", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 14, - "panels": [], - "title": "Checkpoints", - "type": "row" + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 22 }, - { + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "checkpoint_downloader_num_workers{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total number of checkpoints downloaded by the event processor in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "description": "Displays the current checkpoint lag between the local store and the full node in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 10 - } - ] - }, - "unit": "none" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 22 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_processor_total_downloaded_checkpoints{service=\"storage_node\"}", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Total Downloaded Checkpoints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the latest full checkpoint downloaded by the event processor in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 18 + { + "color": "green", + "value": 1 + } + ] }, - "id": 54, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "max(checkpoint_downloader_checkpoint_lag{service=\"storage_node\"})", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Checkpoint Lag", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 22 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the number of active workers in the checkpoint downloader's worker pool for the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_processor_latest_downloaded_checkpoint{service=\"storage_node\"}", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Latest Downloaded Checkpoint", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Checkpoint downloader lag over time (same metric as the Checkpoint Lag stat). Shows how many checkpoints the download pipeline is behind the full node. Spikes indicate the node is falling behind on fetching checkpoints (e.g. full node slow, network issues, or local overload).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lag (Checkpoints)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 18 + { + "color": "yellow", + "value": 10 + } + ] }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "checkpoint_downloader_num_workers{service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Active Workers", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total number of checkpoints downloaded by the event processor in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 18 + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(checkpoint_downloader_checkpoint_lag{service=\"storage_node\"})", + "legendFormat": "Latest Lag", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Lag Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Shows the timeline of the latest full checkpoint downloaded by the event processor in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Checkpoint", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_event_processor_total_downloaded_checkpoints{service=\"storage_node\"}", - "legendFormat": "", - "range": false, - "refId": "A" - }], - "title": "Total Downloaded Checkpoints", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the latest full checkpoint downloaded by the event processor in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 18 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_processor_latest_downloaded_checkpoint{service=\"storage_node\"}", + "legendFormat": "Latest Checkpoint", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Download Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 40, + "panels": [], + "title": "Shards", + "type": "row" + }, + { + "datasource": { + "uid": "ctibdrobbprrc6ulkutg" + }, + "description": "Displays a summary of owned and read-only shard statuses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "editorMode": "code", - "expr": "walrus_event_processor_latest_downloaded_checkpoint{service=\"storage_node\"}", - "legendFormat": "", - "range": false, - "refId": "A" - }], - "title": "Latest Downloaded Checkpoint", - "type": "stat" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the maximum checkpoint lag over time in the Walrus network, helping to identify delays between local store and full node.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Lag (Checkpoints)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 10 - } - ] - }, - "unit": "none" + "gridPos": { + "h": 5, + "w": 9, + "x": 0, + "y": 33 + }, + "id": 67, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "columns": [ + { + "selector": "data.shardSummary.owned", + "text": "Owned", + "type": "number" + }, + { + "selector": "data.shardSummary.readOnly", + "text": "Read Only", + "type": "number" + } + ], + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "columnar": false, + "root_is_not_array": true + }, + "refId": "A", + "root_selector": "$.*", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET" + } + } + ], + "title": "Summary", + "type": "stat" + }, + { + "datasource": { + "uid": "ctibdrobbprrc6ulkutg" + }, + "description": "Displays the statuses of owned shards.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 22 + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 15, + "x": 9, + "y": 33 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "columns": [ + { + "selector": "data.shardSummary.ownedShardStatus.unknown", + "text": "Unknown", + "type": "number" + }, + { + "selector": "data.shardSummary.ownedShardStatus.ready", + "text": "Ready", + "type": "number" + }, + { + "selector": "data.shardSummary.ownedShardStatus.inTransfer", + "text": "In Transfer", + "type": "number" + }, + { + "selector": "data.shardSummary.ownedShardStatus.inRecovery", + "text": "In Recovery", + "type": "number" + } + ], + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "columnar": false, + "root_is_not_array": true + }, + "refId": "A", + "root_selector": "$.*", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET" + } + } + ], + "title": "Shard Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total unencoded size of all uploaded blobs based on metadata.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 38 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "walrus_uploaded_metadata_unencoded_blob_bytes_sum{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Metadata Blob Size Processed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total number of storage confirmations issued.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 38 + }, + "id": 64, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "walrus_storage_confirmations_issued_total{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Storage Confirmations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total count of metadata operations (stored and retrieved) in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "editorMode": "code", - "expr": "max(checkpoint_downloader_checkpoint_lag{service=\"storage_node\"})", - "legendFormat": "Latest Lag", - "range": true, - "refId": "A" - }], - "title": "Checkpoint Lag Over Time", - "type": "timeseries" + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the timeline of the latest full checkpoint downloaded by the event processor in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Checkpoint", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "none" + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 38 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "walrus_metadata_stored_total + walrus_metadata_retrieved_total{service=\"storage_node\"}", + "legendFormat": "Total Metadata Operations", + "range": true, + "refId": "A" + } + ], + "title": "Total Metadata Operations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the total count of sliver operations (retrieved and stored) across all types (primary and secondary).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 22 + { + "color": "green", + "value": 1 + } + ] }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 38 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "walrus_slivers_retrieved_total{sliver_type=\"primary\", service=\"storage_node\"} + walrus_slivers_stored_total{sliver_type=\"primary\", service=\"storage_node\"}", + "range": true, + "refId": "A" + } + ], + "title": "Total Sliver Operations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the rate of sliver operations (retrieved and stored) over time, separated by type (primary or secondary).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate (slivers/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_event_processor_latest_downloaded_checkpoint{service=\"storage_node\"}", - "legendFormat": "Latest Checkpoint", - "range": true, - "refId": "A" - }], - "title": "Checkpoint Download Over Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(walrus_slivers_retrieved_total{sliver_type=\"primary\", service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Retrieved - Primary", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "rate(walrus_slivers_retrieved_total{sliver_type=\"secondary\", service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Retrieved - Secondary", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "rate(walrus_slivers_stored_total{sliver_type=\"primary\", service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Stored - Primary", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "rate(walrus_slivers_stored_total{sliver_type=\"secondary\", service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Stored - Secondary", + "range": true, + "refId": "D" + } + ], + "title": "Sliver Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the rate of metadata operations (stored and retrieved) over time, enabling a comparison of storage and retrieval rates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate (metadata/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 20, - "panels": [], - "title": "Events", - "type": "row" + "unit": "none" + }, + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(walrus_metadata_stored_total{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Metadata Stored Rate", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "rate(walrus_metadata_retrieved_total{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Metadata Retrieval Rate", + "range": true, + "refId": "B" + } + ], + "title": "Rates of Metadata Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the rate at which storage confirmations are issued over time in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate (confirmations/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] }, - "description": "Displays the number of Walrus events currently in the pending state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "orange", - "value": 1 - } - ] - }, - "unit": "none" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(walrus_storage_confirmations_issued_total{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "Confirmations", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Storage Confirmations Issued", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 20, + "panels": [], + "title": "Events", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the number of Walrus events currently in the pending state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 29 + { + "color": "orange", + "value": 1 + } + ] }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_event_cursor_progress{state=\"pending\", service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Pending Events", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the number of Walrus events currently in the persisted state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_cursor_progress{state=\"pending\", service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Pending Events", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the number of Walrus events currently in the persisted state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 29 + { + "color": "green", + "value": 1 + } + ] }, - "id": 57, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_event_cursor_progress{state=\"persisted\", service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Persisted Events", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 57, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the average processing time for each event type in the Walrus network, with color thresholds for performance monitoring.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_cursor_progress{state=\"persisted\", service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Persisted Events", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the progress of persisted events in the Walrus network over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Persisted Progress", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "sum by (event_type) (rate(walrus_event_process_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) / sum by (event_type) (rate(walrus_event_process_duration_seconds_count{service=\"storage_node\"}[$__rate_interval]))", - "legendFormat": "{{event_type}}", - "range": true, - "refId": "A" - }], - "title": "Average Event Processing Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the progress of persisted events in the Walrus network over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Persisted Progress", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_event_cursor_progress{state=\"persisted\", service=\"storage_node\"}", + "legendFormat": "Persisted Events", + "range": true, + "refId": "A" + } + ], + "title": "Persisted Event Cursor Progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the average processing time for each event type in the Walrus network, with color thresholds for performance monitoring.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_event_cursor_progress{state=\"persisted\", service=\"storage_node\"}", - "legendFormat": "Persisted Events", - "range": true, - "refId": "A" - }], - "title": "Persisted Event Cursor Progress", - "type": "timeseries" + "unit": "s" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 59, - "panels": [], - "title": "Span Latency", - "type": "row" + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 55 }, - { + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (event_type) (rate(walrus_event_process_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) / sum by (event_type) (rate(walrus_event_process_duration_seconds_count{service=\"storage_node\"}[$__rate_interval]))", + "legendFormat": "{{event_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Average Event Processing Time", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 59, + "panels": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "prometheus", + "uid": "be65m7mkyeh34c" }, "description": "Displays the latency distribution for spans in the storage node service, categorized by span names.", "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency(ms)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "ns" + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency(ms)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "overrides": [] + "unit": "ns" + }, + "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 + "h": 9, + "w": 24, + "x": 0, + "y": 62 }, "id": 60, "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [{ + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "expr": "histogram_quantile(0.95, sum(rate(tracing_span_latencies_bucket{service=\"storage_node\"}[$__rate_interval])) by (le, span_name))", - "legendFormat": "95th percentile - {{span_name}}", + "legendFormat": "{{span_name}}", "refId": "A" - }], - "title": "Latency Distribution", + } + ], + "title": "Latency Distribution (95th percentile)", "type": "timeseries" - }, - { + }, + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" + "type": "prometheus", + "uid": "be65m7mkyeh34c" }, "description": "Displays the average latency for spans in the storage node service.", "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency(ms)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "ns" + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency(ms)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "overrides": [] + "unit": "ns" + }, + "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 49 + "h": 9, + "w": 24, + "x": 0, + "y": 71 }, "id": 61, "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "timezone": [ - "browser" + "legend": { + "calcs": [ + "lastNotNull" ], - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [{ + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "expr": "tracing_span_latencies_sum{service=\"storage_node\"} / tracing_span_latencies_count{service=\"storage_node\"}", "legendFormat": "{{span_name}}", "refId": "B" - }], + } + ], "title": "Average Latency", "type": "timeseries" + } + ], + "title": "Span Latency", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 58 - }, - "id": 40, - "panels": [], - "title": "Shards", - "type": "row" + "id": 56, + "panels": [], + "title": "Recovery", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" }, - { - "datasource": { - "uid": "ctibdrobbprrc6ulkutg" - }, - "description": "Displays a summary of owned and read-only shard statuses.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 9, - "x": 0, - "y": 59 - }, - "id": 67, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "description": "Displays the current number of blob recoveries that are in-progress on the node in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "columns": [{ - "selector": "data.shardSummary.owned", - "text": "Owned", - "type": "number" - }, - { - "selector": "data.shardSummary.readOnly", - "text": "Read Only", - "type": "number" - } - ], - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true + { + "color": "yellow", + "value": 5 }, - "refId": "A", - "root_selector": "$.*", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "data": "", - "method": "GET" + { + "color": "red", + "value": 10 } - }], - "title": "Summary", - "type": "stat" + ] + }, + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 63 + }, + "id": 55, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "uid": "ctibdrobbprrc6ulkutg" - }, - "description": "Displays the statuses of owned shards.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 15, - "x": 9, - "y": 59 - }, - "id": 66, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(walrus_recover_blob_backlog{state=\"in-progress\", service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Blob Recovery In-Progress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the current number of blob recoveries queued by the node in the Walrus network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "columns": [{ - "selector": "data.shardSummary.ownedShardStatus.unknown", - "text": "Unknown", - "type": "number" - }, - { - "selector": "data.shardSummary.ownedShardStatus.ready", - "text": "Ready", - "type": "number" - }, - { - "selector": "data.shardSummary.ownedShardStatus.inTransfer", - "text": "In Transfer", - "type": "number" - }, - { - "selector": "data.shardSummary.ownedShardStatus.inRecovery", - "text": "In Recovery", - "type": "number" - } - ], - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true + { + "color": "yellow", + "value": 5 }, - "refId": "A", - "root_selector": "$.*", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "data": "", - "method": "GET" + { + "color": "red", + "value": 10 } - }], - "title": "Shard Status", - "type": "stat" + ] + }, + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 63 + }, + "id": 58, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total unencoded size of all uploaded blobs based on metadata.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "bytes" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "max(walrus_recover_blob_backlog{state=\"queued\", service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Blob Recovery Queued", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Current number of ongoing blob syncs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 64 + { + "color": "yellow", + "value": 1 + } + ] }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "walrus_uploaded_metadata_unencoded_blob_bytes_sum{service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Total Metadata Blob Size Processed", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total number of storage confirmations issued.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 64 - }, - "id": 64, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "walrus_storage_confirmations_issued_total{service=\"storage_node\"}", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Total Storage Confirmations", - "type": "stat" + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 63 }, - { + "id": 75, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total count of metadata operations (stored and retrieved) in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 64 + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_node_recovery_ongoing_blob_syncs{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Ongoing Blob Syncs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Recover blob progress.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "walrus_metadata_stored_total + walrus_metadata_retrieved_total{service=\"storage_node\"}", - "legendFormat": "Total Metadata Operations", - "range": true, - "refId": "A" - }], - "title": "Total Metadata Operations", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 63 + }, + "id": 76, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the total count of sliver operations (retrieved and stored) across all types (primary and secondary).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "red" - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 64 + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_node_recovery_recover_blob_progress{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Recover Blob Progress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Sync blob metadata progress.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "walrus_slivers_retrieved_total{sliver_type=\"primary\", service=\"storage_node\"} + walrus_slivers_stored_total{sliver_type=\"primary\", service=\"storage_node\"}", - "range": true, - "refId": "A" - }], - "title": "Total Sliver Operations", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "id": 77, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the rate of sliver operations (retrieved and stored) over time, separated by type (primary or secondary).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Rate (slivers/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "blue" - }] - }, - "unit": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_sync_blob_metadata_progress{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Sync Metadata Progress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the timeline of blob recovery backlog states in the Walrus network, highlighting transitions between 'in-progress' and 'queued' statuses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "Blobs", + "axisPlacement": "auto", + "axisSoftMax": 10, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 1, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 68 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "color": "yellow", + "value": 5 }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + { + "color": "red", + "value": 10 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(walrus_slivers_retrieved_total{sliver_type=\"primary\", service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Retrieved - Primary", - "range": true, - "refId": "A" - }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 5, + "options": { + "fieldOptions": { + "calcs": [], + "values": false + }, + "graphMode": "bars", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "showPoints": "auto", + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "walrus_recover_blob_backlog{state=~\"in-progress|queued\", service=\"storage_node\"}", + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Blob Recovery Backlog Timeline", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the time (in seconds) spent recovering blobs in the Walrus network, categorized by skip and success statuses.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "lines", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "editorMode": "code", - "expr": "rate(walrus_slivers_retrieved_total{sliver_type=\"secondary\", service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Retrieved - Secondary", - "range": true, - "refId": "B" + "color": "green" }, { - "editorMode": "code", - "expr": "rate(walrus_slivers_stored_total{sliver_type=\"primary\", service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Stored - Primary", - "range": true, - "refId": "C" + "color": "yellow", + "value": 2 }, { - "editorMode": "code", - "expr": "rate(walrus_slivers_stored_total{sliver_type=\"secondary\", service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Stored - Secondary", - "range": true, - "refId": "D" + "color": "red", + "value": 5 } - ], - "title": "Sliver Operations Rate", - "type": "timeseries" + ] + }, + "unit": "s" + }, + "overrides": [] }, - { + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 6, + "options": { + "fieldOptions": { + "calcs": [], + "values": false + }, + "graphMode": "line", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "showPoints": "auto", + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the rate of metadata operations (stored and retrieved) over time, enabling a comparison of storage and retrieval rates.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Rate (metadata/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(walrus_recover_blob_duration_seconds_bucket{service=\"storage_node\"}[$__rate_interval])) by (le, status))", + "legendFormat": "95th percentile - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "sum(rate(walrus_recover_blob_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) by (status) / sum(rate(walrus_recover_blob_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])) by (status)", + "legendFormat": "Average - {{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Blob Recovery Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the time spent recovering blob parts (metadata, primary, secondary) in the Walrus network, categorized by status (success, skip).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "lines", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 68 - }, - "id": 52, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "color": "yellow", + "value": 2 }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + { + "color": "red", + "value": 5 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(walrus_metadata_stored_total{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Metadata Stored Rate", - "range": true, - "refId": "A" - }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 7, + "options": { + "fieldOptions": { + "calcs": [], + "values": false + }, + "graphMode": "line", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "showPoints": "auto", + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(walrus_recover_blob_part_duration_seconds_bucket{service=\"storage_node\"}[$__rate_interval])) by (le, part, status))", + "legendFormat": "95th Percentile - {{part}} - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "editorMode": "code", + "expr": "sum(rate(walrus_recover_blob_part_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) by (part, status) / sum(rate(walrus_recover_blob_part_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])) by (part, status)", + "legendFormat": "Average - {{part}} - {{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Blob Part Recovery Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 78, + "panels": [], + "title": "Garbage collection", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Last epoch for which garbage collection was completed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "editorMode": "code", - "expr": "rate(walrus_metadata_retrieved_total{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Metadata Retrieval Rate", - "range": true, - "refId": "B" + "color": "green" } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 79, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Rates of Metadata Operations", - "type": "timeseries" + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true }, - { + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the rate at which storage confirmations are issued over time in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Rate (confirmations/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "blue" - }] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 68 - }, - "id": 49, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_garbage_collection_last_epoch{status=\"completed\", service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "GC Last Completed Epoch", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Total expired blob objects deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(walrus_storage_confirmations_issued_total{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Confirmations", - "range": true, - "refId": "A" - }], - "title": "Rate of Storage Confirmations Issued", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 76 - }, - "id": 56, - "panels": [], - "title": "Recovery", - "type": "row" + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 80 }, - { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the current number of blob recoveries that are in-progress on the node in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 77 - }, - "id": 55, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "max(walrus_recover_blob_backlog{state=\"in-progress\", service=\"storage_node\"})", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Blob Recovery In-Progress", - "type": "stat" + "id": 80, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true }, - { + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the current number of blob recoveries queued by the node in the Walrus network.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 77 + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_garbage_collection_expired_blob_objects_deleted_total{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "GC Expired Blobs Deleted", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Total blob data deletion attempts by status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] }, - "id": 58, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "max(walrus_recover_blob_backlog{state=\"queued\", service=\"storage_node\"})", - "legendFormat": "", - "range": true, - "refId": "A" - }], - "title": "Blob Recovery Queued", - "type": "stat" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 80 + }, + "id": 81, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the timeline of blob recovery backlog states in the Walrus network, highlighting transitions between 'in-progress' and 'queued' statuses.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "Blobs", - "axisPlacement": "auto", - "axisSoftMax": 10, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 1, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 77 - }, - "id": 5, - "options": { - "fieldOptions": { - "calcs": [], - "values": false - }, - "graphMode": "bars", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "showPoints": "auto", - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "sum(walrus_garbage_collection_blob_data_deletion_attempts_total{service=\"storage_node\"}) by (status)", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "GC Blob Data Deletion Attempts", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 84 + }, + "id": 82, + "panels": [], + "title": "Sui RPC & backlogs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Average Sui RPC call latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "walrus_recover_blob_backlog{state=~\"in-progress|queued\", service=\"storage_node\"}", - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - }], - "title": "Blob Recovery Backlog Timeline", - "type": "timeseries" + "unit": "s" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the time (in seconds) spent recovering blobs in the Walrus network, categorized by skip and success statuses.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "Seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "lines", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 82 - }, - "id": 6, - "options": { - "fieldOptions": { - "calcs": [], - "values": false - }, - "graphMode": "line", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "showPoints": "auto", - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "sum(rate(walrus_sui_rpc_call_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) / sum(rate(walrus_sui_rpc_call_duration_seconds_count{service=\"storage_node\"}[$__rate_interval]))", + "legendFormat": "Average", + "range": true, + "refId": "A" + } + ], + "title": "Sui RPC Call Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Cumulative retries for Sui RPC calls. get_full_checkpoint: retries when fetching full checkpoint data from the primary full node; high values indicate primary or network instability. get_full_checkpoint_with_failover: retries on the backup full node after failover; high values indicate both primary and failover are under stress.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(walrus_recover_blob_duration_seconds_bucket{service=\"storage_node\"}[$__rate_interval])) by (le, status))", - "legendFormat": "95th percentile - {{status}}", - "range": true, - "refId": "A" - }, + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "get_full_checkpoint" + }, + "properties": [ + { + "id": "displayName", + "value": "Primary" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "get_full_checkpoint_with_failover" + }, + "properties": [ { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "sum(rate(walrus_recover_blob_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) by (status) / sum(rate(walrus_recover_blob_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])) by (status)", - "legendFormat": "Average - {{status}}", - "range": true, - "refId": "B" + "id": "displayName", + "value": "Failover" } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 0, + "y": 90 + }, + "id": 84, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Blob Recovery Duration", - "type": "timeseries" + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "value_and_name", + "wideLayout": true }, - { + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the time spent recovering blob parts (metadata, primary, secondary) in the Walrus network, categorized by status (success, skip).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "Seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "lines", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 82 - }, - "id": 7, - "options": { - "fieldOptions": { - "calcs": [], - "values": false - }, - "graphMode": "line", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "showPoints": "auto", - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [{ - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(walrus_recover_blob_part_duration_seconds_bucket{service=\"storage_node\"}[$__rate_interval])) by (le, part, status))", - "legendFormat": "95th Percentile - {{part}} - {{status}}", - "range": true, - "refId": "A" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_sui_rpc_retry_count{service=\"storage_node\"}", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Sui RPC Retries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Pending metadata cache entries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, { - "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "editorMode": "code", - "expr": "sum(rate(walrus_recover_blob_part_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval])) by (part, status) / sum(rate(walrus_recover_blob_part_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])) by (part, status)", - "legendFormat": "Average - {{part}} - {{status}}", - "range": true, - "refId": "B" + "color": "yellow", + "value": 100 } - ], - "title": "Blob Part Recovery Duration", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 88 + ] }, - "id": 62, - "panels": [], - "title": "Blocks", - "type": "row" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 6, + "x": 9, + "y": 90 + }, + "id": 85, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the block cache hit rate for each column family in the Walrus Storage Node over time, helping identify trends and patterns in cache usage.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Hit Rate (hits/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "hits" - }, - "overrides": [{ - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "event_store" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [{ - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - }] - }] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 89 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "timezone": [ - "browser" - ], - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_pending_metadata_cache_entries{service=\"storage_node\"}", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Pending Metadata Cache", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Pending sliver cache: blobs, bytes, slivers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(block_cache_hit_count{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "{{cf_name}}", - "range": true, - "refId": "A" - }], - "title": "Block Cache Hit Rate Over Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 90 + }, + "id": 86, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the block read rate for each column family in the Walrus Storage Node, indicating the volume of data being read from the file system over time. Useful for analyzing potential bottlenecks caused by large blocks or queries.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Data Read (MB/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "green" - }] - }, - "unit": "Bps" - }, - "overrides": [{ - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "shard-314/primary-slivers" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [{ - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - }] - }] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 89 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_pending_sliver_cache_blobs{service=\"storage_node\"}", + "legendFormat": "Blobs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_pending_sliver_cache_bytes{service=\"storage_node\"}", + "legendFormat": "Bytes", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "walrus_pending_sliver_cache_slivers{service=\"storage_node\"}", + "legendFormat": "Slivers", + "range": true, + "refId": "C" + } + ], + "title": "Pending Sliver Cache", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 87, + "panels": [], + "title": "RocksDB", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Total RocksDB background errors across column families. Alert if > 0.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + { + "color": "red", + "value": 1 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(block_read_byte{service=\"storage_node\"}[$__rate_interval]) / (1024 * 1024)", - "legendFormat": "{{cf_name}}", - "range": true, - "refId": "A" - }], - "title": "Block Read Rate Over Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 96 + }, + "id": 88, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the total bytes read for values returned by Get operations in the Walrus Storage Node over time. This metric highlights read activity by column family.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Read Bytes (MB/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "blue" - }] - }, - "unit": "decbytes" - }, - "overrides": [{ - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "checkpoint_store" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [{ - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - }] - }] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 99 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "sum(rocksdb_background_errors{service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "RocksDB Background Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Number of column families with compaction pending (0/1 per CF).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(get_read_bytes{service=\"storage_node\"}[$__rate_interval]) / (1024 * 1024)", - "legendFormat": "{{cf_name}}", - "range": true, - "refId": "A" - }], - "title": "Get Read Bytes Over Time", - "type": "timeseries" + "unit": "none" + }, + "overrides": [] }, - { + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 96 + }, + "id": 89, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Shows the block read count for each column family in the Walrus Storage Node over time. This metric highlights instances of direct file system reads, indicating potential cache misses or scenarios where the block cache is disabled.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Read Count (reads/sec)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "blue" - }] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 99 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "sum(rocksdb_compaction_pending{service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "RocksDB Compaction Pending", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Block cache usage vs capacity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + { + "color": "yellow", + "value": 0.8 } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(block_read_count{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "{{cf_name}}", - "range": true, - "refId": "A" - }], - "title": "Block Read Count Over Time", - "type": "timeseries" + "unit": "percentunit" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 109 - }, - "id": 37, - "panels": [], - "title": "HTTP Requests", - "type": "row" + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 96 }, - { + "id": 90, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { "datasource": { - "type": "prometheus", - "uid": "be65m7mkyeh34c" - }, - "description": "Displays the duration of HTTP requests for different routes and their status codes in the Walrus Storage Node over time. The right column in the legend shows the HTTP method, route (Name), and status code (Status).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Duration (seconds)", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ - "color": "blue" - }] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 110 + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "expr": "sum(rocksdb_block_cache_usage{service=\"storage_node\"}) / sum(rocksdb_block_cache_capacity{service=\"storage_node\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "RocksDB Block Cache Usage", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 100 + }, + "id": 62, + "panels": [], + "title": "Blocks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Shows the block cache hit rate for each column family in the Walrus Storage Node over time, helping identify trends and patterns in cache usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hit Rate (hits/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "hits" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "event_store" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(block_cache_hit_count{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "{{cf_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Cache Hit Rate Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Shows the block read count for each column family in the Walrus Storage Node over time. This metric highlights instances of direct file system reads, indicating potential cache misses or scenarios where the block cache is disabled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Read Count (reads/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] }, - "id": 39, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "desc" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 110 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(block_read_count{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "{{cf_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Read Count Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Bytes read from storage per second by column family (MiB/s). Counter; rate derived from block_read_byte. Useful for analyzing read volume and bottlenecks. If the panel is empty, check time range or scrape target.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Data Read (MB/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "shard-314/primary-slivers" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(block_read_byte{service=\"storage_node\"}[$__rate_interval]) / (1024 * 1024)", + "legendFormat": "{{cf_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Read Rate Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Bytes read for values returned by Get operations per second by column family (MiB/s). Counter; rate derived from get_read_bytes. If Grafana shows 'metric might not be a counter', the metric is a counter; the warning can be ignored.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Read Bytes (MB/sec)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "checkpoint_store" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 129 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(get_read_bytes{service=\"storage_node\"}[$__rate_interval]) / (1024 * 1024)", + "legendFormat": "{{cf_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Get Read Bytes Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 138 + }, + "id": 37, + "panels": [], + "title": "HTTP Requests", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "be65m7mkyeh34c" + }, + "description": "Displays the duration of HTTP requests for different routes and their status codes in the Walrus Storage Node over time. The right column in the legend shows the HTTP method, route (Name), and status code (Status).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (seconds)", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] }, - "pluginVersion": "11.6.0", - "targets": [{ - "editorMode": "code", - "expr": "rate(http_request_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval]) / rate(http_request_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])", - "legendFormat": "Name: {{method}} {{route}} {{status_code}}", - "range": true, - "refId": "A" - }], - "title": "HTTP Request Duration Over Time", - "type": "timeseries" - } + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 139 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "rate(http_server_request_duration_seconds_sum{service=\"storage_node\"}[$__rate_interval]) / rate(http_server_request_duration_seconds_count{service=\"storage_node\"}[$__rate_interval])", + "legendFormat": "{{http_request_method}} {{http_route}} ({{http_response_status_code}})", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Request Duration Over Time", + "type": "timeseries" + } ], "preload": false, "schemaVersion": 41, "tags": [], "templating": { - "list": [] + "list": [] }, "time": { - "from": "now-30m", - "to": "now" + "from": "now-30m", + "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Walrus Storage Node", "uid": "de65msz8531moe", - "version": 1 -} \ No newline at end of file + "version": 9 + } \ No newline at end of file