diff --git a/README.md b/README.md index 7928814..9fb7ad1 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ grafana_host="http://localhost:3000" grafana_cred="admin:admin" # Keep grafana_folder empty for adding the dashboards in "General" folder grafana_folder="AWS CloudWatch" -ds=(1516 677 139 674 590 659 758 623 617 551 653 969 650 644 607 593 707 575 1519 581 584 2969 8050 11099 11154 11155 12979 13018 13040 13104 13892 14189 14391 14392 14954 14955 15016 16899 17741 17978 20007 20008 20009); +ds=(1516 677 139 674 590 659 758 623 617 551 653 969 650 644 607 593 707 575 1519 581 584 2969 8050 11099 11154 11155 12979 13018 13040 13104 13892 14189 14391 14392 14954 14955 15016 16899 17741 17978 20007 20008 20009 20144); folderId=$(curl -s -k -u "$grafana_cred" $grafana_host/api/folders | jq -r --arg grafana_folder "$grafana_folder" '.[] | select(.title==$grafana_folder).id') if [ -z "$folderId" ] ; then echo "Didn't get folderId" ; else echo "Got folderId $folderId" ; fi for d in "${ds[@]}"; do @@ -167,6 +167,9 @@ Single click provisioning [![Gitpod ready-to-test](https://img.shields.io/badge/ ### [AWS Kinesis](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-kinesis) [![AWS Kinesis](aws-kinesis/aws-kinesis.png)](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-kinesis) +### [AWS Kinesis Firehose](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-kinesis-firehose) +[![AWS Kinesis Firehose](aws-kinesis-firehose/aws-kinesis-firehose.png)](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-kinesis-firehose) + ### [AWS Lambda](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-lambda) [![AWS Lambda](aws-lambda/aws-lambda.png)](https://github.com/monitoringartist/grafana-aws-cloudwatch-dashboards/tree/master/aws-lambda) diff --git a/aws-kinesis-firehose/aws-kinesis-firehose.json b/aws-kinesis-firehose/aws-kinesis-firehose.json new file mode 100644 index 0000000..900f52e --- /dev/null +++ b/aws-kinesis-firehose/aws-kinesis-firehose.json @@ -0,0 +1,1104 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "cloudwatch", + "name": "CloudWatch", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.1.5" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Visualize AWS Kinesis Data Firehose metrics", + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "Data Ingestion Through Direct PUT", + "type": "row" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#e24d42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IncomingBytes Sum" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "IncomingRecords", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "A", + "region": "$region", + "returnData": false, + "statistic": "Sum" + }, + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "IncomingBytes", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "B", + "region": "$region", + "returnData": false, + "statistic": "Sum" + } + ], + "title": "IncomingRecords/IncomingBytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ThrottledRecords Average" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "RecordsPerSecondLimit", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "A", + "region": "$region", + "returnData": false, + "statistic": "Average" + }, + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "ThrottledRecords", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "B", + "region": "$region", + "returnData": false, + "statistic": "Average" + } + ], + "title": "RecordsPerSecondLimit/ThrottledRecords", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 6, + "panels": [], + "title": "Delivery to Splunk", + "type": "row" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DeliveryToSplunk.Bytes Sum" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "DeliveryToSplunk.Records", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "A", + "region": "$region", + "returnData": false, + "statistic": "Average" + }, + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "DeliveryToSplunk.Bytes", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "B", + "region": "$region", + "returnData": false, + "statistic": "Sum" + } + ], + "title": "DeliveryToSplunk.Records/DeliveryToSplunk.Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DeliveryToSplunk.DataFreshness Average" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "DeliveryToSplunk.Success", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "A", + "region": "$region", + "returnData": false, + "statistic": "SampleCount" + }, + { + "alias": "{{metric}} {{stat}}", + "application": { + "filter": "" + }, + "datasource": { + "uid": "$datasource" + }, + "dimensions": { + "DeliveryStreamName": "$deliverystreamname" + }, + "expression": "", + "functions": [], + "group": { + "filter": "" + }, + "highResolution": false, + "host": { + "filter": "" + }, + "id": "", + "item": { + "filter": "" + }, + "label": "${PROP('MetricName')} ${PROP('Stat')}", + "matchExact": true, + "metricEditorMode": 0, + "metricName": "DeliveryToSplunk.DataFreshness", + "metricQueryType": 0, + "mode": 0, + "namespace": "AWS/Firehose", + "options": { + "showDisabledItems": false + }, + "period": "$agg", + "refId": "B", + "region": "$region", + "returnData": false, + "statistic": "Average" + } + ], + "title": "DeliveryToSplunk.Success/DeliveryToSplunk.DataFreshness", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "panels": [], + "type": "row" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 2, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": " | \nAWS CloudWatch Kinesis documentation | \nInstalled from Grafana.com dashboards", + "mode": "html" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Documentation", + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "cloudwatch", + "queryValue": "trac", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": {}, + "hide": 0, + "label": "Aggregation", + "name": "agg", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_agg" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1s,5s,10s,30s,1m,5m,15m,1h,6h,1d,7d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": {}, + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "definition": "regions()", + "hide": 0, + "includeAll": false, + "label": "Region", + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "cloudwatch", + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "DeliveryStreamName", + "multi": false, + "name": "deliverystreamname", + "options": [], + "query": { + "attributeName": "", + "dimensionFilters": {}, + "dimensionKey": "DeliveryStreamName", + "ec2Filters": {}, + "instanceID": "", + "metricName": "IncomingBytes", + "namespace": "AWS/Firehose", + "queryType": "dimensionValues", + "refId": "CloudWatchVariableQueryEditor-VariableQuery", + "region": "$region", + "resourceType": "", + "tags": {} + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "AWS Kinesis Firehose", + "uid": "AWSFirehose", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/dashboards.libsonnet b/dashboards.libsonnet index 167e043..0aaa4ec 100644 --- a/dashboards.libsonnet +++ b/dashboards.libsonnet @@ -25,6 +25,7 @@ 'aws-fsx.json': (import 'aws-fsx/aws-fsx.json'), 'aws-inspector.json': (import 'aws-inspector/aws-inspector.json'), 'aws-kinesis.json': (import 'aws-kinesis/aws-kinesis.json'), + 'aws-kinesis-firehose.json': (import 'aws-kinesis-firehose/aws-kinesis-firehose.json'), 'aws-lambda.json': (import 'aws-lambda/aws-lambda.json'), 'aws-logs.json': (import 'aws-logs/aws-logs.json'), 'aws-network-firewall.json': (import 'aws-network-firewall/aws-network-firewall.json'),