From 6f25a30634fee48be929b5dd96252fe364cdedc9 Mon Sep 17 00:00:00 2001 From: Klaus Sperner Date: Fri, 5 Feb 2021 12:55:24 +0100 Subject: [PATCH 1/4] Add multigraph plugin for http response codes and times In contrast to using http_responsecode and http_loadtime with the same configuration, this plugin performs only one request per site and munin run to gather its statistics. --- plugins/http/http_response | 198 +++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 plugins/http/http_response diff --git a/plugins/http/http_response b/plugins/http/http_response new file mode 100644 index 000000000..63a35e209 --- /dev/null +++ b/plugins/http/http_response @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# vim: expandtab:ts=4:sw=4 + +: << =cut + +=head1 NAME + +http_response - Monitor HTTP response statistics + +=head1 CONFIGURATION + +The following environment variables are used + + sites - Sites to check + - separated by spaces + - can contain basic auth credentials + - defaults to "http://localhost/" + max_time - Timeout for each site check in seconds + - defaults to 5 seconds + short_label - Switch for shortening the label below the graph + - defaults to false + +=head2 CONFIGURATION EXAMPLE + + [http_response] + env.sites http://example.com/ https://user:secret@example2.de + env.max_time 20 + env.short_label true + +=head1 PREREQUISITES + +This plugin needs at least bash version 4 to run + +=head1 NOTES + +This plugin unifies the functionalities of the following plugins into one +multigraph plugin + + http_loadtime - https://gallery.munin-monitoring.org/plugins/munin/http_loadtime/ + http_responsecode - https://gallery.munin-monitoring.org/plugins/munin-contrib/http_responsecode/ + +In contrast to using these two plugins with the same configuration, this plugin +performs only one request per site and munin run to gather its statistics. + +=head1 AUTHOR + +Copyright (C) 2020 Klaus Sperner + +=head1 LICENSE + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; version 2 dated June, +1991. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 MAGIC MARKERS + + #%# family=manual + +=cut + +. "$MUNIN_LIBDIR/plugins/plugin.sh" + +readonly uri_regex='^(https?://)([^:]*):(.*)@(.*)$' + +strip_credentials_from_url() { + if [[ "$1" =~ $uri_regex ]]; then + echo "${BASH_REMATCH[1]}${BASH_REMATCH[4]}" + else + echo "$1" + fi +} + +extract_username_from_url() { + if [[ "$1" =~ $uri_regex ]]; then + echo "${BASH_REMATCH[2]}" + else + echo "" + fi +} + +extract_password_from_url() { + if [[ "$1" =~ $uri_regex ]]; then + echo "${BASH_REMATCH[3]}" + else + echo "" + fi +} + +compute_label() { + if [[ "${short_label,,}" == "true" || "${short_label,,}" == "yes" ]]; then + if [[ ${#1} -gt 33 ]]; then + echo "${1:0:30}..." + else + echo "$1" + fi + else + echo "$1" + fi +} + +if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then + >&2 echo "The plugin http_response needs at least bash version 4" + exit 1 +fi + +sites=${sites:-"http://localhost/"} +max_time=${max_time:-5} +short_label=${short_label:-"false"} + +if [[ "$1" == "config" ]]; then + echo 'multigraph http_response_code' + echo 'graph_args --base 1000 -l 0 -u 511' + echo 'graph_title HTTP Response Codes' + echo 'graph_vlabel Repsonse Code' + echo 'graph_category network' + echo 'graph_info This graph shows HTTP response code statistics' + echo 'graph_printf %3.0lf' + for site in $sites; do + site_without_credentials=$( strip_credentials_from_url "$site" ) + siteid="$( clean_fieldname "$site_without_credentials" )" + echo "$siteid.label $( compute_label "$site_without_credentials" )" + echo "$siteid.info HTTP response code statistics for $site_without_credentials" + echo "$siteid.critical 99:399"; + done + echo 'multigraph http_response_time' + echo 'graph_args --base 1000 -l 0' + echo 'graph_title HTTP Response Times' + echo 'graph_vlabel Response Time in seconds' + echo 'graph_category network' + echo 'graph_info This graph shows HTTP response time statistics' + for site in $sites; do + site_without_credentials=$( strip_credentials_from_url "$site" ) + siteid="$( clean_fieldname "$site_without_credentials" )" + echo "$siteid.label $( compute_label "$site_without_credentials" )" + echo "$siteid.info HTTP response time statistics for $site_without_credentials" + done + exit 0 +fi + +declare -A response_codes +declare -A response_times + +for site in $sites; do + site_without_credentials=$( strip_credentials_from_url "$site" ) + username=$( extract_username_from_url "$site" ) + password=$( extract_password_from_url "$site" ) + + curl_config_file="" + curl_auth_opt=() + if [ -n "$username" ]; then + if [ -z "$password" ]; then + >&2 echo "Invalid configuration: username specified without password" + exit 1 + fi + curl_config_file=$(mktemp) || exit 1 + trap 'rm -f "$curl_config_file"' EXIT + echo "user=${username}:${password}" >> "$curl_config_file" + curl_auth_opt=(--config "$curl_config_file") + fi + + siteid="$( clean_fieldname "$site_without_credentials" )" + statuscode= + loadtime= + start=$(date +%s.%N) + statuscode=$( curl "${curl_auth_opt[@]}" --write-out '%{http_code}' --max-time "$max_time" --silent --output /dev/null "$site_without_credentials" ) + returncode=$? + loadtime=$( echo "$start" "$(date +%s.%N)" | awk '{ print($2 - $1); }' ) + if [[ $returncode -ne 0 ]]; then + loadtime=0 + fi + response_codes+=(["$siteid"]="$statuscode") + response_times+=(["$siteid"]="$loadtime") + + if [ -n "$curl_config_file" ]; then + rm -f "$curl_config_file" + fi +done + +echo 'multigraph http_response_code' +for siteid in "${!response_codes[@]}"; do + echo "${siteid}.value ${response_codes[${siteid}]}" +done + +echo 'multigraph http_response_time' +for siteid in "${!response_times[@]}"; do + echo "${siteid}.value ${response_times[${siteid}]}" +done + From 3fba496092d1b5553e16deee1ab060857cebb7e7 Mon Sep 17 00:00:00 2001 From: Klaus Sperner Date: Fri, 5 Feb 2021 13:23:03 +0100 Subject: [PATCH 2/4] make plugin http_response executable --- plugins/http/http_response | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 plugins/http/http_response diff --git a/plugins/http/http_response b/plugins/http/http_response old mode 100644 new mode 100755 From 00e9b169b89c9c1bb66043891a3e81bd334fa636 Mon Sep 17 00:00:00 2001 From: Klaus Sperner Date: Fri, 17 Sep 2021 15:57:41 +0200 Subject: [PATCH 3/4] Various improvements to the http_response plugin - ignore include during shellcheck - check if needed programs are installed - use long argument names for graph_args - remove un-needed empty definitions - improve variable names - validate parameter max_time - update copyright statement - write all configurable arguments into one array - use same check for short_label and follow_redirect - reformat documentation --- plugins/http/http_response | 190 ++++++++++++++++++++++++++++--------- 1 file changed, 144 insertions(+), 46 deletions(-) diff --git a/plugins/http/http_response b/plugins/http/http_response index ac68799ee..df901b52b 100755 --- a/plugins/http/http_response +++ b/plugins/http/http_response @@ -9,18 +9,67 @@ http_response - Monitor HTTP response statistics =head1 CONFIGURATION -The following environment variables are used - - sites - Sites to check - - separated by spaces - - can contain basic auth credentials - - defaults to "http://localhost/" - max_time - Timeout for each site check in seconds - - defaults to 5 seconds - short_label - Switch for shortening the label below the graph - - defaults to false - follow_redirect - Follow http redirects - - defaults to false +The following environment variables are used: + +=over + +=item * + +sites - The sites to check + +=over + +=item * + +separated by spaces + +=item * + +can contain basic auth credentials + +=item * + +defaults to C + +=back + +=item * + +max_time - Timeout for each site check in seconds + +=over + +=item * + +defaults to C<5> + +=back + +=item * + +short_label - Switch for shortening the label below the graph + +=over + +=item * + +defaults to C + +=back + +=item * + +follow_redirect - Switch for following http redirects + +=over + +=item * + +defaults to C + +=back + +=back =head2 CONFIGURATION EXAMPLE @@ -32,22 +81,57 @@ The following environment variables are used =head1 PREREQUISITES -This plugin needs at least bash version 4 to run +This plugin needs at least bash version 4 to run. + +Additionally the following programs must be present on your system: + +=over + +=item * + +awk + +=item * + +curl + +=item * + +date + +=item * + +echo + +=item * + +mktemp + +=back -=head1 NOTES +=head1 SEE ALSO This plugin unifies the functionalities of the following plugins into one -multigraph plugin +multigraph plugin: - http_loadtime - https://gallery.munin-monitoring.org/plugins/munin/http_loadtime/ - http_responsecode - https://gallery.munin-monitoring.org/plugins/munin-contrib/http_responsecode/ +=over + +=item * + +L + +=item * + +L + +=back In contrast to using these two plugins with the same configuration, this plugin performs only one request per site and munin run to gather its statistics. =head1 AUTHOR -Copyright (C) 2020 Klaus Sperner +Copyright (C) 2020-2021 Klaus Sperner =head1 LICENSE @@ -71,6 +155,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. =cut +# shellcheck disable=SC1091 . "$MUNIN_LIBDIR/plugins/plugin.sh" readonly uri_regex='^(https?://)([^:]*):(.*)@(.*)$' @@ -111,19 +196,35 @@ compute_label() { fi } +check_programs_installed() { + for program in "$@"; do + if ! hash "$program" 2>/dev/null; then + >&2 echo "The plugin http_response needs $program but it is not installed. Aborting." + exit 1 + fi + done +} + if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then >&2 echo "The plugin http_response needs at least bash version 4" exit 1 fi +check_programs_installed awk curl date echo mktemp + sites=${sites:-"http://localhost/"} max_time=${max_time:-5} short_label=${short_label:-"false"} follow_redirect=${follow_redirect:-"false"} +if [[ ! "$max_time" =~ ^[0-9]+$ ]]; then + >&2 echo "Invalid configuration: max_time $max_time must contain only digits" + exit 1 +fi + if [[ "$1" == "config" ]]; then echo 'multigraph http_response_code' - echo 'graph_args --base 1000 -l 0 -u 511' + echo 'graph_args --base 1000 --lower-limit 0 --upper-limit 511' echo 'graph_title HTTP Response Codes' echo 'graph_vlabel Response Code' echo 'graph_category network' @@ -131,10 +232,10 @@ if [[ "$1" == "config" ]]; then echo 'graph_printf %3.0lf' for site in $sites; do site_without_credentials=$( strip_credentials_from_url "$site" ) - siteid="$( clean_fieldname "$site_without_credentials" )" - echo "$siteid.label $( compute_label "$site_without_credentials" )" - echo "$siteid.info HTTP response code statistics for $site_without_credentials" - echo "$siteid.critical 99:399"; + site_id="$( clean_fieldname "$site_without_credentials" )" + echo "$site_id.label $( compute_label "$site_without_credentials" )" + echo "$site_id.info HTTP response code statistics for $site_without_credentials" + echo "$site_id.critical 99:399"; done echo 'multigraph http_response_time' echo 'graph_args --base 1000 -l 0' @@ -144,9 +245,9 @@ if [[ "$1" == "config" ]]; then echo 'graph_info This graph shows HTTP response time statistics' for site in $sites; do site_without_credentials=$( strip_credentials_from_url "$site" ) - siteid="$( clean_fieldname "$site_without_credentials" )" - echo "$siteid.label $( compute_label "$site_without_credentials" )" - echo "$siteid.info HTTP response time statistics for $site_without_credentials" + site_id="$( clean_fieldname "$site_without_credentials" )" + echo "$site_id.label $( compute_label "$site_without_credentials" )" + echo "$site_id.info HTTP response time statistics for $site_without_credentials" done exit 0 fi @@ -159,8 +260,8 @@ for site in $sites; do username=$( extract_username_from_url "$site" ) password=$( extract_password_from_url "$site" ) + configurable_arguments=() curl_config_file="" - curl_auth_opt=() if [ -n "$username" ]; then if [ -z "$password" ]; then >&2 echo "Invalid configuration: username specified without password" @@ -169,27 +270,24 @@ for site in $sites; do curl_config_file=$(mktemp) || exit 1 trap 'rm -f "$curl_config_file"' EXIT echo "user=${username}:${password}" >> "$curl_config_file" - curl_auth_opt=(--config "$curl_config_file") + configurable_arguments+=(--config "$curl_config_file") fi - curl_arg="" - if $follow_redirect; then - curl_arg="--location" + if [[ "${follow_redirect,,}" == "true" || "${follow_redirect,,}" == "yes" ]]; then + configurable_arguments+=(--location) fi - siteid="$( clean_fieldname "$site_without_credentials" )" - statuscode= - loadtime= + site_id="$( clean_fieldname "$site_without_credentials" )" start=$(date +%s.%N) - statuscode=$( curl "${curl_auth_opt[@]}" --write-out '%{http_code}' --max-time "$max_time" $curl_arg --silent --output /dev/null "$site_without_credentials" ) - returncode=$? - loadtime=$( echo "$start" "$(date +%s.%N)" | awk '{ print($2 - $1); }' ) - if [[ $returncode -ne 0 ]]; then - loadtime="U" - statuscode="U" + status_code=$( curl "${configurable_arguments[@]}" --write-out '%{http_code}' --max-time "$max_time" --silent --output /dev/null "$site_without_credentials" ) + return_code=$? + load_time=$( echo "$start" "$(date +%s.%N)" | awk '{ print($2 - $1); }' ) + if [[ $return_code -ne 0 ]]; then + load_time="U" + status_code="U" fi - response_codes+=(["$siteid"]="$statuscode") - response_times+=(["$siteid"]="$loadtime") + response_codes+=(["$site_id"]="$status_code") + response_times+=(["$site_id"]="$load_time") if [ -n "$curl_config_file" ]; then rm -f "$curl_config_file" @@ -197,12 +295,12 @@ for site in $sites; do done echo 'multigraph http_response_code' -for siteid in "${!response_codes[@]}"; do - echo "${siteid}.value ${response_codes[${siteid}]}" +for site_id in "${!response_codes[@]}"; do + echo "${site_id}.value ${response_codes[${site_id}]}" done echo 'multigraph http_response_time' -for siteid in "${!response_times[@]}"; do - echo "${siteid}.value ${response_times[${siteid}]}" +for site_id in "${!response_times[@]}"; do + echo "${site_id}.value ${response_times[${site_id}]}" done From 16b6a9c9df32f2dbdd59ee04cb16ea83bdf08d9d Mon Sep 17 00:00:00 2001 From: Klaus Sperner Date: Fri, 17 Sep 2021 17:23:29 +0200 Subject: [PATCH 4/4] unify error messages --- plugins/http/http_response | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/http/http_response b/plugins/http/http_response index df901b52b..5745f9a03 100755 --- a/plugins/http/http_response +++ b/plugins/http/http_response @@ -206,7 +206,7 @@ check_programs_installed() { } if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then - >&2 echo "The plugin http_response needs at least bash version 4" + >&2 echo "The plugin http_response needs at least bash version 4. Aborting." exit 1 fi @@ -218,7 +218,7 @@ short_label=${short_label:-"false"} follow_redirect=${follow_redirect:-"false"} if [[ ! "$max_time" =~ ^[0-9]+$ ]]; then - >&2 echo "Invalid configuration: max_time $max_time must contain only digits" + >&2 echo "Invalid configuration: max_time $max_time must contain only digits. Aborting." exit 1 fi