diff --git a/readme.md b/readme.md index 3cab166d..737c94ae 100644 --- a/readme.md +++ b/readme.md @@ -84,12 +84,14 @@ start analyzing for anomalies! ### Alerts Skyline can alert you! In your settings.py, add any alerts you want to the ALERTS list, according to the schema `(metric keyword, strategy, expiration seconds)` where -`strategy` is one of `smtp`, `hipchat`, or `pagerduty`. You can also add your own +`strategy` is one of `smtp`, `hipchat`, `pagerduty` or `syslog`. You can also add your own alerting strategies. For every anomalous metric, Skyline will search for the given keyword and trigger the corresponding alert(s). To prevent alert fatigue, Skyline will only alert once every for any given metric/strategy combination. To enable Hipchat integration, uncomment the python-simple-hipchat -line in the requirements.txt file. +line in the requirements.txt file. If using syslog then the `EXPIRATION_TIME` +should be set to 1 for this to be effective in catching every anomaly, e.g. +`("stats", "syslog", 1)` ### How do you actually detect anomalies? An ensemble of algorithms vote. Majority rules. Batteries __kind of__ included. diff --git a/src/analyzer/alerters.py b/src/analyzer/alerters.py index 04f42096..9df079ca 100644 --- a/src/analyzer/alerters.py +++ b/src/analyzer/alerters.py @@ -63,6 +63,22 @@ def alert_hipchat(alert, metric): hipster.method('rooms/message', method='POST', parameters={'room_id': room, 'from': 'Skyline', 'color': settings.HIPCHAT_OPTS['color'], 'message': 'Anomaly: %s : %s' % (link, metric[1], metric[0])}) +def alert_syslog(alert, metric): + import sys + import syslog + syslog_ident = settings.SYSLOG_OPTS['ident'] + message = str("Anomalous metric: %s (value: %s)" % (metric[1], metric[0])) + if sys.version_info[:2] == (2, 6): + syslog.openlog(syslog_ident, syslog.LOG_PID, syslog.LOG_LOCAL4) + elif sys.version_info[:2] == (2, 7): + syslog.openlog(ident="skyline", logoption=syslog.LOG_PID, facility=syslog.LOG_LOCAL4) + elif sys.version_info[:1] == (3): + syslog.openlog(ident="skyline", logoption=syslog.LOG_PID, facility=syslog.LOG_LOCAL4) + else: + syslog.openlog(syslog_ident, syslog.LOG_PID, syslog.LOG_LOCAL4) + syslog.syslog(4, message) + + def trigger_alert(alert, metric): if '@' in alert[1]: diff --git a/src/settings.py.example b/src/settings.py.example index 5a65a1f0..9529726e 100644 --- a/src/settings.py.example +++ b/src/settings.py.example @@ -120,6 +120,7 @@ ENABLE_ALERTS = True # ("metric1", "smtp", EXPIRATION_TIME), # ("metric2", "pagerduty", EXPIRATION_TIME), # ("metric3", "hipchat", EXPIRATION_TIME), +# ("stats", "syslog", EXPIRATION_TIME), # ) ALERTS = ( ("skyline", "smtp", 1800), @@ -158,6 +159,14 @@ PAGERDUTY_OPTS = { "key": "your_pagerduty_service_api_key", } +# syslog alerts requires an ident +# Adds a LOG_WARNING message to the LOG_LOCAL4 which is local and will ship to +# any syslog or rsyslog down the line. The EXPIRATION_TIME for the syslog alert +# method should be set to 1 to fire every anomaly into the syslog +SYSLOG_OPTS = { + "ident": "skyline", +} + """ Horizon settings