diff --git a/scrapylib/eventnotify.py b/scrapylib/eventnotify.py new file mode 100644 index 0000000..6fc8e52 --- /dev/null +++ b/scrapylib/eventnotify.py @@ -0,0 +1,33 @@ +import boto +from scrapy.exceptions import NotConfigured + + +# signal +event_signal = object() + + +class EventNotify(object): + + def __init__(self, settings): + self._ses_credentials = (settings.get('EVENTNOTIFY_SESID'), + settings.get('EVENTNOTIFY_SESKEY')) + self._from = settings.get('EVENTNOTIFY_FROM', '') + self._to = settings.getlist('EVENTNOTIFY_TO', []) + self._subject = settings.get('EVENTNOTIFY_SUBJECT', '') + self._body = settings.get('EVENTNOTIFY_BODY', '') + if not (all(self._ses_credentials) and self._to and self._from): + raise NotConfigured + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + if not settings.getbool('EVENTNOTIFY_ENABLED'): + raise NotConfigured + o = cls(settings) + crawler.signals.connect(o.notify, signal=event_signal) + return o + + def notify(self, **kwargs): + ses = boto.connect_ses(*self._ses_credentials) + ses.send_email(self._from, self._subject.format(**kwargs), + self._body.format(**kwargs), self._to) diff --git a/tests/test_eventnotify.py b/tests/test_eventnotify.py new file mode 100644 index 0000000..7b23c05 --- /dev/null +++ b/tests/test_eventnotify.py @@ -0,0 +1,62 @@ +import unittest + +from scrapy.utils.test import get_crawler +from scrapy.exceptions import NotConfigured + +from scrapylib.eventnotify import EventNotify, event_signal + + +class EventNotifyTestCase(unittest.TestCase): + + ext_cls = EventNotify + + def _mock_crawler(self, settings=None): + class MockedDownloader(object): + slots = {} + + class MockedEngine(object): + downloader = MockedDownloader() + fake_spider_closed_result = None + + def close_spider(self, spider, reason): + self.fake_spider_closed_result = (spider, reason) + + crawler = get_crawler(settings) + crawler.engine = MockedEngine() + return crawler + + def test_enabled(self): + settings = {'EVENTNOTIFY_ENABLED': True} + crawler = self._mock_crawler(settings) + self.assertRaises(NotConfigured, self.ext_cls.from_crawler, crawler) + settings['EVENTNOTIFY_SESID'] = 'id_123' + settings['EVENTNOTIFY_SESKEY'] = 'key_123' + crawler = self._mock_crawler(settings) + self.assertRaises(NotConfigured, self.ext_cls.from_crawler, crawler) + settings['EVENTNOTIFY_FROM'] = 'info@scrapinghub.com' + settings['EVENTNOTIFY_TO'] = 'someone@domain.com' + crawler = self._mock_crawler(settings) + self.ext_cls.from_crawler(crawler) + + def test_event_signal(self): + settings = { + 'EVENTNOTIFY_TO': 'someone@domain.com', + 'EVENTNOTIFY_FROM': 'info@scrapinghub.com', + 'EVENTNOTIFY_SESID': 'id_123', + 'EVENTNOTIFY_SESKEY': 'key_123', + 'EVENTNOTIFY_ENABLED': True, + } + + class MockedEventNotify(self.ext_cls): + + def __init__(self, *args, **kwargs): + super(MockedEventNotify, self).__init__(*args, **kwargs) + self.last_event = None + + def notify(self, **kwargs): + self.last_event = kwargs + + crawler = self._mock_crawler(settings) + ext = MockedEventNotify.from_crawler(crawler) + crawler.signals.send_catch_log(signal=event_signal, message='scrapy') + self.assertEqual('scrapy', ext.last_event.get('message'))