diff --git a/intelmq/bots/collectors/github_api/REQUIREMENTS.txt b/intelmq/bots/collectors/github_api/REQUIREMENTS.txt deleted file mode 100644 index 03fa26c75..000000000 --- a/intelmq/bots/collectors/github_api/REQUIREMENTS.txt +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Tomas Bellus -# SPDX-License-Identifier: AGPL-3.0-or-later - -requests diff --git a/intelmq/bots/collectors/github_api/_collector_github_api.py b/intelmq/bots/collectors/github_api/_collector_github_api.py index 545670d79..8c8e72866 100644 --- a/intelmq/bots/collectors/github_api/_collector_github_api.py +++ b/intelmq/bots/collectors/github_api/_collector_github_api.py @@ -7,13 +7,10 @@ GITHUB API Collector bot """ import base64 +from requests import exceptions from intelmq.lib.bot import CollectorBot - -try: - import requests -except ImportError: - requests = None +from intelmq.lib.mixins import HttpMixin static_params = { 'headers': { @@ -22,14 +19,11 @@ } -class GithubAPICollectorBot(CollectorBot): +class GithubAPICollectorBot(CollectorBot, HttpMixin): basic_auth_username = None basic_auth_password = None def init(self): - if requests is None: - raise ValueError('Could not import requests. Please install it.') - self.__user_headers = static_params['headers'] if self.basic_auth_username is not None and self.basic_auth_password is not None: self.__user_headers.update(self.__produce_auth_header(self.basic_auth_username, self.basic_auth_password)) @@ -47,13 +41,13 @@ def process_request(self): def github_api(self, api_path: str, **kwargs) -> dict: try: - response = requests.get(f"{api_path}", params=kwargs, headers=self.__user_headers) + response = self.http_get(api_path, headers=self.__user_headers, params=kwargs) if response.status_code == 401: # bad credentials raise ValueError(response.json()['message']) else: return response.json() - except requests.RequestException: + except exceptions.RequestException: raise ValueError(f"Unknown repository {api_path!r}.") @staticmethod diff --git a/intelmq/bots/collectors/github_api/collector_github_contents_api.py b/intelmq/bots/collectors/github_api/collector_github_contents_api.py index f98efd34b..d9faa8933 100644 --- a/intelmq/bots/collectors/github_api/collector_github_contents_api.py +++ b/intelmq/bots/collectors/github_api/collector_github_contents_api.py @@ -14,17 +14,14 @@ 'regex': file regex (DEFAULT = '*.json') """ import re +from requests import exceptions from intelmq.lib.exceptions import InvalidArgument from intelmq.bots.collectors.github_api._collector_github_api import GithubAPICollectorBot +from intelmq.lib.mixins import HttpMixin -try: - import requests -except ImportError: - requests = None - -class GithubContentsAPICollectorBot(GithubAPICollectorBot): +class GithubContentsAPICollectorBot(GithubAPICollectorBot, HttpMixin): "Collect files from a GitHub repository via the API. Optionally with GitHub credentials." regex: str = None # TODO: could be re repository: str = None @@ -62,7 +59,7 @@ def process_request(self): if item['extra'] != {}: report.add('extra.file_metadata', item['extra']) self.send_message(report) - except requests.RequestException as e: + except exceptions.RequestException as e: raise ConnectionError(e) def __recurse_repository_files(self, base_api_url: str, extracted_github_files: list = None) -> list: @@ -75,7 +72,7 @@ def __recurse_repository_files(self, base_api_url: str, extracted_github_files: elif github_file['type'] == 'file' and bool(re.search(self.regex, github_file['name'])): extracted_github_file_data = { 'download_url': github_file['download_url'], - 'content': requests.get(github_file['download_url']).content, + 'content': self.http_get(github_file['download_url']).content, 'extra': {} } for field_name in self.__extra_fields: diff --git a/intelmq/bots/collectors/mail/collector_mail_url.py b/intelmq/bots/collectors/mail/collector_mail_url.py index 9d13ea5a6..6dfe7f4cc 100644 --- a/intelmq/bots/collectors/mail/collector_mail_url.py +++ b/intelmq/bots/collectors/mail/collector_mail_url.py @@ -8,6 +8,7 @@ """ import io import re +from requests import exceptions from intelmq.lib.mixins import HttpMixin from intelmq.lib.splitreports import generate_reports @@ -50,7 +51,7 @@ def process_message(self, uid, message): self.logger.info("Downloading report from %r.", url) try: resp = self.http_get(url) - except requests.exceptions.Timeout: + except exceptions.Timeout: self.logger.error("Request timed out %i times in a row." % self.http_timeout_max_tries) erroneous = True diff --git a/intelmq/bots/collectors/microsoft/collector_azure.py b/intelmq/bots/collectors/microsoft/collector_azure.py index c33fc7d9b..282577a19 100644 --- a/intelmq/bots/collectors/microsoft/collector_azure.py +++ b/intelmq/bots/collectors/microsoft/collector_azure.py @@ -11,7 +11,7 @@ from intelmq.lib.bot import CollectorBot from intelmq.lib.exceptions import MissingDependencyError -from intelmq.lib.mixins import CacheMixin +from intelmq.lib.mixins import CacheMixin, HttpMixin try: from azure.storage.blob import ContainerClient @@ -23,7 +23,7 @@ create_configuration = None # noqa -class MicrosoftAzureCollectorBot(CollectorBot, CacheMixin): +class MicrosoftAzureCollectorBot(CollectorBot, CacheMixin, HttpMixin): "Fetch data blobs from a Microsoft Azure container" connection_string: str = "" container_name: str = "" diff --git a/intelmq/bots/collectors/shodan/collector_stream.py b/intelmq/bots/collectors/shodan/collector_stream.py index aeed4ff26..d09b95260 100644 --- a/intelmq/bots/collectors/shodan/collector_stream.py +++ b/intelmq/bots/collectors/shodan/collector_stream.py @@ -19,6 +19,7 @@ from typing import List from intelmq.lib.bot import CollectorBot +from intelmq.lib.mixins import HttpMixin try: import shodan @@ -27,7 +28,7 @@ shodan = None -class ShodanStreamCollectorBot(CollectorBot): +class ShodanStreamCollectorBot(CollectorBot, HttpMixin): "Collect the Shodan stream from the Shodan API" api_key: str = "" countries: List[str] = [] @@ -36,7 +37,7 @@ def init(self): if shodan is None: raise ValueError("Library 'shodan' is needed but not installed.") - self.set_request_parameters() + self.setup() if tuple(int(v) for v in pkg_resources.get_distribution("shodan").version.split('.')) <= (1, 8, 1): if self.proxy: raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.') diff --git a/intelmq/bots/experts/do_portal/expert.py b/intelmq/bots/experts/do_portal/expert.py index fecf4224a..2101c83d6 100644 --- a/intelmq/bots/experts/do_portal/expert.py +++ b/intelmq/bots/experts/do_portal/expert.py @@ -8,27 +8,18 @@ a "502 Bad Gateway" status code is treated the same as a timeout, i.e. will be retried instead of a fail. """ -try: - import requests -except ImportError: - requests = None - +from intelmq.lib.mixins import HttpMixin import intelmq.lib.utils as utils from intelmq.lib.bot import ExpertBot -class DoPortalExpertBot(ExpertBot): +class DoPortalExpertBot(ExpertBot, HttpMixin): """Retrieve abuse contact information for the source IP address from a do-portal instance""" mode: str = "append" portal_api_key: str = None portal_url: str = None def init(self): - if requests is None: - raise ValueError("Library 'requests' could not be loaded. Please install it.") - - self.set_request_parameters() - self.url = self.portal_url + '/api/1.0/ripe/contact?cidr=%s' self.http_header.update({ "Content-Type": "application/json", @@ -36,12 +27,7 @@ def init(self): "API-Authorization": self.portal_api_key }) - self.session = utils.create_request_session(self) - retries = requests.urllib3.Retry.from_int(self.http_timeout_max_tries) - retries.status_forcelist = [502] - adapter = requests.adapters.HTTPAdapter(max_retries=retries) - self.session.mount('http://', adapter) - self.session.mount('https://', adapter) + self.session = self.http_session() def process(self): event = self.receive_message() diff --git a/intelmq/bots/experts/geohash/expert.py b/intelmq/bots/experts/geohash/expert.py index c5c9ba724..e7a3b8fef 100644 --- a/intelmq/bots/experts/geohash/expert.py +++ b/intelmq/bots/experts/geohash/expert.py @@ -9,6 +9,7 @@ https://github.com/joyanujoy/geolib ''' from intelmq.lib.bot import ExpertBot +from intelmq.lib.exceptions import MissingDependencyError try: from geolib import geohash @@ -23,7 +24,7 @@ class GeohashExpertBot(ExpertBot): def init(self): if not geohash: - raise ValueError("Library 'geolib' is required, please install it.") + raise MissingDependencyError("geolib") def process(self): event = self.receive_message() diff --git a/intelmq/bots/experts/http/expert_content.py b/intelmq/bots/experts/http/expert_content.py index e7521e965..7c1f57c79 100644 --- a/intelmq/bots/experts/http/expert_content.py +++ b/intelmq/bots/experts/http/expert_content.py @@ -7,10 +7,10 @@ from typing import List from intelmq.lib.bot import ExpertBot -from intelmq.lib.utils import create_request_session +from intelmq.lib.mixins import HttpMixin -class HttpContentExpertBot(ExpertBot): +class HttpContentExpertBot(ExpertBot, HttpMixin): """ Test if a given string is part of the content for a given URL @@ -29,8 +29,7 @@ class HttpContentExpertBot(ExpertBot): __session = None def init(self): - self.set_request_parameters() - self.__session = create_request_session(self) + self.__session = self.http_session() def process(self): event = self.receive_message() diff --git a/intelmq/bots/experts/http/expert_status.py b/intelmq/bots/experts/http/expert_status.py index ef920dca8..d6bfeb0a1 100644 --- a/intelmq/bots/experts/http/expert_status.py +++ b/intelmq/bots/experts/http/expert_status.py @@ -8,9 +8,10 @@ from intelmq.lib.bot import ExpertBot from intelmq.lib.utils import create_request_session +from intelmq.lib.mixins import HttpMixin -class HttpStatusExpertBot(ExpertBot): +class HttpStatusExpertBot(ExpertBot, HttpMixin): """ Fetch the HTTP Status for a given URL @@ -31,8 +32,7 @@ def process(self): event = self.receive_message() if self.field in event: - self.set_request_parameters() - session = create_request_session(self) + session = self.http_session() try: response = session.get(event[self.field]) diff --git a/intelmq/bots/experts/national_cert_contact_certat/expert.py b/intelmq/bots/experts/national_cert_contact_certat/expert.py index b7a7035e2..4eb084a88 100644 --- a/intelmq/bots/experts/national_cert_contact_certat/expert.py +++ b/intelmq/bots/experts/national_cert_contact_certat/expert.py @@ -20,30 +20,22 @@ """ from intelmq.lib.bot import ExpertBot +from intelmq.lib.mixins import HttpMixin from intelmq.lib.utils import create_request_session from intelmq.lib.exceptions import MissingDependencyError -try: - import requests -except ImportError: - requests = None - URL = 'https://contacts.cert.at/cgi-bin/abuse-nationalcert.pl' -class NationalCERTContactCertATExpertBot(ExpertBot): +class NationalCERTContactCertATExpertBot(ExpertBot, HttpMixin): """Add country and abuse contact information from the CERT.at national CERT Contact Database. Set filter to true if you want to filter out events for Austria. Set overwrite_cc to true if you want to overwrite an existing country code value""" filter: bool = False http_verify_cert: bool = True overwrite_cc: bool = False def init(self): - if requests is None: - raise MissingDependencyError("requests") - - self.set_request_parameters() - self.session = create_request_session(self) + self.session = self.http_session() def process(self): event = self.receive_message() diff --git a/intelmq/bots/experts/rdap/expert.py b/intelmq/bots/experts/rdap/expert.py index a786b87f8..44aea579c 100644 --- a/intelmq/bots/experts/rdap/expert.py +++ b/intelmq/bots/experts/rdap/expert.py @@ -3,18 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # -*- coding: utf-8 -*- +import requests from intelmq.lib.bot import ExpertBot from intelmq.lib.utils import create_request_session -from intelmq.lib.exceptions import MissingDependencyError -from intelmq.lib.mixins import CacheMixin +from intelmq.lib.mixins import CacheMixin, HttpMixin -try: - import requests -except ImportError: - requests = None - -class RDAPExpertBot(ExpertBot, CacheMixin): +class RDAPExpertBot(ExpertBot, CacheMixin, HttpMixin): """ Get RDAP data""" rdap_order: list = ['abuse', 'technical', 'administrative', 'registrant', 'registrar'] rdap_bootstrapped_servers: dict = {} @@ -30,11 +25,7 @@ class RDAPExpertBot(ExpertBot, CacheMixin): __session: requests.Session def init(self): - if requests is None: - raise MissingDependencyError("requests") - - self.set_request_parameters() - self.__session = create_request_session(self) + self.__session = self.http_session() # get overall rdap data from iana resp = self.__session.get('https://data.iana.org/rdap/dns.json') @@ -73,7 +64,7 @@ def process(self): if result: event.add('source.abuse_contact', result, overwrite=self.overwrite) else: - self.__session = create_request_session(self) + self.__session = self.http_session() domain_parts = url.split('.') domain_suffix = None while domain_suffix is None: diff --git a/intelmq/bots/experts/ripe/expert.py b/intelmq/bots/experts/ripe/expert.py index b347dfe2e..181072975 100644 --- a/intelmq/bots/experts/ripe/expert.py +++ b/intelmq/bots/experts/ripe/expert.py @@ -15,12 +15,7 @@ import intelmq.lib.utils as utils from intelmq.lib.bot import ExpertBot from intelmq.lib.exceptions import MissingDependencyError -from intelmq.lib.mixins import CacheMixin - -try: - import requests -except ImportError: - requests = None +from intelmq.lib.mixins import CacheMixin, HttpMixin STATUS_CODE_ERROR = 'HTTP status code was {}. Possible problem at the connection endpoint or network issue.' @@ -41,7 +36,7 @@ def clean_geo(geo_data): return geo_data -class RIPEExpertBot(ExpertBot, CacheMixin): +class RIPEExpertBot(ExpertBot, CacheMixin, HttpMixin): """Fetch abuse contact and/or geolocation information for the source and/or destination IP addresses and/or ASNs of the events""" mode: str = "append" query_ripe_db_asn: bool = True @@ -77,9 +72,6 @@ class RIPEExpertBot(ExpertBot, CacheMixin): } def init(self): - if requests is None: - raise MissingDependencyError("requests") - self.__query = { "db_asn": self.query_ripe_db_asn, "db_ip": self.query_ripe_db_ip, @@ -87,12 +79,7 @@ def init(self): "stat_ip": self.query_ripe_stat_ip, "stat_geo": self.query_ripe_stat_geolocation, } - - self.__initialize_http_session() - - def __initialize_http_session(self): - self.set_request_parameters() - self.http_session = utils.create_request_session(self) + self.session = self.http_session() def process(self): event = self.receive_message() @@ -134,8 +121,8 @@ def __perform_cached_query(self, type, resource): else: return json.loads(cached_value) else: - response = self.http_session.get(self.QUERY[type].format(resource), - data="", timeout=self.http_timeout_sec) + response = self.session.get(self.QUERY[type].format(resource), + data="", timeout=self.http_timeout_sec) if response.status_code != 200: if type == 'db_asn' and response.status_code == 404: diff --git a/intelmq/bots/experts/splunk_saved_search/expert.py b/intelmq/bots/experts/splunk_saved_search/expert.py index 94d85a0f7..244326328 100644 --- a/intelmq/bots/experts/splunk_saved_search/expert.py +++ b/intelmq/bots/experts/splunk_saved_search/expert.py @@ -56,19 +56,13 @@ intelmq.exceptions.KeyExists. """ - -try: - import requests -except ImportError: - requests = None - -import intelmq.lib.utils as utils +from intelmq.lib.mixins import HttpMixin from intelmq.lib.bot import ExpertBot -from intelmq.lib.exceptions import MissingDependencyError, ConfigurationError +from intelmq.lib.exceptions import ConfigurationError import time -class SplunkSavedSearchBot(ExpertBot): +class SplunkSavedSearchBot(ExpertBot, HttpMixin): """Enrich an event from Splunk search results""" auth_token: str = None multiple_result_handling = ["warn", "use_first", "send"] @@ -83,9 +77,6 @@ class SplunkSavedSearchBot(ExpertBot): _is_multithreadable = False def init(self): - if requests is None: - raise MissingDependencyError("requests") - if self.url is None: raise ConfigurationError("Connection", "No Splunk API URL specified") if self.auth_token is None: @@ -103,11 +94,9 @@ def init(self): if "ignore" in self.multiple_result_handling and "use_first" in self.multiple_result_handling: raise ConfigurationError("Processing", "Cannot both ignore and use multiple search results") - self.set_request_parameters() - self.http_header.update({"Authorization": f"Bearer {self.auth_token}"}) - self.session = utils.create_request_session(self) + self.session = self.http_session() self.session.keep_alive = False def update_event(self, event, search_result): diff --git a/intelmq/bots/experts/tuency/expert.py b/intelmq/bots/experts/tuency/expert.py index 15521cb90..b3139f6f8 100644 --- a/intelmq/bots/experts/tuency/expert.py +++ b/intelmq/bots/experts/tuency/expert.py @@ -17,18 +17,18 @@ {"ip":{"destinations":[{"source":"portal","name":"Thurner","contacts":[{"email":"test@example.vom"}]}]},"domain":{"destinations":[{"source":"portal","name":"Thurner","contacts":[{"email":"abuse@example.at"}]}]},"suppress":true,"interval":{"unit":"immediate","length":1}} """ +from intelmq.lib.mixins import HttpMixin from intelmq.lib.bot import ExpertBot -from intelmq.lib.utils import create_request_session, parse_relative +from intelmq.lib.utils import parse_relative -class TuencyExpertBot(ExpertBot): +class TuencyExpertBot(ExpertBot, HttpMixin): url: str # Path to the tuency instance authentication_token: str overwrite: bool = True def init(self): - self.set_request_parameters() - self.session = create_request_session(self) + self.session = self.http_session() self.session.headers["Authorization"] = f"Bearer {self.authentication_token}" self.url = f"{self.url}intelmq/lookup" diff --git a/intelmq/bots/outputs/elasticsearch/output.py b/intelmq/bots/outputs/elasticsearch/output.py index 85cc608c0..c3a102965 100644 --- a/intelmq/bots/outputs/elasticsearch/output.py +++ b/intelmq/bots/outputs/elasticsearch/output.py @@ -16,6 +16,7 @@ from intelmq.lib.bot import OutputBot from intelmq.lib.exceptions import MissingDependencyError +from intelmq.lib.mixins import HttpMixin try: from elasticsearch import Elasticsearch @@ -55,7 +56,7 @@ def get_event_date(event_dict: dict) -> datetime.date: return event_date -class ElasticsearchOutputBot(OutputBot): +class ElasticsearchOutputBot(OutputBot, HttpMixin): """Send events to an Elasticsearch database server""" elastic_host: str = '127.0.0.1' # TODO: could be ipadd elastic_index: str = 'intelmq' @@ -77,10 +78,10 @@ def init(self): if isinstance(self.flatten_fields, str): self.flatten_fields = self.flatten_fields.split(',') - self.set_request_parameters() # Not all parameters set here are used by the ES client + self.setup() # Not all parameters set here are used by the ES client self.es = Elasticsearch([{'host': self.elastic_host, 'port': self.elastic_port}], - http_auth=self.auth, + http_auth=self.__auth, use_ssl=self.use_ssl, verify_certs=self.http_verify_cert, ca_certs=self.ssl_ca_certificate, diff --git a/intelmq/bots/outputs/restapi/output.py b/intelmq/bots/outputs/restapi/output.py index ef7469703..b966cbfd4 100644 --- a/intelmq/bots/outputs/restapi/output.py +++ b/intelmq/bots/outputs/restapi/output.py @@ -4,18 +4,13 @@ # -*- coding: utf-8 -*- from typing import Iterable +from requests import exceptions -try: - import requests -except ImportError: - requests = None - -import intelmq.lib.utils as utils +from intelmq.lib.mixins.http import HttpMixin from intelmq.lib.bot import OutputBot -from intelmq.lib.exceptions import MissingDependencyError -class RestAPIOutputBot(OutputBot): +class RestAPIOutputBot(OutputBot, HttpMixin): """Send events to a REST API listener through HTTP POST""" auth_token_name: str = None auth_token: str = None @@ -27,10 +22,7 @@ class RestAPIOutputBot(OutputBot): _auth: Iterable[str] = None def init(self): - if requests is None: - raise MissingDependencyError("requests") - - self.set_request_parameters() + self.session = self.http_session() if self.auth_token_name and self.auth_token: if self.auth_type == 'http_header': @@ -40,8 +32,6 @@ def init(self): self.auth = self.auth_token_name, self.auth_token self.http_header.update({"Content-Type": "application/json; charset=utf-8"}) - - self.session = utils.create_request_session(self) self.session.keep_alive = False def process(self): @@ -58,7 +48,7 @@ def process(self): req = self.session.post(self.host, timeout=self.http_timeout_sec, **kwargs) - except requests.exceptions.Timeout: + except exceptions.Timeout: timeoutretries += 1 if req is None and timeoutretries >= self.http_timeout_max_tries: diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 2bfe56756..2ecccdbb3 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -76,12 +76,6 @@ class Bot: error_max_retries: int = 3 error_procedure: str = "pass" error_retry_delay: int = 15 - http_proxy: Optional[str] = None - http_timeout_max_tries: int = 3 - http_timeout_sec: int = 30 - http_user_agent: str = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" - http_verify_cert: Union[bool, str] = True - https_proxy: Optional[str] = None instances_threads: int = 0 load_balance: bool = False log_processed_messages_count: int = 500 @@ -852,37 +846,6 @@ def run(cls, parsed_args=None): if not instance.is_multithreaded: instance.start() - def set_request_parameters(self): - self.http_header: dict = getattr(self, 'http_header', {}) - self.http_verify_cert: bool = getattr(self, 'http_verify_cert', True) - self.ssl_client_cert: Optional[str] = getattr(self, 'ssl_client_certificate', None) - - if (hasattr(self, 'http_username') and - hasattr(self, 'http_password') and - self.http_username): - self.auth = (self.http_username, - self.http_password) - else: - self.auth = None - - if self.http_proxy and self.https_proxy: - self.proxy = {'http': self.http_proxy, - 'https': self.https_proxy} - elif self.http_proxy or self.https_proxy: - self.logger.warning('Only %s_proxy seems to be set.' - 'Both http and https proxies must be set.', - 'http' if self.http_proxy else 'https') - self.proxy = {} - else: - self.proxy = {} - - self.http_timeout_sec: Optional[int] = getattr(self, 'http_timeout_sec', None) - self.http_timeout_max_tries: int = getattr(self, 'http_timeout_max_tries', 1) - # Be sure this is always at least 1 - self.http_timeout_max_tries = self.http_timeout_max_tries if self.http_timeout_max_tries >= 1 else 1 - - self.http_header['User-agent'] = self.http_user_agent - @staticmethod def check(parameters: dict) -> Optional[List[List[str]]]: """ diff --git a/intelmq/lib/mixins/http.py b/intelmq/lib/mixins/http.py index 6a6befb96..ae58d7cde 100644 --- a/intelmq/lib/mixins/http.py +++ b/intelmq/lib/mixins/http.py @@ -7,12 +7,8 @@ `set_request_parameters` in intelmq.lib.bot.Bot """ -from intelmq.lib.exceptions import MissingDependencyError - -try: - import requests -except ImportError: - requests = None +import requests +from typing import Optional class TimeoutHTTPAdapter(requests.adapters.HTTPAdapter): @@ -41,12 +37,12 @@ class HttpMixin: http_header: dict = {} http_user_agent: str = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" http_verify_cert: bool = True - http_proxy = None - https_proxy = None + http_proxy: Optional[str] = None + https_proxy: Optional[str] = None http_timeout_max_tries: int = 3 http_timeout_sec: int = 30 - http_username = None - http_password = None + http_username: str = None + http_password: str = None def __init__(self, **kwargs): self.logger.debug("Running HTTP Mixin initialization.") @@ -55,9 +51,6 @@ def __init__(self, **kwargs): def setup(self): self.logger.debug("Setting up HTTP Mixin.") - if requests is None: - raise MissingDependencyError("requests") - self.__session = requests.Session() # tls settings diff --git a/intelmq/lib/pipeline.py b/intelmq/lib/pipeline.py index cf58b453c..9cb2c21b4 100644 --- a/intelmq/lib/pipeline.py +++ b/intelmq/lib/pipeline.py @@ -7,6 +7,7 @@ from itertools import chain from typing import Dict, Optional import ssl +import requests import redis @@ -18,10 +19,6 @@ try: import pika - try: - import requests - except ImportError: - requests = None except ImportError: pika = None @@ -588,9 +585,6 @@ def _get_queues(self) -> dict: auth = (self.username, self.password) else: auth = ('guest', 'guest') - if requests is None: - self.logger.error("Library 'requests' is needed to get queue status. Please install it.") - return {} response = requests.get(self.monitoring_url + 'api/queues', auth=auth, timeout=5) if response.status_code == 401: diff --git a/intelmq/tests/bots/collectors/github_api/test_collector.py b/intelmq/tests/bots/collectors/github_api/test_collector.py index 6b87ea617..493935803 100644 --- a/intelmq/tests/bots/collectors/github_api/test_collector.py +++ b/intelmq/tests/bots/collectors/github_api/test_collector.py @@ -6,10 +6,11 @@ """ Testing Github API Collectors """ +from cgitb import text import json import os from unittest import TestCase, main as unittest_main -from unittest.mock import MagicMock, patch +import requests_mock import intelmq.lib.exceptions as exceptions import intelmq.lib.test as test @@ -78,24 +79,7 @@ } -def print_requests_get_parameters(url, *args, **kwargs): - if 'headers' in kwargs and kwargs['headers']['Accept'] == 'application/vnd.github.v3.text-match+json': - """ - mocking of Github API requests - """ - main_mock = MagicMock() - main_mock.return_value.json = MagicMock() - main_mock.return_value = RAW_CONTENTS - main_mock.json.return_value = JSON_CONTENTS - return main_mock - else: - """ - mocking of basic GET request - """ - main_mock = MagicMock(content=EXAMPLE_CONTENT_STR) - return main_mock - - +@requests_mock.Mocker() class TestGithubContentsAPICollectorBot(test.BotTestCase, TestCase): """ A TestCase for GithubContentsAPICollectorBot. @@ -105,9 +89,9 @@ class TestGithubContentsAPICollectorBot(test.BotTestCase, TestCase): def set_bot(cls): cls.bot_reference = collector_github_contents_api.GithubContentsAPICollectorBot - @patch('intelmq.bots.collectors.github_api.collector_github_contents_api.requests.get') - def test_message_queue_should_contain_the_right_fields(self, requests_get_mock): - requests_get_mock.side_effect = print_requests_get_parameters + def test_message_queue_should_contain_the_right_fields(self, mocker): + mocker.get("https://api.github.com/repos/{0}/contents".format(SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG']['repository']), status_code=200, json=JSON_CONTENTS) + mocker.get("https://a_download.url/contents.txt", status_code=200, text=EXAMPLE_CONTENT_STR) self.run_bot(parameters=SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG'], prepare=True) @@ -115,27 +99,23 @@ def test_message_queue_should_contain_the_right_fields(self, requests_get_mock): for i in range(len(self.get_output_queue())): self.assertMessageEqual(i, SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['EXPECTED_REPORTS'][i]) - @patch('intelmq.bots.collectors.github_api.collector_github_contents_api.requests.get') - def test_collector_should_fail_with_bad_repository_error(self, requests_get_mock): - import requests - requests_get_mock.side_effect = requests.RequestException() + def test_collector_should_fail_with_bad_repository_error(self, mocker): + mocker.get("https://api.github.com/repos/{0}/contents".format(SHOULD_FAIL_BECAUSE_REPOSITORY_IS_NOT_VALID_CONFIG['CONFIG']['repository'])) self.allowed_error_count = 1 # allow only single and final Error to be raised self.run_bot(parameters=SHOULD_FAIL_BECAUSE_REPOSITORY_IS_NOT_VALID_CONFIG['CONFIG'], prepare=True) self.assertRegexpMatchesLog(pattern=".*Unknown repository.*") # assert the expected ValueError msg - @patch('intelmq.bots.collectors.github_api.collector_github_contents_api.requests.get') - def test_collector_should_fail_with_bad_credentials(self, requests_get_mock): - requests_get_mock.return_value.json = MagicMock(return_value={'message': 'Bad Credentials'}) - requests_get_mock.return_value.configure_mock(status_code=401) + def test_collector_should_fail_with_bad_credentials(self, mocker): + mocker.get("https://api.github.com/repos/{0}/contents".format(SHOULD_FAIL_WITH_BAD_CREDENTIALS['CONFIG']['repository']), status_code=401, json={'message': 'Bad Credentials'}) self.allowed_error_count = 1 self.run_bot(parameters=SHOULD_FAIL_WITH_BAD_CREDENTIALS['CONFIG'], prepare=True) self.assertRegexpMatchesLog(pattern=".*Bad Credentials.*") - @patch('intelmq.bots.collectors.github_api.collector_github_contents_api.requests.get') - def test_adding_extra_fields_should_warn(self, requests_get_mock): - requests_get_mock.side_effect = print_requests_get_parameters + def test_adding_extra_fields_should_warn(self, mocker): + mocker.get("https://api.github.com/repos/{0}/contents".format(SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG']['repository']), status_code=200, json=JSON_CONTENTS) + mocker.get("https://a_download.url/contents.txt", status_code=200, text=EXAMPLE_CONTENT_STR) custom_config = SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG'].copy() custom_config['extra_fields'] = 'aaa,bbb' @@ -153,7 +133,9 @@ def test_adding_extra_fields_should_warn(self, requests_get_mock): "raw": utils.base64_encode(EXAMPLE_CONTENT_STR) }) - def test_collector_init_should_fail_with_invalid_argument(self): + def test_collector_init_should_fail_with_invalid_argument(self, mocker): + mocker.get("https://api.github.com/repos/{0}/contents".format(SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG']['repository'])) + custom_config = SHOULD_PASS_WITH_TXT_FILES_AND_EXTRA_FIELD_SIZE_TEST['CONFIG'].copy() config_with_wrong_regex = custom_config.copy() diff --git a/intelmq/tests/bots/experts/ripe/test_expert.py b/intelmq/tests/bots/experts/ripe/test_expert.py index cf8717994..430a7aeb0 100644 --- a/intelmq/tests/bots/experts/ripe/test_expert.py +++ b/intelmq/tests/bots/experts/ripe/test_expert.py @@ -70,10 +70,10 @@ "source.geolocation.city": "Lansing", "source.geolocation.latitude": 42.7348, "source.geolocation.longitude": -84.6245 - } + } INDEX_ERROR = {"__type": "Event", "source.ip": "228.66.141.189", - } + } @test.skip_internet() class TestRIPEExpertBot(test.BotTestCase, unittest.TestCase): @@ -83,7 +83,7 @@ class TestRIPEExpertBot(test.BotTestCase, unittest.TestCase): def tearDown(self): if self.bot is not None: - self.bot.http_session.close() + pass @classmethod def set_bot(cls): @@ -94,7 +94,7 @@ def set_bot(cls): 'query_ripe_stat_asn': False, 'redis_cache_db': 4, 'query_ripe_stat_geolocation': False, - } + } cls.use_cache = True def test_db_ipv4_lookup(self): @@ -128,7 +128,7 @@ def test_ripe_stat_error_json(self, mocker): 'query_ripe_stat_asn': True, 'query_ripe_stat_ip': True, 'query_ripe_stat_geolocation': False, - } + } self.input_message = EMPTY_INPUT self.prepare_bot(parameters=parameters) old = self.bot.QUERY['stat'] @@ -186,11 +186,11 @@ def test_ripe_db_ip_errors(self, mocker): """ Test RIPE DB IP for errors. """ self.input_message = EXAMPLE_INPUT self.prepare_bot(parameters={'query_ripe_db_asn': False, - 'query_ripe_db_ip': True, - 'query_ripe_stat_ip': False, - 'query_ripe_stat_asn': False, - 'query_ripe_stat_geolocation': False, - }) + 'query_ripe_db_ip': True, + 'query_ripe_stat_ip': False, + 'query_ripe_stat_asn': False, + 'query_ripe_stat_geolocation': False, + }) old = self.bot.QUERY['db_ip'] self.bot.QUERY['db_ip'] = 'http://localhost/{}' mocker.get('/93.184.216.34', status_code=404) @@ -211,7 +211,7 @@ def test_replace(self): 'query_ripe_stat_ip': True, 'query_ripe_stat_asn': False, 'query_ripe_stat_geolocation': False, - }) + }) self.assertMessageEqual(0, EMPTY_REPLACED) self.assertEqual(self.cache.get('stat:127.0.0.1'), b'__no_contact') self.cache.flushdb() # collides with test_ripe_stat_errors @@ -224,7 +224,7 @@ def test_ripe_db_as_404(self): 'query_ripe_stat_ip': False, 'query_ripe_stat_asn': False, 'query_ripe_stat_geolocation': False, - }) + }) self.assertMessageEqual(0, DB_404_AS) @unittest.expectedFailure @@ -234,7 +234,7 @@ def test_geolocation(self): 'query_ripe_db_ip': False, 'query_ripe_stat_ip': False, 'query_ripe_stat_asn': True, - }) + }) self.assertMessageEqual(0, GEOLOCA_OUTPUT1) @unittest.expectedFailure @@ -245,7 +245,7 @@ def test_geolocation_overwrite(self): 'query_ripe_db_ip': False, 'query_ripe_stat_ip': False, 'query_ripe_stat_asn': True, - }) + }) self.assertMessageEqual(0, GEOLOCA_OUTPUT1) @unittest.expectedFailure @@ -255,7 +255,7 @@ def test_geolocation_not_overwrite(self): 'query_ripe_db_ip': False, 'query_ripe_stat_ip': False, 'query_ripe_stat_asn': True, - }) + }) self.assertMessageEqual(0, GEOLOCA_OUTPUT3) def test_index_error(self): diff --git a/intelmq/tests/bots/outputs/restapi/test_output.py b/intelmq/tests/bots/outputs/restapi/test_output.py index e272cf347..571d31abd 100644 --- a/intelmq/tests/bots/outputs/restapi/test_output.py +++ b/intelmq/tests/bots/outputs/restapi/test_output.py @@ -22,6 +22,7 @@ def callback(request, context): return callback +@requests_mock.Mocker() class TestRestAPIOutputBot(test.BotTestCase, unittest.TestCase): @classmethod @@ -36,26 +37,24 @@ def set_bot(cls): cls.default_input_message = {'__type': 'Event', 'source.ip': '10.0.0.1'} - @requests_mock.Mocker() def test_event(self, mocker): """ Test if data is posted correctly to webserver. """ mocker.post('http://localhost/', text=request_callback({'source': {'ip': '10.0.0.1'}}), - request_headers={'Authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=', - 'Content-Type': 'application/json; charset=utf-8'}) + headers={'Authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=', + 'Content-Type': 'application/json; charset=utf-8'}) self.run_bot() - @requests_mock.Mocker() def test_status_check(self, mocker): """ Test if response from webserver is correctly validated. """ mocker.post('http://localhost/', - status_code=500, - request_headers={'Authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=', - 'Content-Type': 'application/json; charset=utf-8'}) + status_code=500, + headers={'Authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=', + 'Content-Type': 'application/json; charset=utf-8'}) self.run_bot(allowed_error_count=1) self.assertLogMatches('requests.exceptions.HTTPError: 500 Server Error: None for url: http://localhost/', 'ERROR')