Merge pull request #2 from GEANT/dv_custom_servers/verbose

Support  custom blacklists

README.md

@@ -13,8 +13,7 @@ Nagios or Icinga, Python3 with the following modules:
 ## Usage
 
 ```lang-none
-$ ./check_dnsbl.py --help
-usage: check_dnsbl.py [-h] --host HOST [--warn WARN] [--crit CRIT] [--providers PROVIDERS]
+usage: check_dnsbl.py [-h] --host HOST [--warn WARN] [--crit CRIT] [--providers PROVIDERS] [--verbose]
 
 Check if a hostname/IP address appears in DNS based blacklists
 
@@ -23,4 +22,101 @@ optional arguments:
   --host HOST           the IP/host to check
   --warn WARN, -w WARN  WARN when host appears in this many blacklists. Defaults to 1
   --crit CRIT, -c CRIT  CRIT when host appears in this many blacklists. Defaults to 2
+  --providers PROVIDERS, --blacklists PROVIDERS
+                        Comma or space separated list of DNS blacklist provider hostnames. Defaults to: all.s5h.net, aspews.ext.sorbs.net,
+                        b.barracudacentral.org, bl.nordspam.com, bl.spamcop.net, blackholes.five-ten-sg.com, blacklist.woody.ch, bogons.cymru.com,
+                        cbl.abuseat.org, combined.abuse.ch, combined.rbl.msrbl.net, db.wpbl.info, dnsbl-2.uceprotect.net, dnsbl-3.uceprotect.net,
+                        dnsbl.cyberlogic.net, dnsbl.dronebl.org, dnsbl.sorbs.net, drone.abuse.ch, dul.ru, dyna.spamrats.com, images.rbl.msrbl.net,
+                        ips.backscatterer.org, ix.dnsbl.manitu.net, korea.services.net, matrix.spfbl.net, noptr.spamrats.com,
+                        phishing.rbl.msrbl.net, proxy.bl.gweep.ca, proxy.block.transip.nl, psbl.surriel.com, rbl.interserver.net,
+                        relays.bl.gweep.ca, relays.bl.kundenserver.de, relays.nether.net, residential.block.transip.nl, singular.ttk.pte.hu,
+                        spam.dnsbl.sorbs.net, spam.rbl.msrbl.net, spam.spamrats.com, spambot.bls.digibase.ca, spamlist.or.kr, spamrbl.imp.ch,
+                        spamsources.fabel.dk, ubl.lashback.com, virbl.bit.nl, virus.rbl.msrbl.net, virus.rbl.jp, wormrbl.imp.ch, z.mailspike.net,
+                        zen.spamhaus.org.
+  --verbose, -v         Show verbose output
+```
+
+## Examples
+
+
+```sh
+# Default with just a host
+./check_dnsbl.py --host de-smtp-1.mimecast.com
+OK: None of de-smtp-1.mimecast.com's IP addresses (62.140.10.21, 51.163.159.21) appear on a blacklist
+```
+
+```sh
+# Verbose, will list the used blacklists
+./check_dnsbl.py --host de-smtp-1.mimecast.com --verbose
+OK: None of de-smtp-1.mimecast.com's IP addresses (62.140.10.21, 51.163.159.21) appear on a blacklist 
+Blacklists used:
+
+all.s5h.net
+aspews.ext.sorbs.net
+b.barracudacentral.org
+bl.nordspam.com
+bl.spamcop.net
+blackholes.five-ten-sg.com
+blacklist.woody.ch
+bogons.cymru.com
+cbl.abuseat.org
+combined.abuse.ch
+combined.rbl.msrbl.net
+db.wpbl.info
+dnsbl-2.uceprotect.net
+dnsbl-3.uceprotect.net
+dnsbl.cyberlogic.net
+dnsbl.dronebl.org
+dnsbl.sorbs.net
+drone.abuse.ch
+dul.ru
+dyna.spamrats.com
+images.rbl.msrbl.net
+ips.backscatterer.org
+ix.dnsbl.manitu.net
+korea.services.net
+matrix.spfbl.net
+noptr.spamrats.com
+phishing.rbl.msrbl.net
+proxy.bl.gweep.ca
+proxy.block.transip.nl
+psbl.surriel.com
+rbl.interserver.net
+relays.bl.gweep.ca
+relays.bl.kundenserver.de
+relays.nether.net
+residential.block.transip.nl
+singular.ttk.pte.hu
+spam.dnsbl.sorbs.net
+spam.rbl.msrbl.net
+spam.spamrats.com
+spambot.bls.digibase.ca
+spamlist.or.kr
+spamrbl.imp.ch
+spamsources.fabel.dk
+ubl.lashback.com
+virbl.bit.nl
+virus.rbl.msrbl.net
+virus.rbl.jp
+wormrbl.imp.ch
+z.mailspike.net
+zen.spamhaus.org
+```
+
+```sh
+# Use custom blacklists
+/check_dnsbl.py --host de-smtp-1.mimecast.com --blacklists zen.spamhaus.org,proxy.block.transip.nl -v
+OK: None of de-smtp-1.mimecast.com's IP addresses (62.140.10.21, 51.163.159.21) appear on a blacklist
+Blacklists used:
+
+zen.spamhaus.org
+proxy.block.transip.nl
+```
+
+
+```sh
+# Approximation of the blacklists that are used by mxtoolbox.com
+# See 'mxtoolbox.blacklists.txt'
+./check_dnsbl.py --host outbound2.mail.transip.nl --blacklists 'bl.0spam.org rbl.abuse.ro spam.dnsbl.anonmails.de ips.backscatterer.org b.barracudacentral.org bl.blocklist.de dnsbl.calivent.com.pe v4.fullbogons.cymru.com v6.fullbogons.cymru.com tor.dan.me.uk torexit.dan.me.uk bl.drmx.org dnsbl.dronebl.org spamsources.fabel.dk hostkarma.junkemailfilter.com dnsrbl.imp.ch spamrbl.imp.ch wormrbl.imp.ch uribl.swinog.ch rblspamassassin.interserver.net rbl.interserver.net mail-abuse.blacklist.jippg.org dnsbl.kempt.net ubl.unsubscore.com bl.mailspike.net phishing.rbl.msrbl.net spam.rbl.msrbl.net ix.dnsbl.manitu.net bl.nordspam.com bl.nosolicitado.org psbl.surriel.com all.spamrats.com all.s5h.net rbl.schulte.org backscatter.spameatingmonkey.net bl.spameatingmonkey.net korea.services.net spam.dnsbl.sorbs.net dnsbl.sorbs.net bl.ipv6.spameatingmonkey.net bl.spamcop.net zen.spamhaus.org dnsbl.spfbl.net bl.suomispam.net truncate.gbudb.net dnsbl-1.uceprotect.net dnsbl-2.uceprotect.net dnsbl-3.uceprotect.net blacklist.woody.ch ipv6.blacklist.woody.ch db.wpbl.info dnsbl.zapbl.net'
+WARNING: outbound2.mail.transip.nl's IP address 149.210.149.73 appears in 1 blacklist: hostkarma.junkemailfilter.com
 ```

check_dnsbl.py

@@ -4,6 +4,8 @@
 import argparse
 import socket
 import ipaddress
+import re
+from pprint import pprint
 
 def nagios_exit(message, code):
     print(message)
@@ -17,26 +19,34 @@ def is_ipaddr(string):
         return False
 
 try:
+    from pydnsbl.providers import BASE_PROVIDERS, Provider
+
     parser = argparse.ArgumentParser(description='Check if a hostname/IP address appears in DNS based blacklists')
     parser.add_argument('--host', help='the IP/host to check', required=True)
     parser.add_argument('--warn', '-w',
-                        help='WARN when host appears in this many blacklists. Defaults to 1',
-                        required=False, type=int, default=1)
+            help='WARN when host appears in this many blacklists. Defaults to 1',
+            required=False, type=int, default=1)
     parser.add_argument('--crit', '-c',
-                        help='CRIT when host appears in this many blacklists. Defaults to 2',
-                        required=False, type=int, default=2)
-    # TODO
-    #  parser.add_argument('--providers',
-    #                      help='Comma separated list of DNS blacklist provider hostname. Defaults to the _BASE_PROVIDERS set that is listed at https://github.com/dmippolitov/pydnsbl/blob/master/pydnsbl/providers.py'
-    #                      )
+            help='CRIT when host appears in this many blacklists. Defaults to 2',
+            required=False, type=int, default=2)
+    parser.add_argument('--providers', '--blacklists',
+            help=f"Comma or space separated list of DNS blacklist provider hostnames. Defaults to: {', '.join([p.host for p in BASE_PROVIDERS])}.",
+            default=','.join([p.host for p in BASE_PROVIDERS]),
+            required=False,
+            )
+    parser.add_argument('--verbose', '-v',
+            help='Show verbose output',
+            action="store_true")
 
     args = parser.parse_args()
 
     host = args.host
     warn = args.warn
     crit = args.crit
-    #  providers = args.providers
+    providers = re.split(r',+| +', args.providers)
+    verbose = args.verbose
 
+    #  pprint(providers)
     # Start with a clean slate
     ok_msg = []
     warn_msg = []
@@ -47,10 +57,10 @@ def is_ipaddr(string):
     # Find all IPv4 and IPv6 addresses
     ip_addresses = [a[4][0] for a in socket.getaddrinfo(host=host, port=0, proto=socket.IPPROTO_TCP)]
 
-    checker = pydnsbl.DNSBLIpChecker()
+    checker = pydnsbl.DNSBLIpChecker(providers=[Provider(prov) for prov in providers])
 
     # List of blacklist results per IP
-    results = [p for p in [checker.check(ip) for ip in ip_addresses] if p.blacklisted]
+    results = [p for p in map(checker.check, ip_addresses) if p.blacklisted]
 
     msg = []
     total_hits = 0
@@ -61,7 +71,6 @@ def is_ipaddr(string):
             reported_host = host
         else:
             reported_host = f"{host}'s IP address {result.addr}"
-
         msg.append(f"{reported_host} appears in {len(detected_by)} blacklist{'s' if len(detected_by) > 1 else ''}: {', '.join(list(detected_by.keys()))}")
 
     if total_hits == 1 and crit > warn:
@@ -74,13 +83,18 @@ def is_ipaddr(string):
         else:
             ok_msg.append(f"None of {host}'s IP addresses ({', '.join(ip_addresses)}) appear on a blacklist")
 
+    if verbose:
+        verbose_text = ['\nBlacklists used:\n\n' +'\n'.join(providers)]
+    else:
+        verbose_text = []
+
 except Exception as e:
     nagios_exit("UNKNOWN: Unknown error: {0}.".format(e), 3)
 
 # Exit with accumulated message(s)
 if crit_msg:
-    nagios_exit("CRITICAL: " + ' '.join(crit_msg + warn_msg), 2)
+    nagios_exit("CRITICAL: " + ' '.join(crit_msg + warn_msg + verbose_text), 2)
 elif warn_msg:
-    nagios_exit("WARNING: " + ' '.join(warn_msg), 1)
+    nagios_exit("WARNING: " + ' '.join(warn_msg + verbose_text), 1)
 else:
-    nagios_exit("OK: " + ' '.join(ok_msg), 0)
+    nagios_exit("OK: " + ' '.join(ok_msg + verbose_text), 0)

mxtoolbox.blacklists.txt

@@ -0,0 +1 @@
+bl.0spam.org rbl.abuse.ro spam.dnsbl.anonmails.de ips.backscatterer.org b.barracudacentral.org bl.blocklist.de dnsbl.calivent.com.pe v4.fullbogons.cymru.com v6.fullbogons.cymru.com tor.dan.me.uk torexit.dan.me.uk bl.drmx.org dnsbl.dronebl.org spamsources.fabel.dk hostkarma.junkemailfilter.com dnsrbl.imp.ch spamrbl.imp.ch wormrbl.imp.ch uribl.swinog.ch rblspamassassin.interserver.net rbl.interserver.net mail-abuse.blacklist.jippg.org dnsbl.kempt.net ubl.unsubscore.com bl.mailspike.net phishing.rbl.msrbl.net spam.rbl.msrbl.net ix.dnsbl.manitu.net bl.nordspam.com bl.nosolicitado.org psbl.surriel.com all.spamrats.com all.s5h.net rbl.schulte.org backscatter.spameatingmonkey.net bl.spameatingmonkey.net korea.services.net spam.dnsbl.sorbs.net dnsbl.sorbs.net bl.ipv6.spameatingmonkey.net bl.spamcop.net zen.spamhaus.org dnsbl.spfbl.net bl.suomispam.net truncate.gbudb.net dnsbl-1.uceprotect.net dnsbl-2.uceprotect.net dnsbl-3.uceprotect.net blacklist.woody.ch ipv6.blacklist.woody.ch db.wpbl.info dnsbl.zapbl.net