Merge branch 'master' into master

Author: codingo

README.md

@@ -33,7 +33,7 @@ $ pip install -r requirements.txt
 | -h, --help | Display help message and exit |
 | -t TARGET_HOSTS | Set the target host. |
 | -b BASE_HOST   | Set host to be used during substitution in wordlist (default to TARGET).|
-| -w WORDLIST | Set the wordlist to use (default ./wordlists/virtual-host-scanning.txt) |
+| -w WORDLISTS | Set the wordlist(s) to use. You may specify multiple wordlists in comma delimited format (e.g. -w "./wordlists/simple.txt, ./wordlists/hackthebox.txt" (default ./wordlists/virtual-host-scanning.txt). |
 | -p PORT  | Set the port to use (default 80). |
 | -r REAL_PORT | The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT). |
 | --ignore-http-codes IGNORE_HTTP_CODES | Comma separated list of http codes to ignore with virtual host scans (default 404). |
@@ -43,6 +43,8 @@ $ pip install -r requirements.txt
 | --fuzzy-logic | If set then all unique content replies are compared and a similarity ratio is given for each pair. This helps to isolate vhosts in situations where a default page isn't static (such as having the time on it). |
 | --no-lookups | Disbale reverse lookups (identifies new targets and append to wordlist, on by default). | 
 | --rate-limit | Amount of time in seconds to delay between each scan (default 0). |
+| --random-agent | If set, each scan will use a random user-agent from a predefined list. |
+| --user-agent | Specify a user agent to use for scans. |
 | --waf | If set then simple WAF bypass headers will be sent. |
 | -oN OUTPUT_NORMAL | Normal output printed to a file when the -oN option is specified with a filename argument. |
 | -oJ OUTPUT_JSON | JSON output printed to a file when the -oJ option is specified with a filename argument. |

VHostScan.py

@@ -7,6 +7,7 @@
 from socket import gethostbyaddr
 from lib.core.virtual_host_scanner import *
 from lib.helpers.output_helper import *
+from lib.helpers.file_helper import get_combined_word_lists, load_random_user_agents
 from lib.core.__version__ import __version__
 
 
@@ -20,7 +21,7 @@ def main():
     print_banner()
     parser = ArgumentParser()
     parser.add_argument("-t",   dest="target_hosts", required=True, help="Set a target range of addresses to target. Ex 10.11.1.1-255" )
-    parser.add_argument("-w",   dest="wordlist", required=False, type=str, help="Set the wordlist to use (default ./wordlists/virtual-host-scanning.txt)", default=False)
+    parser.add_argument("-w",   dest="wordlists", required=False, type=str, help="Set the wordlists to use (default ./wordlists/virtual-host-scanning.txt)", default=False)
     parser.add_argument("-b",   dest="base_host", required=False, help="Set host to be used during substitution in wordlist (default to TARGET).", default=False)
     parser.add_argument("-p",   dest="port", required=False, help="Set the port to use (default 80).", default=80)
     parser.add_argument("-r",   dest="real_port", required=False, help="The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT).", default=False)
@@ -32,45 +33,48 @@ def main():
     parser.add_argument("--fuzzy-logic", dest="fuzzy_logic", action="store_true", help="If set then fuzzy match will be performed against unique hosts (default off).", default=False)
     parser.add_argument("--no-lookups", dest="no_lookup", action="store_true", help="Disable reverse lookups (identifies new targets and appends to wordlist, on by default).", default=False)
     parser.add_argument("--rate-limit", dest="rate_limit", type=int, help='Amount of time in seconds to delay between each scan (default 0).', default=0)
+    parser.add_argument('--random-agent', dest='random_agent', action='store_true', help='If set, then each scan will use random user-agent from predefined list.', default=False)
+    parser.add_argument('--user-agent', dest='user_agent', type=str, help='Specify a user-agent to use for scans')
     parser.add_argument("--waf", dest="add_waf_bypass_headers",   action="store_true", help="If set then simple WAF bypass headers will be sent.", default=False)
     parser.add_argument("-oN",   dest="output_normal", help="Normal output printed to a file when the -oN option is specified with a filename argument." )
     parser.add_argument("-oJ", dest="output_json", help="JSON output printed to a file when the -oJ option is specified with a filename argument." )
     parser.add_argument("-", dest="stdin", action="store_true", help="By passing a blank '-' you tell VHostScan to expect input from stdin (pipe).", default=False)
 
     arguments = parser.parse_args()    
-    wordlist = list()
-    
-    if(arguments.stdin and not arguments.wordlist):
+    wordlist = []
+
+    word_list_types = []
+
+    default_wordlist = "./wordlists/virtual-host-scanning.txt" if not arguments.stdin else None
+
+    if arguments.stdin:
+        word_list_types.append('stdin')
         wordlist.extend(list(line for line in sys.stdin.read().splitlines()))
-        print("[+] Starting virtual host scan for %s using port %s and stdin data" % (arguments.target_hosts, 
-                                                                                        str(arguments.port)))
-    elif(arguments.stdin and arguments.wordlist):
-        if not os.path.exists(arguments.wordlist):
-            wordlist.extend(list(line for line in sys.stdin.read().splitlines()))
-            print("[!] Wordlist %s doesn't exist and can't be appended  to stdin." % arguments.wordlist)
-            print("[+] Starting virtual host scan for %s using port %s and stdin data" % (arguments.target_hosts, 
-                                                                                          str(arguments.port)))
-        else:
-            wordlist.extend(list(line for line in open(arguments.wordlist).read().splitlines()))
-            print("[+] Starting virtual host scan for %s using port %s, stdin data, and wordlist %s" % (arguments.target_hosts, 
-                                                                                                        str(arguments.port), 
-                                                                                                        arguments.wordlist))
-    else:
-        if not arguments.wordlist:
-            wordlist.extend(list(line for line in open("./wordlists/virtual-host-scanning.txt").read().splitlines()))
-            print("[+] Starting virtual host scan for %s using port %s and wordlist %s" % ( arguments.target_hosts, 
-                                                                                            str(arguments.port), 
-                                                                                            "./wordlists/virtual-host-scanning.txt"))
-        else:
-            if not os.path.exists(arguments.wordlist):
-                print("[!] Wordlist %s doesn't exist, unable to scan." % arguments.wordlist)
-                sys.exit()
-            else:
-                wordlist.extend(list(line for line in open(arguments.wordlist).read().splitlines()))
-                print("[+] Starting virtual host scan for %s using port %s and wordlist %s" % ( arguments.target_hosts, 
-                                                                                                str(arguments.port), 
-                                                                                                str(arguments.wordlist)))
-        
+
+    combined = get_combined_word_lists(arguments.wordlists or default_wordlist)
+    word_list_types.append('wordlists: {}'.format(
+        ', '.join(combined['file_paths']),
+    ))
+    wordlist.extend(combined['words'])
+
+    if len(wordlist) == 0:
+        print("[!] No words found in provided wordlists, unable to scan.")
+        sys.exit(1)
+
+    print("[+] Starting virtual host scan for {host} using port {port} and {inputs}".format(
+        host=arguments.target_hosts,
+        port=arguments.port,
+        inputs=', '.join(word_list_types),
+    ))
+
+    user_agents = []
+    if arguments.user_agent:
+        print('[>] User-Agent specified, using it')
+        user_agents = [arguments.user_agent]
+    elif arguments.random_agent:
+        print('[>] Random User-Agent flag set')
+        user_agents = load_random_user_agents()
+
     if(arguments.ssl):
         print("[>] SSL flag set, sending all results over HTTPS")
 
@@ -90,7 +94,7 @@ def main():
             wordlist.extend(aliases)
 
     scanner_args = vars(arguments)
-    scanner_args.update({'target': arguments.target_hosts, 'wordlist': wordlist})
+    scanner_args.update({'target': arguments.target_hosts, 'wordlist': wordlist, 'user_agents': user_agents})
     scanner = virtual_host_scanner(**scanner_args)
 
     scanner.scan()

lib/core/virtual_host_scanner.py

@@ -1,10 +1,14 @@
 import os
+import random
+
 import requests
 import hashlib
 import pandas as pd
 import time
 from lib.core.discovered_host import *
 
+DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
+
 
 class virtual_host_scanner(object):
     """Virtual host scanning class
@@ -44,6 +48,9 @@ def __init__(self, target, wordlist, **kwargs):
         # store associated data for discovered hosts in array for oN, oJ, etc'
         self.hosts = []
 
+        # available user-agents
+        self.user_agents = list(kwargs.get('user_agents')) or [DEFAULT_USER_AGENT]
+
     @property
     def ignore_http_codes(self):
         return self._ignore_http_codes
@@ -63,22 +70,19 @@ def scan(self):
         for virtual_host in self.wordlist:
             hostname = virtual_host.replace('%s', self.base_host)
 
+            headers = {
+                'User-Agent': random.choice(self.user_agents),
+                'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
+                'Accept': '*/*'
+            }
+
             if self.add_waf_bypass_headers:
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
-                    'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
-                    'Accept': '*/*',
+                headers.update({
                     'X-Originating-IP': '127.0.0.1',
                     'X-Forwarded-For': '127.0.0.1',
                     'X-Remote-IP': '127.0.0.1',
                     'X-Remote-Addr': '127.0.0.1'
-                }
-            else:
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
-                    'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
-                    'Accept': '*/*'
-                }
+                })
 
             dest_url = '{}://{}:{}/'.format('https' if self.ssl else 'http', self.target, self.port)
 
@@ -115,7 +119,7 @@ def scan(self):
             # add url and hash into array for likely matches
             self.results.append(hostname + ',' + page_hash)
 
-	    #rate limit the connection, if the int is 0 it is ignored
+            #rate limit the connection, if the int is 0 it is ignored
             time.sleep(self.rate_limit)
 
         self.completed_scan=True

lib/helpers/file_helper.py

@@ -25,4 +25,38 @@ def is_json(json_file):
 
     def write_file(self, contents):
         with open(self.output_file, "w") as o:
-            o.write(contents)
+            o.write(contents)
+
+
+def parse_word_list_argument(argument):
+    if not argument:
+        return []
+
+    if ',' in argument:
+        files = [arg.strip() for arg in argument.split(',')]
+    else:
+        files = [argument.strip()]
+
+    return [
+        path for path in files
+        if os.path.exists(path)
+    ]
+
+
+def get_combined_word_lists(argument):
+    files = parse_word_list_argument(argument)
+    words = []
+
+    for path in files:
+        with open(path) as f:
+            words.extend(f.read().splitlines())
+
+    return {
+        'file_paths': files,
+        'words': words,
+    }
+
+
+def load_random_user_agents():
+    with open('./lib/ua-random-list.txt') as f:
+        return f.readlines()

lib/ua-random-list.txt

@@ -0,0 +1,2 @@
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0