Merge pull request #48 from DeKaN/master

Add support for random user agent and specified agent (#41)

Author: codingo

README.md

@@ -43,6 +43,8 @@ $ pip install -r requirements.txt
 | --fuzzy-logic | If set then all unique content replies are compared and a similarity ratio is given for each pair. This helps to isolate vhosts in situations where a default page isn't static (such as having the time on it). |
 | --no-lookups | Disbale reverse lookups (identifies new targets and append to wordlist, on by default). | 
 | --rate-limit | Amount of time in seconds to delay between each scan (default 0). |
+| --random-agent | If set, each scan will use a random user-agent from a predefined list. |
+| --user-agent | Specify a user agent to use for scans. |
 | --waf | If set then simple WAF bypass headers will be sent. |
 | -oN OUTPUT_NORMAL | Normal output printed to a file when the -oN option is specified with a filename argument. |
 | - | By passing a blank '-' you tell VHostScan to expect input from stdin (pipe). |

VHostScan.py

@@ -7,7 +7,7 @@
 from socket import gethostbyaddr
 from lib.core.virtual_host_scanner import *
 from lib.helpers.output_helper import *
-from lib.helpers.file_helper import get_combined_word_lists
+from lib.helpers.file_helper import get_combined_word_lists, load_random_user_agents
 from lib.core.__version__ import __version__
 
 
@@ -33,6 +33,8 @@ def main():
     parser.add_argument("--fuzzy-logic", dest="fuzzy_logic", action="store_true", help="If set then fuzzy match will be performed against unique hosts (default off).", default=False)
     parser.add_argument("--no-lookups", dest="no_lookup", action="store_true", help="Disable reverse lookups (identifies new targets and appends to wordlist, on by default).", default=False)
     parser.add_argument("--rate-limit", dest="rate_limit", type=int, help='Amount of time in seconds to delay between each scan (default 0).', default=0)
+    parser.add_argument('--random-agent', dest='random_agent', action='store_true', help='If set, then each scan will use random user-agent from predefined list.', default=False)
+    parser.add_argument('--user-agent', dest='user_agent', type=str, help='Specify a user-agent to use for scans')
     parser.add_argument("--waf", dest="add_waf_bypass_headers",   action="store_true", help="If set then simple WAF bypass headers will be sent.", default=False)
     parser.add_argument("-oN",   dest="output_normal", help="Normal output printed to a file when the -oN option is specified with a filename argument." )
     parser.add_argument("-", dest="stdin", action="store_true", help="By passing a blank '-' you tell VHostScan to expect input from stdin (pipe).", default=False)
@@ -64,6 +66,14 @@ def main():
         inputs=', '.join(word_list_types),
     ))
 
+    user_agents = []
+    if arguments.user_agent:
+        print('[>] User-Agent specified, using it')
+        user_agents = [arguments.user_agent]
+    elif arguments.random_agent:
+        print('[>] Random User-Agent flag set')
+        user_agents = load_random_user_agents()
+
     if(arguments.ssl):
         print("[>] SSL flag set, sending all results over HTTPS")
 
@@ -83,7 +93,7 @@ def main():
             wordlist.extend(aliases)
 
     scanner_args = vars(arguments)
-    scanner_args.update({'target': arguments.target_hosts, 'wordlist': wordlist})
+    scanner_args.update({'target': arguments.target_hosts, 'wordlist': wordlist, 'user_agents': user_agents})
     scanner = virtual_host_scanner(**scanner_args)
 
     scanner.scan()

lib/core/__version__.py

@@ -2,5 +2,5 @@
 # |V|H|o|s|t|S|c|a|n|  Developed by @codingo_ & @__timk
 # +-+-+-+-+-+-+-+-+-+  https://github.com/codingo/VHostScan
 
-__version__ = '1.3'
+__version__ = '1.4'
 

lib/core/virtual_host_scanner.py

@@ -1,10 +1,14 @@
 import os
+import random
+
 import requests
 import hashlib
 import pandas as pd
 import time
 from lib.core.discovered_host import *
 
+DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
+
 
 class virtual_host_scanner(object):
     """Virtual host scanning class
@@ -44,6 +48,9 @@ def __init__(self, target, wordlist, **kwargs):
         # store associated data for discovered hosts in array for oN, oJ, etc'
         self.hosts = []
 
+        # available user-agents
+        self.user_agents = list(kwargs.get('user_agents')) or [DEFAULT_USER_AGENT]
+
     @property
     def ignore_http_codes(self):
         return self._ignore_http_codes
@@ -63,22 +70,19 @@ def scan(self):
         for virtual_host in self.wordlist:
             hostname = virtual_host.replace('%s', self.base_host)
 
+            headers = {
+                'User-Agent': random.choice(self.user_agents),
+                'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
+                'Accept': '*/*'
+            }
+
             if self.add_waf_bypass_headers:
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
-                    'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
-                    'Accept': '*/*',
+                headers.update({
                     'X-Originating-IP': '127.0.0.1',
                     'X-Forwarded-For': '127.0.0.1',
                     'X-Remote-IP': '127.0.0.1',
                     'X-Remote-Addr': '127.0.0.1'
-                }
-            else:
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
-                    'Host': hostname if self.real_port == 80 else '{}:{}'.format(hostname, self.real_port),
-                    'Accept': '*/*'
-                }
+                })
 
             dest_url = '{}://{}:{}/'.format('https' if self.ssl else 'http', self.target, self.port)
 
@@ -115,8 +119,8 @@ def scan(self):
             # add url and hash into array for likely matches
             self.results.append(hostname + ',' + page_hash)
 
-	    #rate limit the connection, if the int is 0 it is ignored
-	    time.sleep(self.rate_limit)
+        #rate limit the connection, if the int is 0 it is ignored
+        time.sleep(self.rate_limit)
 
         self.completed_scan=True
 

lib/helpers/file_helper.py

@@ -55,3 +55,8 @@ def get_combined_word_lists(argument):
         'file_paths': files,
         'words': words,
     }
+
+
+def load_random_user_agents():
+    with open('./lib/ua-random-list.txt') as f:
+        return f.readlines()

lib/ua-random-list.txt

@@ -0,0 +1,2 @@
+Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0