Merge pull request #83 from ccsplit/master

Make changes to the setup.py originally suggested by @MohitS10

Author: codingo

.travis.yml

@@ -8,6 +8,6 @@ install:
   - pip install -r test-requirements.txt
   - pip install pep8 
 before_script:
-  - pep8 -v *.py lib/
+  - pep8 -v *.py VHostScan/
 script:
   - pytest

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include VHostScan *.txt

README.md

@@ -28,6 +28,9 @@ Install via pip using:
 $ pip install -r requirements.txt
 ```
 
+Or simply run `python setup.py install` and the dependencies should be installed. If there is an issue regarding
+running `python setup.py build_ext`, you will need to reinstall `numpy` using `pip uninstall numpy` and `pip install numpy==1.12.0`. This should resolve the issue as there are sometimes issues with numpy being installed through setup.py.
+
 # Usage
 
 | Argument        | Description |
@@ -65,7 +68,7 @@ _Note that a number of these examples reference 10.10.10.29. This IP refers to B
 The most straightforward example runs the default wordlist against example.com using the default of port 80:
 
 ```bash
-$ VHostScan.py -t example.com
+$ VHostScan -t example.com
 ```
 
 ### Quick Example with SSL
@@ -81,21 +84,21 @@ $ VHostScan.py -t example.com --ssl
 Say you have an SSH port forward listening on port 4444 fowarding traffic to port 80 on example.com's development machine. You could use the following to make VHostScan connect through your SSH tunnel via localhost:4444 but format the header requests to suit connecting straight to port 80:
 
 ```bash
-$ VHostScan.py -t localhost -b example.com -p 4444 -r 80
+$ VHostScan -t localhost -b example.com -p 4444 -r 80
 ```
 
 ### STDIN
 VHostScan Supports piping from other applications and will treat information passed to VHostScan as wordlist data, for example:
 ```bash
-$ cat bank.htb | VHostScan.py -t 10.10.10.29
+$ cat bank.htb | VHostScan -t 10.10.10.29
 ```
 
 ![VHOSTScan STDIN Example](https://github.com/codingo/codingo.github.io/blob/master/assets/Bank%20VHOST%20Pipe%20Example.png)
 
 ### STDIN and WordList
 You can still specify a wordlist to use along with stdin. In these cases wordlist information will be appended to stdin. For example:
 ```bash
-$ echo -e 'a.example.com\b.example.com' | VHostScan.py -t localhost -w ./wordlists/wordlist.txt
+$ echo -e 'a.example.com\b.example.com' | VHostScan -t localhost -w ./wordlists/wordlist.txt
 ```
 ### Fuzzy Logic
 Here is an example with fuzzy logic enabled. You can see the last comparison is much more similar than the first two (it is comparing the content not the actual hashes):
@@ -111,4 +114,11 @@ pip install -r test-requirements.txt
 pytest
 ```
 
+Or you can optionally run:
+
+```bash
+pip install -r test-requirements.txt
+python setup.py test
+```
+
 If you're thinking of adding a new feature to the project, consider also contributing with a couple of tests. A well-tested codebase is a sane codebase. :)

VHostScan.py VHostScan/VHostScan.py

@@ -1,22 +1,19 @@
 #!/usr/bin/python
 
-import os
 import sys
 import dns.resolver
 from argparse import ArgumentParser
 from socket import gethostbyaddr
-from lib.core.virtual_host_scanner import *
-from lib.helpers.output_helper import *
-from lib.helpers.file_helper import load_random_user_agents
-from lib.helpers.wordlist_helper import WordList
-from lib.core.__version__ import __version__
-from lib.input import cli_argument_parser
-
-DEFAULT_WORDLIST_FILE = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)),
-    'wordlists',
-    'virtual-host-scanning.txt'
-)
+from pkg_resources import resource_filename
+from .lib.core.virtual_host_scanner import *
+from .lib.helpers.output_helper import *
+from .lib.helpers.file_helper import load_random_user_agents
+from .lib.helpers.wordlist_helper import WordList
+from .lib.core.__version__ import __version__
+from .lib.input import cli_argument_parser
+
+DEFAULT_WORDLIST_FILE = resource_filename(
+    'VHostScan', 'wordlists/virtual-host-scanning.txt')
 
 
 def print_banner():

VHostScan/__init__.py

@@ -2,4 +2,4 @@
 # |V|H|o|s|t|S|c|a|n|  Developed by @codingo_ & @__timk
 # +-+-+-+-+-+-+-+-+-+  https://github.com/codingo/VHostScan
 
-__version__ = '1.8.2'
+__version__ = '1.8.3'

lib/__init__.py VHostScan/lib/__init__.py

@@ -5,7 +5,7 @@
 import hashlib
 import pandas as pd
 import time
-from lib.core.discovered_host import *
+from .discovered_host import *
 
 import urllib3
 urllib3.disable_warnings()

lib/core/__init__.py VHostScan/lib/core/__init__.py

@@ -1,5 +1,5 @@
-from lib.core.discovered_host import *
-from lib.helpers.file_helper import *
+from ..core.discovered_host import *
+from .file_helper import *
 import time
 from fuzzywuzzy import fuzz
 import itertools

lib/core/__version__.py VHostScan/lib/core/__version__.py

@@ -1,13 +1,10 @@
 import sys
-import os
-from lib.helpers.file_helper import get_combined_word_lists
-
-DEFAULT_WORDLIST_FILE = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)),
-    '../..',
-    'wordlists',
-    'virtual-host-scanning.txt'
-)
+from .file_helper import get_combined_word_lists
+from pkg_resources import resource_filename
+
+
+DEFAULT_WORDLIST_FILE = resource_filename(
+    'VHostScan', 'wordlists/virtual-host-scanning.txt')
 
 
 class WordList:

lib/core/discovered_host.py VHostScan/lib/core/discovered_host.py

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

lib/core/virtual_host_scanner.py VHostScan/lib/core/virtual_host_scanner.py

@@ -0,0 +1,38 @@
+from setuptools import find_packages, setup
+from VHostScan.lib.core.__version__ import __version__
+
+
+def dependencies(file):
+    with open(file) as f:
+        return f.read().splitlines()
+
+
+with open("README.md") as f:
+    try:
+        import numpy
+        num_installed = True
+    except ImportError:
+        num_installed = False
+    setup(
+        name="VHostScan",
+        license="GPLv3",
+        description="A virtual host scanner that performs reverse lookups, "
+                    "can be used with pivot tools, detect catch-all"
+                    "scenarios, aliases and dynamic default pages.",
+        long_description=f.read(),
+        author="codingo",
+        version=__version__,
+        author_email="[email protected]",
+        url="http://github.com/codingo/VHostScan",
+        packages=find_packages(exclude=('tests')),
+        package_data={'VHostScan': ['*.txt']},
+        entry_points={
+            'console_scripts': [
+                'VHostScan = VHostScan.VHostScan:main'
+            ]
+        },
+        install_requires=dependencies('requirements.txt'),
+        setup_requires=['pytest-runner',
+                        '' if num_installed else 'numpy==1.12.0'],
+        tests_require=dependencies('test-requirements.txt'),
+        include_package_data=True)

lib/helpers/__init__.py VHostScan/lib/helpers/__init__.py

@@ -1,5 +1,3 @@
--r requirements.txt
-
 pytest==3.2.3
 pytest-mock==1.6.3
 pep8==1.7.0

lib/helpers/file_helper.py VHostScan/lib/helpers/file_helper.py

@@ -3,7 +3,7 @@
 import sys
 
 from collections import namedtuple
-from lib.helpers.file_helper import parse_word_list_argument, get_combined_word_lists
+from VHostScan.lib.helpers.file_helper import parse_word_list_argument, get_combined_word_lists
 
 WORDLIST_FILES = {
     'simpsons': ['marge', 'bart', 'homer', 'lisa', 'maggie'],

lib/helpers/output_helper.py VHostScan/lib/helpers/output_helper.py

@@ -2,8 +2,8 @@
 import pytest
 from unittest.mock import patch
 
-from lib.helpers.wordlist_helper import WordList
-from lib.helpers.wordlist_helper import DEFAULT_WORDLIST_FILE
+from VHostScan.lib.helpers.wordlist_helper import WordList
+from VHostScan.lib.helpers.wordlist_helper import DEFAULT_WORDLIST_FILE
 
 
 @pytest.fixture(scope='class')
@@ -28,7 +28,7 @@ def test_get_wordlist_from_stdin(self):
         stdin_wordlist = ['keyword1', 'keyword1']
         expected_wordlist = []
         expected_wordlist.extend(stdin_wordlist)
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist()
             self.assertEqual(wordlist, expected_wordlist)
 
@@ -37,59 +37,59 @@ def test_get_wordlist_from_stdin_and_wordlist(self):
         expected_wordlist = []
         expected_wordlist.extend(stdin_wordlist)
         expected_wordlist.extend(self.user_wordlist)
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(self.user_wordlist_file)
             self.assertEqual(wordlist, expected_wordlist)
 
     def test_using_default_wordlist(self):
         stdin_wordlist = []
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist()
             self.assertEqual(wordlist, self.default_wordlist)
 
     def test_ip_using_prefix(self):
         stdin_wordlist = ['127.0.0.1']
         prefix = 'dev-'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(None, prefix)
             self.assertEqual(wordlist, stdin_wordlist)
 
     def test_ip_using_suffix(self):
         stdin_wordlist = ['127.0.0.1']
         suffix = 'test'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix)
             self.assertEqual(wordlist,stdin_wordlist)
 
     def test_word_with_prefix(self):
         stdin_wordlist = ['www','www2','www3']
         expected_wordlist = stdin_wordlist + ['dev-www','dev-www2','dev-www3']
         prefix = 'dev-'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types =  self.wordlist.get_wordlist(None,prefix)
             self.assertEqual(wordlist,expected_wordlist)
 
     def test_words_with_suffix(self):
         stdin_wordlist = ['www','www2','www3']
         expected_wordlist = stdin_wordlist + ['wwwtest','www2test','www3test']
         suffix = 'test'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix)
             self.assertEqual(wordlist, expected_wordlist)
 
     def test_words_with_host_and_prefix(self):
         stdin_wordlist = ['www.%s']
         expected_wordlist = stdin_wordlist + ['test-www.%s']
         prefix = 'test-'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(None, prefix)
             self.assertEqual(wordlist, expected_wordlist)
 
     def test_words_with_host_and_suffix(self):
         stdin_wordlist = ['www.%s']
         expected_wordlist = stdin_wordlist + ['wwwtest.%s']
         suffix = 'test'
-        with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
+        with patch('VHostScan.lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist):
             wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix)
             self.assertEqual(wordlist, expected_wordlist)
 

lib/helpers/wordlist_helper.py VHostScan/lib/helpers/wordlist_helper.py

@@ -1,7 +1,7 @@
 import argparse
 import pytest
 
-from lib.input import cli_argument_parser
+from VHostScan.lib.input import cli_argument_parser
 
 def test_parse_arguments_default_value(tmpdir):
     words = ['word1', 'word2', 'word3']