version 2.1.5.0 update

LandGrey · LandGrey · commit 88fc9a076630 · 2020-09-14T12:22:00.000+08:00
Add -pattern function
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # pydictor
 
-[![build](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://www.github.com/landgrey/pydictor)  [![Python 2.7&3.4](https://img.shields.io/badge/python-2.7&3.4-yellow.svg)](https://www.python.org/)  ![release](https://img.shields.io/badge/version-2.1.4.1-orange.svg) ![License](https://img.shields.io/badge/license-GPLv3-red.svg)
+[![build](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://www.github.com/landgrey/pydictor)  [![Python 2.7&3.4](https://img.shields.io/badge/python-2.7&3.4-yellow.svg)](https://www.python.org/)  ![release](https://img.shields.io/badge/version-2.1.5.0-orange.svg) ![License](https://img.shields.io/badge/license-GPLv3-red.svg)
 
 
 **README.md [中文版](README_CN.md)**
@@ -71,27 +71,28 @@ python pydictor.py
 #
 #### All of pydictor can generating wordlist
 
-|  type  | wordlist  | identifier | description                              | supported function |
-| :----: | :-------: | :--------: | :--------------------------------------- | :----------------- |
-|  core  |   base    |     C1     | basic wordlist                           | F1 F2 F3 F4        |
-|  core  |   char    |     C2     | custom character wordlist                | F1 F2 F3 F4        |
-|  core  |   chunk   |     C3     | permutation and combination wordlist     | ALL                |
-|  core  |   conf    |     C4     | based on configuration file wordlist     | ALL                |
-|  core  |  extend   |     C5     | extend wordlist based on rules           | ALL                |
-|  core  |   sedb    |     C6     | social engineering wordlist              | ALL                |
-|  tool  | combiner  |     T1     | combine the specify directory files tool |                    |
-|  tool  | comparer  |     T2     | compare two file content difference tool | ALL                |
-|  tool  |  counter  |     T3     | word frequency count tool                | ALL                |
-|  tool  |  handler  |     T4     | handle the input file tool               | ALL                |
-|  tool  | uniqbiner |     T5     | combine and unique the directory files tool | ALL                |
-|  tool  | uniqifer  |     T6     | unique the input file tool               | ALL                |
-|  tool  | hybrider  |     T7     | hybrid couples word list tool            | F1 F2 F3 F4        |
+|  type  | wordlist  | identifier | description                                         | supported function |
+| :----: | :-------: | :--------: | :-------------------------------------------------- | :----------------- |
+|  core  |   base    |     C1     | basic wordlist                                      | F1 F2 F3 F4        |
+|  core  |   char    |     C2     | custom character wordlist                           | F1 F2 F3 F4        |
+|  core  |   chunk   |     C3     | permutation and combination wordlist                | ALL                |
+|  core  |   conf    |     C4     | based on configuration file wordlist                | ALL                |
+|  core  |  pattern  |     C5     | fastly generate pattern wordlist                    | F2 F3 F4           |
+|  core  |  extend   |     C6     | extend wordlist based on rules                      | ALL                |
+|  core  |   sedb    |     C7     | social engineering wordlist                         | ALL                |
+|  tool  | combiner  |     T1     | combine the specify directory files tool            |                    |
+|  tool  | comparer  |     T2     | compare two file content difference tool            | ALL                |
+|  tool  |  counter  |     T3     | word frequency count tool                           | ALL                |
+|  tool  |  handler  |     T4     | handle the input file tool                          | ALL                |
+|  tool  | uniqbiner |     T5     | combine and unique the directory files tool         | ALL                |
+|  tool  | uniqifer  |     T6     | unique the input file tool                          | ALL                |
+|  tool  | hybrider  |     T7     | hybrid couples word list tool                       | F1 F2 F3 F4        |
 | plugin | birthday  |     P1     | birthday keyword wordlist in specify datetime scope | ALL                |
-| plugin |    ftp    |     P2     | against keyword generate ftp password wordlist | ALL                |
-| plugin |   pid4    |     P3     | id card last 4 char wordlist             | ALL                |
-| plugin |   pid6    |     P4     | id card last 6 char wordlist             | ALL                |
-| plugin |   pid8    |     P5     | id card last 8 char wordlist             | ALL                |
-| plugin |  scratch  |     P6     | wordlist based on web pages keywords     | ALL                |
+| plugin |    ftp    |     P2     | against keyword generate ftp password wordlist      | ALL                |
+| plugin |   pid4    |     P3     | id card last 4 char wordlist                        | ALL                |
+| plugin |   pid6    |     P4     | id card last 6 char wordlist                        | ALL                |
+| plugin |   pid8    |     P5     | id card last 8 char wordlist                        | ALL                |
+| plugin |  scratch  |     P6     | wordlist based on web pages keywords                | ALL                |
 
 
 #### function code
diff --git a/README_CN.md b/README_CN.md
@@ -1,5 +1,5 @@
 # pydictor
-[![build](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://www.github.com/landgrey/pydictor)  [![Python 2.7&3.4](https://img.shields.io/badge/python-2.7&3.4-yellow.svg)](https://www.python.org/)  ![release](https://img.shields.io/badge/version-2.1.4.1-orange.svg) ![License](https://img.shields.io/badge/license-GPLv3-red.svg)
+[![build](https://img.shields.io/badge/build-passing-brightgreen.svg)](https://www.github.com/landgrey/pydictor)  [![Python 2.7&3.4](https://img.shields.io/badge/python-2.7&3.4-yellow.svg)](https://www.python.org/)  ![release](https://img.shields.io/badge/version-2.1.5.0-orange.svg) ![License](https://img.shields.io/badge/license-GPLv3-red.svg)
 
 **README.md [English](README.md)**
 
@@ -75,27 +75,28 @@ python pydictor.py
 
 #### pydictor可以生成的所有字典的类型及其说明
 
-|   归属   |    类别     | 标识符  | 描述             | 支持功能代号      |
-| :----: | :-------: | :--: | :------------- | :---------- |
-|  core  |   base    |  C1  | 基础字典           | F1 F2 F3 F4 |
-|  core  |   char    |  C2  | 自定义字符集字典       | F1 F2 F3 F4 |
-|  core  |   chunk   |  C3  | 排列组合字典         | ALL         |
-|  core  |   conf    |  C4  | 配置语法生成字典       | ALL         |
-|  core  |  extend   |  C5  | 规则扩展字典         | ALL         |
-|  core  |   sedb    |  C6  | 社会工程学字典        | ALL         |
-|  tool  | combiner  |  T1  | 字典合并工具         |             |
-|  tool  | comparer  |  T2  | 字典比较相减工具       | ALL         |
-|  tool  |  counter  |  T3  | 词频统计工具         | ALL         |
-|  tool  |  handler  |  T4  | 筛选处理原有字典工具     | ALL         |
-|  tool  | uniqbiner |  T5  | 先合并后去重工具       | ALL         |
-|  tool  | uniqifer  |  T6  | 字典去重工具         | ALL         |
-|  tool  | hybrider  |  T7  | 多字典文件组合工具      | F1 F2 F3 F4 |
-| plugin | birthday  |  P1  | 生日日期字典插件       | ALL         |
-| plugin |    ftp    |  P2  | 关键词生成ftp密码字典插件 | ALL         |
-| plugin |   pid4    |  P3  | 身份证后四位字典插件     | ALL         |
-| plugin |   pid6    |  P4  | 身份证后六位字典插件     | ALL         |
-| plugin |   pid8    |  P5  | 身份证后八位字典插件     | ALL         |
-| plugin |  scratch  |  P6  | 网页原始关键词字典插件    | ALL         |
+|  归属  |   类别    | 标识符 | 描述                      | 支持功能代号 |
+| :----: | :-------: | :----: | :------------------------ | :----------- |
+|  core  |   base    |   C1   | 基础字典                  | F1 F2 F3 F4  |
+|  core  |   char    |   C2   | 自定义字符集字典          | F1 F2 F3 F4  |
+|  core  |   chunk   |   C3   | 排列组合字典              | ALL          |
+|  core  |   conf    |   C4   | 配置语法生成字典          | ALL          |
+|  core  |  pattern  |   C5   | 模式字典快速生成          | F2 F3 F4     |
+|  core  |  extend   |   C6   | 规则扩展字典              | ALL          |
+|  core  |   sedb    |   C7   | 社会工程学字典            | ALL          |
+|  tool  | combiner  |   T1   | 字典合并工具              |              |
+|  tool  | comparer  |   T2   | 字典比较相减工具          | ALL          |
+|  tool  |  counter  |   T3   | 词频统计工具              | ALL          |
+|  tool  |  handler  |   T4   | 筛选处理原有字典工具      | ALL          |
+|  tool  | uniqbiner |   T5   | 先合并后去重工具          | ALL          |
+|  tool  | uniqifer  |   T6   | 字典去重工具              | ALL          |
+|  tool  | hybrider  |   T7   | 多字典文件组合工具        | F1 F2 F3 F4  |
+| plugin | birthday  |   P1   | 生日日期字典插件          | ALL          |
+| plugin |    ftp    |   P2   | 关键词生成ftp密码字典插件 | ALL          |
+| plugin |   pid4    |   P3   | 身份证后四位字典插件      | ALL          |
+| plugin |   pid6    |   P4   | 身份证后六位字典插件      | ALL          |
+| plugin |   pid8    |   P5   | 身份证后八位字典插件      | ALL          |
+| plugin |  scratch  |   P6   | 网页原始关键词字典插件    | ALL          |
 
 #### 字典操作功能及说明对照表  
 
diff --git a/core/PATTERN.py b/core/PATTERN.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# coding:utf-8
+#
+"""
+Copyright (c) 2016-2020 LandGrey (https://github.com/LandGrey/pydictor)
+License: GNU GENERAL PUBLIC LICENSE Version 3
+"""
+
+from __future__ import unicode_literals
+
+import operator
+import itertools
+import functools
+import traceback
+from core.CONF import get_conf_dic
+from lib.fun.filter import encode_filter
+from lib.data.data import pystrs, pyoptions
+from lib.fun.fun import finalsavepath, finishprinter, cool
+from lib.parse.confparse import elementparser, confmatcher
+
+
+def build_pattern_dic(source=""):
+    buffer = []
+    buffer_size = 512
+    storepath = finalsavepath("pattern")
+    pattern_set = patterncore(source)
+    char_list = [sorted(set(i)) for i in pattern_set.values()]
+    try:
+        with open(storepath, "w") as f:
+            for item in map("".join, itertools.product(*char_list)):
+                item = pyoptions.head + item + pyoptions.tail
+                buffer.append(item if pyoptions.encode == "none" else encode_filter(item, encode=pyoptions.encode))
+                if len(buffer) == buffer_size:
+                    f.write(pyoptions.CRLF.join(buffer) + pyoptions.CRLF)
+                    buffer = []
+            f.write(pyoptions.CRLF.join(buffer))
+        finishprinter(storepath)
+    except Exception as e:
+        print(cool.red('[-] Exception as following:') + pyoptions.CRLF)
+        print(traceback.print_exc())
+
+
+def patterncore(resource):
+    pattern_set = {}
+
+    try:
+        confdicts = elementparser(confmatcher(resource))
+    except IndexError:
+        confdicts = {}
+        exit(cool.red("[-] parse element error, please check your parsing element"))
+    finalen = len(confdicts[pystrs.conf_head])
+    for x in range(0, finalen):
+        # pattern_set_list = confdicts[pystrs.conf_char][x]
+        # keep parsing head and tail
+        pattern_set_list = get_conf_dic(int(confdicts[pystrs.conf_minlen][x]),
+                                        int(confdicts[pystrs.conf_maxlen][x]),
+                                        confdicts[pystrs.conf_char][x],
+                                        confdicts[pystrs.conf_encode][x],
+                                        confdicts[pystrs.conf_head][x],
+                                        confdicts[pystrs.conf_tail][x])
+        pattern_set[x] = "".join(pattern_set_list)
+    count = functools.reduce(operator.mul, [len(i) for i in pattern_set.values()], 1)
+    if count >= pyoptions.count_switcher:
+        exit_msg = pyoptions.CRLF + cool.fuchsia("[!] Build items more than pyoptions.count_switcher: %s%s"
+                                                 "[!] Modify /lib/data/data.py count_switcher to adjust it" %
+                                                 (str(pyoptions.count_switcher), pyoptions.CRLF))
+        exit(exit_msg)
+    return pattern_set
diff --git a/docs/doc/usage.md b/docs/doc/usage.md
@@ -25,13 +25,25 @@ python pydictor.py --conf /my/other/awesome.conf
 python pydictor.py --conf "[0-9]{6,6}<none>[a-f,abc,123,!@#]{1,1}<none>" --encode md5 --output parsing.txt
 ```
 
-#### 5. 规则扩展字典
+#### 5. 模式字典快速生成
+
+```
+# note that: 
+# 1. using python3 is fast than python2
+# 2. one element only support single character, like: [***]{1,1}<***>
+
+# generate pattern: abc[%d][%d][%l][%d][%d][%l][%d][%d]
+python3 pydictor.py --head abc --pattern "[0-9]{1,1}<none>[0-9]{1,1}<none>[a-z]{1,1}<none>[0-9]{1,1}<none>[0-9]{1,1}<none>[a-z]{1,1}<none>[0-9]{1,1}<none>[0-9]{1,1}<none>" -o output.txt
+```
+
+#### 6. 规则扩展字典
+
 ```
 python pydictor.py -extend bob --level 4 --len 4 12
 python pydictor.py -extend liwei zwell.com --more --leet 0 1 2 11 21 --level 2 --len 6 16 --occur "<=10" ">0" "<=2" -o /possbile/wordlist.lst
 ```
 
-#### 6. 社会工程学字典
+#### 7. 社会工程学字典
 `python pydictor.py --sedb`
 
 ```
diff --git a/lib/data/data.py b/lib/data/data.py
@@ -2,7 +2,7 @@
 # coding:utf-8
 #
 """
-Copyright (c) 2016-2019 LandGrey (https://github.com/LandGrey/pydictor)
+Copyright (c) 2016-2020 LandGrey (https://github.com/LandGrey/pydictor)
 License: GNU GENERAL PUBLIC LICENSE Version 3
 """
 
@@ -56,7 +56,7 @@ def init_pystrs():
     # start time
     pystrs.startime = time.time()
 
-    pystrs.version = '2.1.4.1#dev'
+    pystrs.version = '2.1.5.0#dev'
 
     # build configuration file element description
     pystrs.conf_head = "head"
@@ -106,7 +106,7 @@ def init_pyoptions():
     pyoptions.maxlen_switcher = 20
 
     # allowed maximum generated items
-    pyoptions.count_switcher = 1000000000000
+    pyoptions.count_switcher = 100000000000
 
     # shredded file rewrite counts
     pyoptions.file_rewrite_count = 1
diff --git a/lib/parse/argsparse.py b/lib/parse/argsparse.py
@@ -11,6 +11,7 @@
 import os
 from lib.fun.fun import cool
 from core.CONF import build_conf_dic
+from core.PATTERN import build_pattern_dic
 from lib.data.data import paths, pyoptions
 
 
@@ -30,6 +31,10 @@ def conf_parser():
         build_conf_dic(source=paths.buildconf_path)
 
 
+def pattern_parser():
+    build_pattern_dic(source=pyoptions.args_pattern)
+
+
 def tool_parser():
     if len(pyoptions.args_tool) >= 1:
         if pyoptions.args_tool[0] in pyoptions.tool_range:
diff --git a/lib/parse/command.py b/lib/parse/command.py
@@ -29,6 +29,7 @@ def parse_args():
            -extend      [string_or_file]
            -plug        [{plugin}]
            --conf       [expression_or_file]
+           --pattern    [expression_or_file]
            --sedb
            -o,--output  [directory]
            -tool        [{tool}]
@@ -76,6 +77,9 @@ def parse_args():
     parser.add_argument('--conf', dest='conf', nargs='?', metavar='file_path', default='default', const='const',
                         help=cool.yellow("Use the configuration string or file build the dictionary"))
 
+    parser.add_argument('--pattern', dest='pattern', nargs='?', metavar='file_path', default='default', const='const',
+                        help=cool.yellow("Use pattern string build the dictionary"))
+
     parser.add_argument('--sedb', dest='sedb', default='',  action="store_true",
                         help=cool.yellow('Enter the Social Engineering Dictionary Builder'))
 
diff --git a/pydictor.py b/pydictor.py
@@ -19,7 +19,7 @@
 from lib.data.data import paths, pyoptions
 from lib.data.text import pydictor_art_text
 from lib.fun.fun import cool
-from lib.parse.argsparse import plug_parser, conf_parser, tool_parser
+from lib.parse.argsparse import plug_parser, conf_parser, pattern_parser, tool_parser
 from lib.parse.command import parse_args
 from lib.parse.tricksparse import sedb_tricks
 
@@ -80,6 +80,7 @@ def init():
     pyoptions.args_plug = args.plug
     pyoptions.args_sedb = args.sedb
     pyoptions.args_conf = args.conf
+    pyoptions.args_pattern = args.pattern
     pyoptions.args_tool = args.tool
     pyoptions.level = args.level
 
@@ -126,5 +127,7 @@ def init():
             exit(e)
     elif pyoptions.args_conf != 'default':
         conf_parser()
+    elif pyoptions.args_pattern != 'default':
+        pattern_parser()
     elif pyoptions.args_tool:
         tool_parser()
