forked from MISP/PyMISP
-
Notifications
You must be signed in to change notification settings - Fork 0
/
vmray_automation.py
213 lines (172 loc) · 10.1 KB
/
vmray_automation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Koen Van Impe
VMRay automatic import
Put this script in crontab to run every /15 or /60
*/5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py
Calls "vmray_import" for all events that have an 'incomplete' VMray analysis
Do inline config in "main"
'''
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
import argparse
import os
import json
import datetime
import time
import requests
import sys
# Suppress those "Unverified HTTPS request is being made"
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def init(url, key):
'''
Template to get MISP module started
'''
return PyMISP(url, key, misp_verifycert, 'json')
def get_vmray_config(url, key, misp_verifycert, default_wait_period):
'''
Fetch configuration settings from MISP
Includes VMRay API and modules URL
'''
try:
misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
if req.status_code == 200:
req_json = req.json()
if 'finalSettings' in req_json:
finalSettings = req_json['finalSettings']
vmray_api = ''
vmray_url = ''
vmray_wait_period = 0
for el in finalSettings:
# Is the vmray import module enabled?
if el['setting'] == 'Plugin.Import_vmray_import_enabled':
vmray_import_enabled = el['value']
if vmray_import_enabled is False:
break
# Get the VMRay API key from the MISP settings
elif el['setting'] == 'Plugin.Import_vmray_import_apikey':
vmray_api = el['value']
# The VMRay URL to query
elif el['setting'] == 'Plugin.Import_vmray_import_url':
vmray_url = el['value'].replace('/', '\\/')
# MISP modules - Port?
elif el['setting'] == 'Plugin.Import_services_port':
module_import_port = el['value']
# MISP modules - URL
elif el['setting'] == 'Plugin.Import_services_url':
module_import_url = el['value'].replace('\/\/', '//')
# Wait period
elif el['setting'] == 'Plugin.Import_vmray_import_wait_period':
vmray_wait_period = abs(int(el['value']))
if vmray_wait_period < 1:
vmray_wait_period = default_wait_period
else:
sys.exit('Did not receive a 200 code from MISP')
if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url:
return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url}
else:
sys.exit('Did not receive all the necessary configuration information from MISP')
except Exception as e:
sys.exit('Unable to get VMRay config from MISP')
def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete):
'''
Search for the events with VMRay samples that are marked incomplete
and then update these events
'''
controller = 'attributes'
vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP
req = None
# Search for the events
try:
result = m.search(controller, tags=custom_tags_incomplete)
response = result['response']
if len(response) == 0:
sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete)
attribute = response['Attribute']
if len(attribute) == 0:
sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete)
timestamp = int(attribute[0]["timestamp"])
# Not enough time has gone by to lookup the analysis jobs
if int((time.time() - timestamp) / 60) < int(wait_period):
if module_DEBUG:
r_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
print("Attribute to recent for wait_period (%s minutes) - timestamp attribute: %s (%s minutes old)" % (wait_period, r_timestamp, round((int(time.time() - timestamp) / 60), 2)))
return False
if module_DEBUG:
print("All attributes older than %s" % int(wait_period))
for att in attribute:
value = att['value']
if vmray_value in value: # We found a sample ID
att_id = att['id']
att_uuid = att['uuid']
# VMRay Sample IDs are stored as VMRay Sample ID: 2796577
vmray_sample_id = value.split(vmray_value)[1].strip()
if vmray_sample_id.isdigit():
event_id = att['event_id']
if module_DEBUG:
print("Found event %s with matching tags %s for sample id %s " % (event_id, custom_tags_incomplete, vmray_sample_id))
# Prepare request to send to vmray_import via misp modules
misp_modules_url = module_import_url + ':' + module_import_port + '/query'
misp_modules_headers = {'Content-Type': 'application/json'}
misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}'
req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers)
if module_DEBUG and req is not None:
print("Response code from submitting to MISP modules %s" % (req.status_code))
# Succesful response from the misp modules?
if req.status_code == 200:
req_json = req.json()
if "error" in req_json:
print("Error code in reply %s " % req_json["error"])
continue
else:
results = req_json["results"]
# Walk through all results in the misp-module reply
for el in results:
to_ids = True
values = el['values']
types = el['types']
if "to_ids" in el:
to_ids = el['to_ids']
if "text" in types:
to_ids = False
comment = el['comment']
if len(comment) < 1:
comment = "Enriched via the vmray_import module"
# Attribute can belong in different types
for type in types:
try:
r = m.add_named_attribute(event_id, type, values, vmray_attribute_category, to_ids, comment)
if module_DEBUG:
print("Add event %s: %s as %s (%s) (toids: %s)" % (event_id, values, type, comment, to_ids))
except Exception as e:
continue
if module_DEBUG:
print("Unable to add attribute %s as type %s for event %s" % (values, type, event_id))
# Remove 'incomplete' state tags
m.untag(att_uuid, custom_tags_incomplete)
# Update tags to 'complete' state
m.tag(att_uuid, custom_tags_complete)
if module_DEBUG:
print("Updated event %s" % event_id)
else:
sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id, vmray_sample_id))
except Exception as e:
sys.exit("Invalid response received from MISP : %s", e)
if __name__ == '__main__':
module_DEBUG = True
# Set some defaults to be used in this module
vmray_attribute_category = 'External analysis'
vmray_include_analysisid = '0'
vmray_include_imphash_ssdeep = '0'
vmray_include_extracted_files = '0'
vmray_include_analysisdetails = '0'
vmray_include_vtidetails = '0'
custom_tags_incomplete = 'workflow:state="incomplete"'
custom_tags_complete = 'workflow:state="complete"'
default_wait_period = 30
misp = init(misp_url, misp_key)
vmray_config = get_vmray_config(misp_url, misp_key, misp_verifycert, default_wait_period)
search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete)