Skip to content

Commit 4eb0bbb

Browse files
fedelemantuanofedelemantuano
committed
Merge branch 'release/2.7.0'
2 parents 66732e6 + 93ab90b commit 4eb0bbb

27 files changed

+378
-117
lines changed

‎.github/ISSUE_TEMPLATE/bug_report.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
---
2+
name: Bug report
3+
about: Create a report to help us improve
4+
5+
---
6+
7+
**Describe the bug**
8+
A clear and concise description of what the bug is:
9+
- which bolt/spout is in error
10+
- Apache Storm error log
11+
- ...
12+
13+
**To Reproduce**
14+
Steps to reproduce the behavior:
15+
1. ...
16+
2. ...
17+
18+
Attach main configuration file of `SpamScope`.
19+
20+
**Expected behavior**
21+
A clear and concise description of what you expected to happen.
22+
23+
**Raw mail**
24+
The raw mail to reproduce the behavior.
25+
You can use a `gist` like [this](https://gist.github.com/fedelemantuano/5dd702004c25a46b2bd60de21e67458e).
26+
The issues without raw mail will be closed.
27+
28+
**Environment:**
29+
- OS: [e.g. Debian, Centos]
30+
- Docker: [yes or no]
31+
- `SpamScope` version [e.g. 3.6.0]
32+
33+
**Additional context**
34+
Add any other context about the problem here.

‎.github/ISSUE_TEMPLATE/feature_request.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
name: Feature request
3+
about: Suggest an idea for this project
4+
5+
---
6+
7+
**Is your feature request related to a problem? Please describe.**
8+
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
9+
10+
**Describe the solution you'd like**
11+
A clear and concise description of what you want to happen.
12+
13+
**Describe alternatives you've considered**
14+
A clear and concise description of any alternative solutions or features you've considered.
15+
16+
**Additional context**
17+
Add any other context or screenshots about the feature request here.

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ _resources
66
.env
77
.idea/
88
.ropeproject
9+
.tox/
910
.vscode/
1011
*.pyc
1112
build/
1213
dist/
1314
logs/
15+
report/
1416
SpamScope.egg-info/
1517
venv/

‎.travis.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ python:
1212
env:
1313
- TIKA_APP_JAR=/tmp/tika-app-${TIKA_VER}.jar
1414
FAUP_PATH=/tmp/faup
15-
ZEMANA_PATH=/tmp/zemana
1615
DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch
1716

1817
before_install:
@@ -48,7 +47,6 @@ install:
4847
- pip install --upgrade pip setuptools
4948
- python setup.py install
5049
- pip install -r requirements_optional.txt
51-
- git clone https://$BITBUCKET_USER:[email protected]/$BITBUCKET_USER/zemana-api.git $ZEMANA_PATH && cd $ZEMANA_PATH && python setup.py install && cd -
5250
- src/cli/faup.sh
5351
- pip install coveralls
5452

‎Makefile

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
.PHONY: clean clean-test clean-pyc clean-build docs help
2+
.DEFAULT_GOAL := help
3+
4+
define BROWSER_PYSCRIPT
5+
import os, webbrowser, sys
6+
7+
try:
8+
from urllib import pathname2url
9+
except:
10+
from urllib.request import pathname2url
11+
12+
webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
13+
endef
14+
export BROWSER_PYSCRIPT
15+
16+
define PRINT_HELP_PYSCRIPT
17+
import re, sys
18+
19+
for line in sys.stdin:
20+
match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
21+
if match:
22+
target, help = match.groups()
23+
print("%-20s %s" % (target, help))
24+
endef
25+
export PRINT_HELP_PYSCRIPT
26+
27+
BROWSER := python -c "$$BROWSER_PYSCRIPT"
28+
29+
help:
30+
@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
31+
32+
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
33+
34+
clean-build: ## remove build artifacts
35+
rm -fr build/
36+
rm -fr dist/
37+
rm -fr .eggs/
38+
find . -name '*.egg-info' -exec rm -fr {} +
39+
find . -name '*.egg' -exec rm -f {} +
40+
41+
clean-pyc: ## remove Python file artifacts
42+
find . -name '*.pyc' -exec rm -f {} +
43+
find . -name '*.pyo' -exec rm -f {} +
44+
find . -name '*~' -exec rm -f {} +
45+
find . -name '__pycache__' -exec rm -fr {} +
46+
47+
clean-test: ## remove test and coverage artifacts
48+
rm -fr .tox/
49+
rm -f .coverage
50+
rm -fr htmlcov/
51+
rm -fr .pytest_cache
52+
53+
lint: ## check style with flake8
54+
flake8 src tests
55+
56+
test: ## run tests quickly with the default Python
57+
python -m unittest discover -s tests -f -v
58+
59+
test-all: ## run tests on every Python version with tox
60+
tox
61+
62+
# docs: ## generate Sphinx HTML documentation, including API docs
63+
# rm -f docs/mailparser.rst
64+
# rm -f docs/modules.rst
65+
# sphinx-apidoc -o docs/ mailparser
66+
# $(MAKE) -C docs clean
67+
# $(MAKE) -C docs html
68+
# $(BROWSER) docs/_build/html/index.html
69+
70+
# servedocs: docs ## compile the docs watching for changes
71+
# watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
72+
73+
release: dist ## package and upload a release
74+
twine upload dist/*
75+
76+
dist: clean ## builds source and wheel package
77+
python setup.py sdist
78+
python setup.py bdist_wheel
79+
ls -l dist
80+
81+
install: clean ## install the package to the active Python's site-packages
82+
python setup.py install

‎ansible/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
ansible==2.5.0
1+
ansible

‎ansible/templates/spamscope.yml.j2

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ phishing:
5151

5252

5353
tokenizer:
54+
# Persistent where store dumps of hashes.
55+
persistent_path: /tmp
56+
5457
# If true mails with same hash are filtered and not analyzed.
5558
# Only the body will not saved
5659
filter_mails: true
@@ -84,19 +87,37 @@ network:
8487
enabled: false
8588
api_key: xxxxxxxxxxxxxxxxxxxxxxxxxx
8689

87-
8890
# RawMail bolt configuration
8991
raw_mail:
9092
# SpamAssassin analysis: https://spamassassin.apache.org/
9193
spamassassin:
9294
enabled: false
9395

94-
96+
# Dialect analysis: https://sissden.eu/blog/analysis-of-smtp-dialects
97+
dialect:
98+
enabled: false
99+
100+
# elasticsearch instance where are postfix logs
101+
elasticsearch:
102+
hosts:
103+
- "node1:9200"
104+
- "node2"
105+
106+
# Prefix with dash '-' of Postfix index in Elasticsearch
107+
# The format of indices should be postfix-2018.12.30
108+
index.prefix.postfix: postfix-
109+
95110
# Attachments bolt configuration
96111
attachments:
97-
# The lists of all components must be under lists keyword to load them
98-
# automatically
99112
commons:
113+
# enable or disable filter on size
114+
size.filter.enabled: false
115+
116+
# max size to analyze in bytes
117+
max.size: 3145728
118+
119+
# The lists of all components must be under lists keyword to load them
120+
# automatically
100121
lists:
101122
blacklist_content_types:
102123
# All content types to remove from results

‎conf/spamscope.example.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ raw_mail:
113113
# Dialect analysis: https://sissden.eu/blog/analysis-of-smtp-dialects
114114
dialect:
115115
enabled: false
116-
116+
117117
# elasticsearch instance where are postfix logs
118118
elasticsearch:
119119
hosts:
@@ -127,14 +127,15 @@ raw_mail:
127127

128128
# Attachments bolt configuration
129129
attachments:
130-
# The lists of all components must be under lists keyword to load them
131-
# automatically
132130
commons:
133131
# enable or disable filter on size
134132
size.filter.enabled: false
133+
135134
# max size to analyze in bytes
136135
max.size: 3145728
137136

137+
# The lists of all components must be under lists keyword to load them
138+
# automatically
138139
lists:
139140
blacklist_content_types:
140141
# All content types to remove from results

‎conf/templates/commons.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"settings": {
55
"index.codec": "best_compression",
66
"index.number_of_replicas": 0,
7+
"index.number_of_shards": 1,
78
"index.refresh_interval": "5s",
89
"index.mapping.total_fields.limit": 100000,
910
"index.mapping.ignore_malformed": true

‎conf/templates/spamscope_attachments.json

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,44 @@
11
{
22
"order": 0,
3-
"version": 2,
4-
"index_patterns": "spamscope_attachments-*",
3+
"version": 3,
4+
"index_patterns": [
5+
"spamscope_attachments-*"
6+
],
57
"settings": {
6-
"analysis": {
7-
"analyzer": {
8-
"header": {
9-
"tokenizer": "uax_url_email",
10-
"filter": [
11-
"lowercase"
12-
]
13-
},
14-
"html_body": {
15-
"char_filter": [
16-
"html_strip"
17-
],
18-
"tokenizer": "uax_url_email",
19-
"filter": [
20-
"lowercase"
21-
]
22-
},
23-
"path_pattern": {
24-
"tokenizer": "path_hierarchy",
25-
"filter": [
26-
"lowercase"
27-
]
8+
"index": {
9+
"codec": "best_compression",
10+
"mapping": {
11+
"ignore_malformed": "true"
12+
},
13+
"refresh_interval": "5s",
14+
"analysis": {
15+
"analyzer": {
16+
"header": {
17+
"filter": [
18+
"lowercase"
19+
],
20+
"tokenizer": "uax_url_email"
21+
},
22+
"html_body": {
23+
"filter": [
24+
"lowercase"
25+
],
26+
"char_filter": [
27+
"html_strip"
28+
],
29+
"tokenizer": "uax_url_email"
30+
},
31+
"path_pattern": {
32+
"filter": [
33+
"lowercase"
34+
],
35+
"tokenizer": "path_hierarchy"
36+
}
2837
}
29-
}
30-
},
31-
"index.codec": "best_compression",
32-
"index.number_of_shards": 1,
33-
"index.number_of_replicas": 0,
34-
"index.refresh_interval": "5s",
35-
"index.mapping.ignore_malformed": true
38+
},
39+
"number_of_shards": "1",
40+
"number_of_replicas": "0"
41+
}
3642
},
3743
"mappings": {
3844
"_doc": {
@@ -86,5 +92,29 @@
8692
}
8793
}
8894
}
95+
},
96+
"aliases": {
97+
"attachments": {},
98+
"attachments_thug": {
99+
"filter": {
100+
"exists": {
101+
"field": "thug"
102+
}
103+
}
104+
},
105+
"attachments_tika": {
106+
"filter": {
107+
"exists": {
108+
"field": "tika"
109+
}
110+
}
111+
},
112+
"attachments_virustotal": {
113+
"filter": {
114+
"exists": {
115+
"field": "virustotal"
116+
}
117+
}
118+
}
89119
}
90-
}
120+
}

0 commit comments

Comments
 (0)