tests elasticsearch: use service and apply mapping

use the elasticsearch container as service instead of installing
elasticsearch from packages manually
run the elasticmapper to create the mapping

small fixes in docs
add a mapping in the elasticmapper

.github/workflows/scripts/setup-full.sh

@@ -5,15 +5,6 @@
 set -x
 set -e
 
-# Set up and start elasticsearch
-curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.1-amd64.deb
-sudo dpkg -i --force-confnew elasticsearch-7.6.1-amd64.deb
-sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options
-sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options
-echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options
-sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
-sudo systemctl start elasticsearch
-
 sudo apt update
 if [ $python_version == '3.8' ]; then
 	# for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539
@@ -54,5 +45,8 @@ intelmq_psql_initdb
 sed -i 's/events/tests/g' /tmp/initdb.sql
 psql -w -v ON_ERROR_STOP=on -d postgresql://intelmq@localhost/intelmq -f /tmp/initdb.sql
 
+# Initialize the elasticsearch database
+./contrib/elasticsearch/elasticmapper --harmonization-file intelmq/etc/harmonization.conf
+
 # Import the OpenPGP test key
 gpg --import intelmq/tests/assets/key-public.pgp

.github/workflows/unittests.yml

@@ -42,6 +42,10 @@ jobs:
         image: mongo
         ports:
           - 27017:27017
+      elasticsearch:
+        image: elasticsearch:7.6.1
+        ports:
+          - 9200:9200
 
     steps:
     - name: Checkout repository

contrib/elasticsearch/elasticmapper

@@ -22,6 +22,7 @@ TYPES = {
     "ASN": "integer",
     "Base64": "text",
     "Boolean": "boolean",
+    "ClassificationTaxonomy": "keyword",
     "ClassificationType": "keyword",
     "DateTime": "date",
     "Float": "long",

docs/user/bots.md

@@ -4420,8 +4420,7 @@ This bot outputs to Elasticsearch.
 
 Only ElasticSearch version 7 supported.
 
-It is also possible to feed data into ElasticSearch using ELK-Stack via Redis and Logstash, see `ELK-Stack`
-{.interpreted-text role="doc"} for more information. This methods supports various different versions of ElasticSearch.
+It is also possible to feed data into ElasticSearch using ELK-Stack via Redis and Logstash, see [ELK-Stack](../../admin/database/elasticsearch) for more information. This methods supports various different versions of ElasticSearch.
 
 **Parameters:**
 
@@ -4493,7 +4492,7 @@ indices at once, use an alias (<https://www.elastic.co/guide/en/elasticsearch/re
 The data in ES can be retrieved with the HTTP-Interface:
 
 ```bash
- curl -XGET 'http://localhost:9200/intelmq/events/_search?pretty=True'
+ curl -XGET 'http://localhost:9200/intelmq/events/_search?pretty=true'
 ```
 
 ---